2019-12-07T19:33:20.813+0000 7f4cd91d6bc0 0 ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev), process ceph-osd, pid 34109 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) set_cache_shards 1 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 -1 auth: error reading file: /var/lib/ceph/osd/ceph-7/keyring: can't open /var/lib/ceph/osd/ceph-7/keyring: (2) No such file or directory 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 -1 created new key in keyring /var/lib/ceph/osd/ceph-7/keyring 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) mkfs path /var/lib/ceph/osd/ceph-7 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:20.816+0000 7f4cd91d6bc0 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 -1 bluestore(/var/lib/ceph/osd/ceph-7) _read_fsid unparsable uuid 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) mkfs generated fsid 0f8b5d06-9920-4212-962e-b5b150d77eaa 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _setup_block_symlink_or_file name block path size 96636764160 create=1 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _setup_block_symlink_or_file resized block file to 90 GiB 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _setup_block_symlink_or_file name block.wal path size 100663296 create=0 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _setup_block_symlink_or_file name block.db path size 0 create=0 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 0 meta) 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:20.817+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 bluefs _add_block_extent bdev 1 0xacccc0000~e6670000 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _add_block_extent done 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log want_seq 0 txn(seq 0 len 0x0 crc 0xffffffff) not dirty, dirty_files empty, no-op 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 bluefs mkfs osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs _init_alloc 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_add_free done 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 bluefs mkfs uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x400000 from 0 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x400000 from 1 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate 0x400000/10000,400000,0 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate extent: 0xacccc0000~400000/10000,400000,0 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs mkfs op_alloc_add 1 0xacccc0000~e6670000 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0x0 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x0 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0x1000 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000015 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 1 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _write_super super block length(encoded): 102 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _write_super superblock 1 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _write_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs _write_super v 1 crc 0xa2c1798c offset 0x1000 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 0 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 20 bluefs _stop_alloc 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700e900 shutdown 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs mkfs success 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 1 bluefs mount 2019-12-07T19:33:20.818+0000 7f4cd91d6bc0 10 bluefs _open_super 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 20 bluefs _init_alloc 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 10 bluefs _replay 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.819+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_add_free done 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs _replay 0x1000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs _replay log file size was 0x1000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs _replay done 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free done 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs mount log write pos set to 0x1000 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs mkdir db 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs mkdir db.slow 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options using custom Env 0x55ed97c47b60 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:20.820+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Write Ahead Log file in db.wal: 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.create_if_missing: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.env: 0x55ed97c47b60 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.info_log: 0x55ed97c46f40 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_manager: 0x55ed97c25d10 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs mkdir db 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs mkdir db 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs lock_file dir db (0x55ed97c4aa20) file LOCK not found, creating 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) with 0x55ed96fda2c0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs stat dir db (0x55ed97c4aa20) file CURRENT not found 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_open.cc:228] Creating manifest 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs open_for_write db/MANIFEST-000001 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/MANIFEST-000001 vsel_hint 0x1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97181900 on file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 13 < min_flush_size 524288 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 13 < min_flush_size 524288 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97181900 file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0x0~d to file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0x0 0x0~d to file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _allocate len 0xd from 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate extent: 0xacd0c0000~10000/10000,10000,0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 2 (was clean) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd0c0000~10000 x_off 0x0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0xd and padding block with 0xff3 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0xd 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (2) on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]), flushing log 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x1000~1000 to file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x1000 0x1000~1000 to file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x2000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000013 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 2 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _truncate 0xd file file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs open_for_write db/000001.dbtmp 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/000001.dbtmp vsel_hint 0x1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97181900 on file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97181900 file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0x0~10 to file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0x0 0x0~10 to file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 3 (was clean) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0x10 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (3) on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x2000~1000 to file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x2000 0x2000~1000 to file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x3000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000015 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 3 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs rename db/000001.dbtmp -> db/CURRENT 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs rename db/CURRENT file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs sync_metadata 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x3000~1000 to file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x3000 0x3000~1000 to file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x4000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000017 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 4 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs sync_metadata done in 0.000085 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x4000 expected 3000 ratio 1.33333 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs stat dir db (0x55ed97c4aa20) file IDENTITY not found 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs open_for_write db/000000.dbtmp 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/000000.dbtmp vsel_hint 0x1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97181900 on file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97181900 file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0x0~25 to file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0x0 0x0~25 to file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x25 from 1 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate extent: 0xacd0e0000~10000/10000,10000,0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 5 (was clean) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd0e0000~10000 x_off 0x0 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x25 and padding block with 0xfdb 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0x25 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (5) on file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]), flushing log 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x4000~1000 to file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x4000 0x4000~1000 to file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x5000 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000015 2019-12-07T19:33:20.821+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 5 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs rename db/000000.dbtmp -> db/IDENTITY 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs rename db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf290 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ed96fda268) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55ed9700ed10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs mkdir db 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_write db.wal/000003.log 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db.wal/000003.log vsel_hint 0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97181900 on file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs sync_metadata 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x5000~1000 to file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x5000 0x5000~1000 to file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x5000 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x6000 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000016 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 6 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs sync_metadata done in 0.000090 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x6000 expected 3000 ratio 2 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_write db/OPTIONS-000004.dbtmp 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/OPTIONS-000004.dbtmp vsel_hint 0x1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97181cc0 on file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181cc0 ignoring, length 4948 < min_flush_size 524288 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97181cc0 file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181cc0 0x0~1354 to file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181cc0 pos 0x0 0x0~1354 to file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x1354 from 1 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate extent: 0xacd0f0000~10000/10000,10000,0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 7 (was clean) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x0 2019-12-07T19:33:20.822+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x354 and padding block with 0xcac 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181cc0 pos now 0x1354 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181cc0 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181cc0 done in 0.000000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (7) on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]), flushing log 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x6000~1000 to file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x6000 0x6000~1000 to file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x6000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x7000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000014 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 7 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181cc0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181cc0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181cc0 type 0 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs open_for_read db/OPTIONS-000004.dbtmp (sequential) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf340 on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf340 0x0~2000 from file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1354 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0f0000~10000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _read left 0x2000 len 0x1354 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _read got 4948 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs rename db/OPTIONS-000004.dbtmp -> db/OPTIONS-000005 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs rename db/OPTIONS-000005 file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 4 rocksdb: DB pointer 0x55ed97c10800 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _open_fm initializing freespace 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 freelist create size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 freelist allocate 0x0~2000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _open_fm reserved 0xbb3330000 for bluefs 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 1 freelist init 2019-12-07T19:33:20.823+0000 7f4cc7a7f700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:20.823+0000 7f4cc7a7f700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_ondisk_format_super ondisk_format 3 min_compat_ondisk_format 3 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 311 < min_flush_size 524288 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 311 < min_flush_size 524288 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97181900 file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0x0~137 to file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0x0 0x0~137 to file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x137 from 0 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x137 from 1 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 allocate extent: 0xacd100000~10000/10000,10000,0 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 8 (was clean) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x0 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x137 and padding block with 0xec9 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0x137 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (8) on file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]), flushing log 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x7000~1000 to file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x7000 0x7000~1000 to file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x7000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x8000 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000015 2019-12-07T19:33:20.823+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 8 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x8000 expected 3000 ratio 2.66667 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 0 meta) 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 1 meta) 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 1 meta) 2019-12-07T19:33:20.824+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 2 meta) 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _sync_bluefs_and_fm bluefs_extents at KV is now 0x[acccc0000~e6670000] 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 106 < min_flush_size 524288 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 106 < min_flush_size 524288 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97181900 file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0x137~6a to file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0x137 0x137~6a to file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x137 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluefs _flush_range using partial tail 0x137 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x1a1 and padding block with 0xe5f 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0x1a1 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000000 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _close_fm 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 1 freelist shutdown 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _truncate 0x1a1 file file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.825+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 1 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs unlock_file 0x55ed96fda2c0 on file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 1 bluefs umount 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs sync_metadata 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181b80 0x8000~1000 to file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181b80 pos 0x8000 0x8000~1000 to file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x8000 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181b80 pos now 0x9000 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181b80 done in 0.000015 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 9 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs sync_metadata done in 0.000086 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181b80 type 0 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 20 bluefs _stop_alloc 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700e900 shutdown 2019-12-07T19:33:20.826+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:21.081+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck check (regular) 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 2 meta) 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 2 meta) 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 2 meta) 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 2 meta) 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bluefs mount 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluefs _open_super 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 20 bluefs _init_alloc 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluefs _replay 2019-12-07T19:33:21.322+0000 7f4cd91d6bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_add_free done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _replay done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free 0xacd100000~10000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e900 init_rm_free done 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options using custom Env 0x55ed97c49240 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf550 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf550 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:21.323+0000 7f4cd91d6bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.env: 0x55ed97c49240 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.info_log: 0x55ed97c47440 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_manager: 0x55ed97ccee70 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf550 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf550 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ed96fda428) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55ed9700ea10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf550 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747201325720, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf6b0 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf6b0 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 20 bluefs _read got 417 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747201325752, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old nid_max 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old blobid_max 0 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta freelist_type bitmap 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta per_pool_omap=1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_csum csum_type crc32c 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 1 freelist init 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 freelist init size 0x1680000000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:21.324+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:21.325+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc opening allocation metadata 2019-12-07T19:33:21.325+0000 7f4cd91d6bc0 10 freelist enumerate_next start 2019-12-07T19:33:21.325+0000 7f4cd91d6bc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:21.325+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_add_free 0x2000~167fffe000 2019-12-07T19:33:21.326+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_add_free done 2019-12-07T19:33:21.326+0000 7f4cd91d6bc0 10 freelist enumerate_next end 2019-12-07T19:33:21.326+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:21.326+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:21.326+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free done 2019-12-07T19:33:21.326+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_collections 2019-12-07T19:33:21.326+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263437 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:21.326+0000 7f4cc7a7f700 5 bluestore.MempoolThread(0x55ed97c4ca08) _resize_shards cache_size: 2845357447 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:21.329+0000 7f4cd91d6bc0 1 bluefs fsck 2019-12-07T19:33:21.329+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:21.329+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open walking object keyspace 2019-12-07T19:33:21.330+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking shared_blobs 2019-12-07T19:33:21.330+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking pool_statfs 2019-12-07T19:33:21.330+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking for stray omap data 2019-12-07T19:33:21.330+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking deferred events 2019-12-07T19:33:21.330+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:21.332+0000 7f4cd91d6bc0 10 freelist enumerate_next start 2019-12-07T19:33:21.332+0000 7f4cd91d6bc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:21.376+0000 7f4cc7a7f700 20 bluestore.MempoolThread(0x55ed97c4ca08) _resize_shards cache_size: 2845357447 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:21.394+0000 7f4cd91d6bc0 10 freelist enumerate_next end 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 2 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open 0 objects, 0 of them sharded. 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 2 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open 0 extents to 0 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.068998 seconds 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _flush_cache 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 1 bluefs umount 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 0 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 20 bluefs _stop_alloc 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700e900 shutdown 2019-12-07T19:33:21.395+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:21.580+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700e800 shutdown 2019-12-07T19:33:21.580+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _close_fm 2019-12-07T19:33:21.580+0000 7f4cd91d6bc0 1 freelist shutdown 2019-12-07T19:33:21.580+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 2 meta) 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 0 bluestore(/var/lib/ceph/osd/ceph-7) mkfs success 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) set_cache_shards 1 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _mount path /var/lib/ceph/osd/ceph-7 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck check (regular) 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:21.830+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bluefs mount 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluefs _open_super 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 20 bluefs _init_alloc 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluefs _replay 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.831+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_add_free done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _replay done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free 0xacd100000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700e800 init_rm_free done 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options using custom Env 0x55ed97c493e0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf970 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf970 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.env: 0x55ed97c493e0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.info_log: 0x55ed97c48ea0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_manager: 0x55ed97ccfa70 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf970 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf970 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ed96fda498) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55ed9700e910 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf970 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747201834179, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf810 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf810 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:21.832+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 20 bluefs _read got 417 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747201834220, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old nid_max 0 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old blobid_max 0 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta freelist_type bitmap 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta per_pool_omap=1 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_csum csum_type crc32c 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 1 freelist init 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist init size 0x1680000000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc opening allocation metadata 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist enumerate_next start 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:21.833+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_add_free 0x2000~167fffe000 2019-12-07T19:33:21.834+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_add_free done 2019-12-07T19:33:21.834+0000 7f4cd91d6bc0 10 freelist enumerate_next end 2019-12-07T19:33:21.834+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:21.834+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:21.834+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:21.834+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_collections 2019-12-07T19:33:21.834+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 1 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:21.834+0000 7f4cc7879700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263443 2019-12-07T19:33:21.834+0000 7f4cc7879700 5 bluestore.MempoolThread(0x55ed97c4ca08) _resize_shards cache_size: 2845357303 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:21.836+0000 7f4cd91d6bc0 1 bluefs fsck 2019-12-07T19:33:21.836+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:21.836+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open walking object keyspace 2019-12-07T19:33:21.837+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking shared_blobs 2019-12-07T19:33:21.837+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking pool_statfs 2019-12-07T19:33:21.837+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking for stray omap data 2019-12-07T19:33:21.837+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking deferred events 2019-12-07T19:33:21.837+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:21.839+0000 7f4cd91d6bc0 10 freelist enumerate_next start 2019-12-07T19:33:21.839+0000 7f4cd91d6bc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:21.884+0000 7f4cc7879700 20 bluestore.MempoolThread(0x55ed97c4ca08) _resize_shards cache_size: 2845357303 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:21.901+0000 7f4cd91d6bc0 10 freelist enumerate_next end 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 2 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open 0 objects, 0 of them sharded. 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 2 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open 0 extents to 0 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.068220 seconds 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _flush_cache 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 1 bluefs umount 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181e00 type 0 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 20 bluefs _stop_alloc 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700e800 shutdown 2019-12-07T19:33:21.902+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:22.086+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700ef00 shutdown 2019-12-07T19:33:22.086+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _close_fm 2019-12-07T19:33:22.086+0000 7f4cd91d6bc0 1 freelist shutdown 2019-12-07T19:33:22.086+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bluefs mount 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluefs _open_super 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 20 bluefs _init_alloc 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluefs _replay 2019-12-07T19:33:22.336+0000 7f4cd91d6bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_add_free done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _replay done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options using custom Env 0x55ed97c486a0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.env: 0x55ed97c486a0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.info_log: 0x55ed97c49780 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_manager: 0x55ed97cce210 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:22.337+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ed96fda5d0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55ed9700ed10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747202339638, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf1e0 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf1e0 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 20 bluefs _read got 417 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747202339671, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old nid_max 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old blobid_max 0 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta freelist_type bitmap 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta per_pool_omap=1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_csum csum_type crc32c 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 1 freelist init 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist init size 0x1680000000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700f000 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc opening allocation metadata 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist enumerate_next start 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:22.338+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700f000 init_add_free 0x2000~167fffe000 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700f000 init_add_free done 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 10 freelist enumerate_next end 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700f000 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700f000 init_rm_free done 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 1 bluefs umount 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 0 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 20 bluefs _stop_alloc 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700ef00 shutdown 2019-12-07T19:33:22.339+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 0 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 1 bluefs mount 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _open_super 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _init_alloc 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_add_free done 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:22.591+0000 7f4cd91d6bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 4096 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _replay done 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 init_rm_free done 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options using custom Env 0x55ed97c495e0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf600 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf600 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.env: 0x55ed97c495e0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.info_log: 0x55ed97c496e0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_manager: 0x55ed97cce510 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs mkdir db 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs mkdir db 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) with 0x55ed96fda5c0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/CURRENT file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf600 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf600 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 16 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf600 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ed96fda5f0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55ed9700ed10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf600 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 13 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs mkdir db 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747202593965, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf1e0 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf1e0 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _read got 417 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_write db/000004.sst 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/000004.sst vsel_hint 0x1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97cf0140 on file(ino 8 size 0x0 mtime 2019-12-07T19:33:22.594011+0000 allocated 0 extents []) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 1383 < min_flush_size 524288 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97cf0140 file(ino 8 size 0x0 mtime 2019-12-07T19:33:22.594011+0000 allocated 0 extents []) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 0x0~567 to file(ino 8 size 0x0 mtime 2019-12-07T19:33:22.594011+0000 allocated 0 extents []) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97cf0140 pos 0x0 0x0~567 to file(ino 8 size 0x0 mtime 2019-12-07T19:33:22.594011+0000 allocated 0 extents []) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x567 from 1 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate extent: 0xacd110000~10000/10000,10000,0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 10 (was clean) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd110000~10000 x_off 0x0 2019-12-07T19:33:22.592+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x567 and padding block with 0xa99 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97cf0140 pos now 0x567 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf0140 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf0140 done in 0.000000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (10) on file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]), flushing log 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 10 len 0x56 crc 0x50e36ed9) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0x9000~1000 to file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0x9000 0x9000~1000 to file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x9000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0xa000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000024 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 10 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97cf0140 type 2 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf1e0 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf1e0 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read got 1383 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _read_random h 0x55ed96fdf1e0 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random got 53 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _read_random h 0x55ed96fdf1e0 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random got 85 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _read_random h 0x55ed96fdf1e0 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random got 696 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _read_random h 0x55ed96fdf1e0 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random got 28 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _read_random h 0x55ed96fdf1e0 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _read_random got 74 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747202594542, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1383, "table_properties": {"data_size": 447, "index_size": 28, "filter_size": 69, "raw_key_size": 314, "raw_average_key_size": 22, "raw_value_size": 127, "raw_average_value_size": 9, "num_data_blocks": 1, "num_entries": 14, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 5 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_write db/MANIFEST-000005 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/MANIFEST-000005 vsel_hint 0x1 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97cf0140 on file(ino 9 size 0x0 mtime 2019-12-07T19:33:22.594585+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 35 < min_flush_size 524288 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 44 < min_flush_size 524288 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97cf0140 file(ino 9 size 0x0 mtime 2019-12-07T19:33:22.594585+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 0x0~66 to file(ino 9 size 0x0 mtime 2019-12-07T19:33:22.594585+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97cf0140 pos 0x0 0x0~66 to file(ino 9 size 0x0 mtime 2019-12-07T19:33:22.594585+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x66 from 1 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate extent: 0xacd120000~10000/10000,10000,0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 11 (was clean) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x66 and padding block with 0xf9a 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97cf0140 pos now 0x66 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf0140 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf0140 done in 0.000000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (11) on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]), flushing log 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 11 len 0x5a crc 0x195e9a0c) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0xa000~1000 to file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0xa000 0xa000~1000 to file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xa000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0xb000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000018 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 11 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_write db/000005.dbtmp 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/000005.dbtmp vsel_hint 0x1 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97cf0280 on file(ino 10 size 0x0 mtime 2019-12-07T19:33:22.594861+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97cf0280 file(ino 10 size 0x0 mtime 2019-12-07T19:33:22.594861+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0280 0x0~10 to file(ino 10 size 0x0 mtime 2019-12-07T19:33:22.594861+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97cf0280 pos 0x0 0x0~10 to file(ino 10 size 0x0 mtime 2019-12-07T19:33:22.594861+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate extent: 0xacd130000~10000/10000,10000,0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 12 (was clean) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97cf0280 pos now 0x10 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf0280 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf0280 done in 0.000000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (12) on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]), flushing log 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 12 len 0x57 crc 0xae0b9370) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0xb000~1000 to file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0xb000 0xb000~1000 to file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xb000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0xc000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000017 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 12 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97cf0280 type 0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs rename db/000005.dbtmp -> db/CURRENT 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs rename dir db (0x55ed97c4ad80) file CURRENT already exists, unlinking 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _drop_link had refs 1 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _drop_link destroying file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs rename db/CURRENT file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747202595075, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_write db.wal/000006.log 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db.wal/000006.log vsel_hint 0 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97cf0280 on file(ino 11 size 0x0 mtime 2019-12-07T19:33:22.595084+0000 allocated 0 extents []) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs unlink db.wal/000003.log 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _drop_link had refs 1 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _drop_link destroying file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs unlink db/MANIFEST-000001 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _drop_link had refs 1 on file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _drop_link destroying file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs sync_metadata 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 13 len 0xc8 crc 0x6dbf5b68) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0xc000~1000 to file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0xc000 0xc000~1000 to file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xc000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0xd000 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000016 2019-12-07T19:33:22.593+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 13 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 release 0xacd0c0000~20000 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 release 0xacd100000~10000 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 release done 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs sync_metadata done in 0.000091 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0xd000 expected 3000 ratio 4.33333 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs open_for_write db/OPTIONS-000007.dbtmp 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs open_for_write mapping db/OPTIONS-000007.dbtmp vsel_hint 0x1 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs open_for_write h 0x55ed97cf03c0 on file(ino 12 size 0x0 mtime 2019-12-07T19:33:22.595230+0000 allocated 0 extents []) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf03c0 ignoring, length 4949 < min_flush_size 524288 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _fsync 0x55ed97cf03c0 file(ino 12 size 0x0 mtime 2019-12-07T19:33:22.595230+0000 allocated 0 extents []) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf03c0 0x0~1355 to file(ino 12 size 0x0 mtime 2019-12-07T19:33:22.595230+0000 allocated 0 extents []) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97cf03c0 pos 0x0 0x0~1355 to file(ino 12 size 0x0 mtime 2019-12-07T19:33:22.595230+0000 allocated 0 extents []) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _allocate len 0x1355 from 1 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700ef00 allocate extent: 0xacd0c0000~10000/10000,10000,0 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range dirty_seq = 14 (was clean) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacd0c0000~10000 x_off 0x0 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range caching tail of 0x355 and padding block with 0xcab 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97cf03c0 pos now 0x1355 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf03c0 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97cf03c0 done in 0.000000 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _fsync file metadata was dirty (14) on file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]), flushing log 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 14 len 0x60 crc 0xec0b3dd3) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0xd000~1000 to file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0xd000 0xd000~1000 to file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xd000 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0xe000 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000014 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 14 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf03c0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf03c0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97cf03c0 type 0 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs open_for_read db/OPTIONS-000007.dbtmp (sequential) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs open_for_read h 0x55ed96fdf340 on file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _read h 0x55ed96fdf340 0x0~2000 from file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1355 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0c0000~10000 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _read left 0x2000 len 0x1355 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _read got 4949 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs rename db/OPTIONS-000007.dbtmp -> db/OPTIONS-000008 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs rename db/OPTIONS-000008 file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 4 rocksdb: DB pointer 0x55ed97ce7600 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _upgrade_super from 3, latest 3 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _upgrade_super done 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_collections 2019-12-07T19:33:22.594+0000 7f4cc807a700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:22.594+0000 7f4cc807a700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 1.35 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Sum 1/0 1.35 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 1.35 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Sum 1/0 1.35 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluefs _read_random h 0x55ed96fdf1e0 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluefs _read_random got 447 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_start 2019-12-07T19:33:22.594+0000 7f4cc987d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread start 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_replay start 2019-12-07T19:33:22.594+0000 7f4cce887700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread start 2019-12-07T19:33:22.594+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:22.594+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach meta fresh osr 0x55ed97cf03c0 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_replay draining osr 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_register_zombie 0x55ed97cf03c0 meta 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all osr_set 0x55ed97cf03c0 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all drain 0x55ed97cf03c0 2019-12-07T19:33:22.594+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:22.594+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:22.594+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all reaping empty zombie osr 0x55ed97cf03c0 2019-12-07T19:33:22.594+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all done 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_replay completed 0 events 2019-12-07T19:33:22.594+0000 7f4cd91d6bc0 10 bluestore.OnodeSpace(0x55ed9700d9a0 in 0x55ed97c54000) clear 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach meta fresh osr 0x55ed97cf03c0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x55ed9700d860 meta 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x55ed97cf03c0 = 0x55ed9702b200 seq 1 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection meta bits 0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection meta bits 0 = 0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x55ed9700d860) get_onode oid #-1:7b3f43c4:::osd_superblock:0# key 0x7f7fffffffffffffff7b3f43c4216f73'd_superblock!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x55ed9700d860) r -2 v.len 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 1 Request: 944 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~233 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:22.595+0000 7f4cce086700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.026305 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~233 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdcd on front/back, now 0x0~1000 2019-12-07T19:33:22.595+0000 7f4cce086700 5 bluestore.MempoolThread(0x55ed97c4ca08) _resize_shards cache_size: 2845366629 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 1112 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x55ed9702b200 1 blobs 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700f000 allocate 0x1000/1000,1000,0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 fbmap_alloc 0x55ed9700f000 allocate extent: 0x2000~1000/1000,1000,0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2000~1000] 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x55ed97c511f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x55ed97c51180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x55ed97c511f0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x55ed97c51180 sbid 0x0)) 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore.blob(0x55ed97c511f0) get_ref 0x0~233 Blob(0x55ed97c511f0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x55ed97c51180 sbid 0x0)) 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore.blob(0x55ed97c511f0) get_ref init 0x1000, 1000 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~233: 0x0~233 Blob(0x55ed97c511f0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x55ed97c51180 sbid 0x0)) 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore.BufferSpace(0x55ed97c51198 in 0x55ed96fe1340) _discard 0x0~1000 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x233 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore.extentmap(0x55ed9702cd88) dirty_range mark inline shard dirty 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 = 0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x55ed9702b200 cost 8848 (2 ios * 4000 + 848 bytes) 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x55ed9702b200 onodes 0x55ed9702cc80 shared_blobs 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore.extentmap(0x55ed9702cd88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore.extentmap(0x55ed9702cd88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x55ed9702b200 allocated 0x[2000~1000] released 0x[] 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 freelist allocate 0x2000~1000 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x55ed9702b200 prepare 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x55ed9702b200 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 3 meta) 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 4 meta) 2019-12-07T19:33:22.595+0000 7f4cd2294700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x55ed9702b200 aio_wait 2019-12-07T19:33:22.595+0000 7f4cd2294700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x55ed9702b200 2019-12-07T19:33:22.595+0000 7f4cd2294700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x55ed9702b200 io_done 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread new_nid_max 1025 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread new_blobid_max 10240 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush 0x55ed97cf0280 ignoring, length 265 < min_flush_size 524288 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x55ed9702cc80 had 1 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs get_usage bdev 1 free 3860922368 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998813) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399548 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush 0x55ed97cf0280 ignoring, length 288 < min_flush_size 524288 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush 0x55ed97cf0280 ignoring, length 288 < min_flush_size 524288 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _fsync 0x55ed97cf0280 file(ino 11 size 0x0 mtime 2019-12-07T19:33:22.595084+0000 allocated 0 extents []) 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush 0x55ed97cf0280 0x0~120 to file(ino 11 size 0x0 mtime 2019-12-07T19:33:22.595084+0000 allocated 0 extents []) 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush_range 0x55ed97cf0280 pos 0x0 0x0~120 to file(ino 11 size 0x0 mtime 2019-12-07T19:33:22.595084+0000 allocated 0 extents []) 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _allocate len 0x120 from 0 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _allocate len 0x120 from 1 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 fbmap_alloc 0x55ed9700ef00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 fbmap_alloc 0x55ed9700ef00 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range dirty_seq = 15 (was clean) 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range file now file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range caching tail of 0x120 and padding block with 0xee0 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 4 meta) 2019-12-07T19:33:22.595+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 5 meta) 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range h 0x55ed97cf0280 pos now 0x120 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs wait_for_aio 0x55ed97cf0280 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs wait_for_aio 0x55ed97cf0280 done in 0.000000 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs flush_bdev 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _fsync file metadata was dirty (15) on file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_and_sync_log op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush_and_sync_log txn(seq 15 len 0x64 crc 0x443daee2) 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush 0x55ed97181900 0xe000~1000 to file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.595+0000 7f4cc987d700 10 bluefs _flush_range 0x55ed97181900 pos 0xe000 0xe000~1000 to file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range file now file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.595+0000 7f4cc987d700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xe000 2019-12-07T19:33:22.596+0000 7f4cc987d700 20 bluefs _flush_range h 0x55ed97181900 pos now 0xf000 2019-12-07T19:33:22.596+0000 7f4cc987d700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:22.596+0000 7f4cc987d700 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:22.596+0000 7f4cc987d700 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000016 2019-12-07T19:33:22.596+0000 7f4cc987d700 20 bluefs flush_bdev 2019-12-07T19:33:22.596+0000 7f4cc987d700 20 bluefs _flush_and_sync_log log_seq_stable 15 2019-12-07T19:33:22.596+0000 7f4cc987d700 20 bluefs _flush_and_sync_log cleaned file file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.596+0000 7f4cc987d700 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:22.596+0000 7f4cc987d700 10 bluefs _should_compact_log current 0xf000 expected 3000 ratio 5 2019-12-07T19:33:22.596+0000 7f4cc987d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread nid_max now 1025 2019-12-07T19:33:22.596+0000 7f4cc987d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread blobid_max now 10240 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:22.596+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000349835s (0.000005780s flush + 0.000344055s kv commit) 2019-12-07T19:33:22.596+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x55ed9702b200> 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:22.596+0000 7f4cce887700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x55ed9702b200 kv_submitted 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x55ed9702b200 2019-12-07T19:33:22.596+0000 7f4cce887700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x55ed9702b200 finishing 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x55ed9702b200 onodes 0x55ed9702cc80 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore.BufferSpace(0x55ed97c51198 in 0x55ed96fe1340) _finish_write discard buffer(0x55ed97005cb0 space 0x55ed97c51198 0x0~1000 writing nocache) 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x55ed9702b200 done 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x55ed97cf03c0 q now empty 2019-12-07T19:33:22.596+0000 7f4cce887700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x55ed9702b200 [] 2019-12-07T19:33:22.596+0000 7f4cce887700 10 fbmap_alloc 0x55ed9700f000 release done 2019-12-07T19:33:22.596+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:22.596+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.596+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 5 meta) 2019-12-07T19:33:22.596+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 6 meta) 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 6 meta) 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) umount 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all osr_set 0x55ed97cf03c0 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all drain 0x55ed97cf03c0 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all done 2019-12-07T19:33:22.597+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:22.597+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:22.597+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:22.597+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) umount stopping kv thread 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_stop 2019-12-07T19:33:22.597+0000 7f4cc987d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:22.597+0000 7f4cce887700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:22.597+0000 7f4cc987d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread finish 2019-12-07T19:33:22.597+0000 7f4cce887700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread finish 2019-12-07T19:33:22.597+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_stop stopping finishers 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_stop stopped 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _flush_cache 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluestore.sharedblob(0x55ed97c51180) put 0x55ed97c51180 removing self from set 0x55ed9700d930 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluestore.BufferSpace(0x55ed97c51198 in 0x55ed96fe1340) _clear 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluestore.OnodeSpace(0x55ed9700d9a0 in 0x55ed97c54000) clear 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) umount closing 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs readdir db 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs readdir db.wal 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0280 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0280 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0280 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0280 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _truncate 0x120 file file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97cf0280 type 1 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97cf0140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _truncate 0x66 file file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97cf0140 type 0 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs unlock_file 0x55ed96fda5c0 on file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 1 bluefs umount 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs sync_metadata 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush_and_sync_log txn(seq 16 len 0x23 crc 0x2ab2536) 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush 0x55ed97181900 0xf000~1000 to file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _flush_range 0x55ed97181900 pos 0xf000 0xf000~1000 to file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluefs _flush_range file now file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xf000 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluefs _flush_range h 0x55ed97181900 pos now 0x10000 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs wait_for_aio 0x55ed97181900 done in 0.000016 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluefs flush_bdev 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluefs _flush_and_sync_log log_seq_stable 16 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs sync_metadata done in 0.000088 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 10 bluefs _close_writer 0x55ed97181900 type 0 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 20 bluefs _stop_alloc 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700ef00 shutdown 2019-12-07T19:33:22.598+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a380 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:22.853+0000 7f4cd91d6bc0 1 fbmap_alloc 0x55ed9700f000 shutdown 2019-12-07T19:33:22.853+0000 7f4cd91d6bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _close_fm 2019-12-07T19:33:22.853+0000 7f4cd91d6bc0 1 freelist shutdown 2019-12-07T19:33:22.853+0000 7f4cd91d6bc0 1 bdev(0x55ed97c5a000 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:23.100+0000 7f4cd91d6bc0 0 created object store /var/lib/ceph/osd/ceph-7 for osd.7 fsid ad52b421-bda1-4750-8dc4-c9e4d1297c01 2019-12-07T19:33:34.999+0000 7f8f2a397bc0 0 ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev), process ceph-osd, pid 36939 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 5 object store type is bluestore 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) set_cache_shards 1 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 0 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 1 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:35.000+0000 7f8f2a397bc0 0 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:35.001+0000 7f8f2a397bc0 1 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.001+0000 7f8f2a397bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:35.001+0000 7f8f2a397bc0 1 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:35.259+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:35.512+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:35.512+0000 7f8f2a397bc0 1 objectstore numa_node 0 2019-12-07T19:33:35.513+0000 7f8f2a397bc0 0 starting osd.7 osd_data /var/lib/ceph/osd/ceph-7 /var/lib/ceph/osd/ceph-7/journal 2019-12-07T19:33:35.513+0000 7f8f2a397bc0 -1 Falling back to public interface 2019-12-07T19:33:35.514+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] _finish_bind bind my_addrs is [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] 2019-12-07T19:33:35.515+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6826/36939,v1:0.0.0.0:6827/36939] _finish_bind bind my_addrs is [v2:0.0.0.0:6826/36939,v1:0.0.0.0:6827/36939] 2019-12-07T19:33:35.516+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6828/36939,v1:0.0.0.0:6829/36939] _finish_bind bind my_addrs is [v2:0.0.0.0:6828/36939,v1:0.0.0.0:6829/36939] 2019-12-07T19:33:35.517+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6830/36939,v1:0.0.0.0:6831/36939] _finish_bind bind my_addrs is [v2:0.0.0.0:6830/36939,v1:0.0.0.0:6831/36939] 2019-12-07T19:33:35.521+0000 7f8f2a397bc0 0 load: jerasure load: lrc load: isa 2019-12-07T19:33:35.521+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:35.521+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:35.521+0000 7f8f2a397bc0 0 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:35.522+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.522+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.522+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.522+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:35.522+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:0.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:1.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:2.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:3.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:4.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:5.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:6.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 osd.7:7.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] start start 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 -- start start 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 -- start start 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6828/36939,v1:0.0.0.0:6829/36939] start start 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6830/36939,v1:0.0.0.0:6831/36939] start start 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6826/36939,v1:0.0.0.0:6827/36939] start start 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 -- start start 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 2 osd.7 0 init /var/lib/ceph/osd/ceph-7 (looks like ssd) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 2 osd.7 0 journal /var/lib/ceph/osd/ceph-7/journal 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) set_cache_shards 32 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _mount path /var/lib/ceph/osd/ceph-7 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck check (regular) 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:35.785+0000 7f8f2a397bc0 0 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 0 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bluefs mount 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluefs _open_super 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 20 bluefs _init_alloc 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:35.786+0000 7f8f2a397bc0 10 bluefs _replay 2019-12-07T19:33:35.787+0000 7f8f2a397bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.787+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.787+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_add_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x50e36ed9) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:22.594011+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0x195e9a0c) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:22.594585+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0xae0b9370) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:22.594861+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0x6dbf5b68) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:22.595084+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0xec0b3dd3) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:22.595230+0000 allocated 0 extents []) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0x443daee2) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x2ab2536) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs _replay done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacd130000~10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacd120000~10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacd110000~10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72900 init_rm_free done 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options using custom Env 0x558c546acf60 2019-12-07T19:33:35.788+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a42000 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read got 16 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read got 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a42000 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read got 102 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.env: 0x558c546acf60 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.info_log: 0x558c546ad1c0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.write_buffer_manager: 0x558c548350b0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a42000 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read got 16 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read got 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a42000 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558c53a3e620) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x558c53a73010 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a42000 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:35.789+0000 7f8f2a397bc0 20 bluefs _read got 102 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs open_for_read h 0x558c53a43810 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs _read h 0x558c53a43810 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read got 1383 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs _read_random h 0x558c53a43810 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random got 53 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs _read_random h 0x558c53a43810 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random got 85 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs _read_random h 0x558c53a43810 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random got 696 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs _read_random h 0x558c53a43810 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random got 28 2019-12-07T19:33:35.789+0000 7f8f19bb2700 10 bluefs _read_random h 0x558c53a43810 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:35.789+0000 7f8f19bb2700 20 bluefs _read_random got 74 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747215791312, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a433f0 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a433f0 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 20 bluefs _read got 288 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747215791369, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old nid_max 1025 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old blobid_max 10240 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43810 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 20 bluefs _read_random got 447 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta freelist_type bitmap 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta per_pool_omap=1 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_csum csum_type crc32c 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 1 freelist init 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 freelist init size 0x1680000000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:35.790+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:35.791+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc opening allocation metadata 2019-12-07T19:33:35.791+0000 7f8f2a397bc0 10 freelist enumerate_next start 2019-12-07T19:33:35.791+0000 7f8f2a397bc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:35.791+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_add_free 0x3000~167fffd000 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_add_free done 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 10 freelist enumerate_next end 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_collections 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _open_collections opened meta 0x558c53a70f00 cnode(bits 0) 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach meta fresh osr 0x558c53be5cc0 2019-12-07T19:33:35.793+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 1 Request: 944 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:35.793+0000 7f8f12ba4700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263783 2019-12-07T19:33:35.793+0000 7f8f12ba4700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:35.793+0000 7f8f12ba4700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845348437 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.797+0000 7f8f2a397bc0 1 bluefs fsck 2019-12-07T19:33:35.797+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x233/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:35.797+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open walking object keyspace 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_check_objects collection meta cnode(bits 0) 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) fsck_check_objects_shallow #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 20 bluestore.blob(0x558c54741e30) get_ref 0x0~233 Blob(0x558c54741e30 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54741f80 sbid 0x0)) 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 20 bluestore.blob(0x558c54741e30) get_ref init 0x1000, 1000 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) fsck_check_objects_shallow 0x0~233: 0x0~233 Blob(0x558c54741e30 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558c54741f80 sbid 0x0)) 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 20 bluestore.sharedblob(0x558c54741f80) put 0x558c54741f80 removing self from set 0x558c53a70fd0 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 20 bluestore.BufferSpace(0x558c54741f98 in 0x558c53a59340) _clear 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking shared_blobs 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking pool_statfs 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking for stray omap data 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking deferred events 2019-12-07T19:33:35.798+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:35.800+0000 7f8f2a397bc0 10 freelist enumerate_next start 2019-12-07T19:33:35.800+0000 7f8f2a397bc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:35.844+0000 7f8f12ba4700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845348437 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.894+0000 7f8f12ba4700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845348437 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.905+0000 7f8f2a397bc0 10 freelist enumerate_next end 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 2 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open 1 objects, 0 of them sharded. 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 2 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open 1 extents to 1 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.113222 seconds 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _flush_cache 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 10 bluestore.OnodeSpace(0x558c53a71040 in 0x558c53aac000) clear 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 1 bluefs umount 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 10 bluefs _close_writer 0x558c53be5b80 type 0 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 20 bluefs _stop_alloc 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 1 fbmap_alloc 0x558c53a72900 shutdown 2019-12-07T19:33:35.906+0000 7f8f2a397bc0 1 bdev(0x558c53ab8380 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:36.042+0000 7f8f2a397bc0 1 fbmap_alloc 0x558c53a73100 shutdown 2019-12-07T19:33:36.042+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _close_fm 2019-12-07T19:33:36.042+0000 7f8f2a397bc0 1 freelist shutdown 2019-12-07T19:33:36.042+0000 7f8f2a397bc0 1 bdev(0x558c53ab8000 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bdev(0x558c53ab8e00 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 0 bdev(0x558c53ab8e00 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bdev(0x558c53ab8e00 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bdev(0x558c53ab8a80 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 0 bdev(0x558c53ab8a80 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bdev(0x558c53ab8a80 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bluefs mount 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluefs _open_super 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 20 bluefs _init_alloc 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluefs _replay 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.296+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_add_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x50e36ed9) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:22.594011+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0x195e9a0c) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:22.594585+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0xae0b9370) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:22.594861+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0x6dbf5b68) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:22.595084+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0xec0b3dd3) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:22.595230+0000 allocated 0 extents []) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0x443daee2) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x2ab2536) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43810 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _replay done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd130000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd120000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd110000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options using custom Env 0x558c546ab6e0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43340 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43340 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 16 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43340 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43340 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43340 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 20 bluefs _read got 102 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.env: 0x558c546ab6e0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.info_log: 0x558c546aadc0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.write_buffer_manager: 0x558c54835e00 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:36.297+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43340 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43340 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read got 16 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43340 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read got 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43340 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558c53a3e6a0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x558c53a72910 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43340 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read got 102 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs open_for_read h 0x558c53a43600 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs _read h 0x558c53a43600 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read got 1383 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs _read_random h 0x558c53a43600 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random got 53 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs _read_random h 0x558c53a43600 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random got 85 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs _read_random h 0x558c53a43600 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random got 696 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs _read_random h 0x558c53a43600 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random got 28 2019-12-07T19:33:36.298+0000 7f8f199ac700 10 bluefs _read_random h 0x558c53a43600 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:36.298+0000 7f8f199ac700 20 bluefs _read_random got 74 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216299618, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43760 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43760 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read got 288 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216299646, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old nid_max 1025 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta old blobid_max 10240 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43600 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 20 bluefs _read_random got 447 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta freelist_type bitmap 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_super_meta per_pool_omap=1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_csum csum_type crc32c 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 1 freelist init 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist init size 0x1680000000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72e00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc opening allocation metadata 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist enumerate_next start 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:36.298+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72e00 init_add_free 0x3000~167fffd000 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72e00 init_add_free done 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 10 freelist enumerate_next end 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72e00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a72e00 init_rm_free done 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 1 bluefs umount 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 10 bluefs _close_writer 0x558c53be5a40 type 0 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 20 bluefs _stop_alloc 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 1 fbmap_alloc 0x558c53a73100 shutdown 2019-12-07T19:33:36.299+0000 7f8f2a397bc0 1 bdev(0x558c53ab8a80 /var/lib/ceph/osd/ceph-7/block) close 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db kv_backend = rocksdb 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db do_bluefs = 1 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_db initializing bluefs 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 1 bdev create path /var/lib/ceph/osd/ceph-7/block type kernel 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 1 bdev(0x558c53ab8a80 /var/lib/ceph/osd/ceph-7/block) open path /var/lib/ceph/osd/ceph-7/block 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 0 bdev(0x558c53ab8a80 /var/lib/ceph/osd/ceph-7/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-7/block failed: (22) Invalid argument 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 1 bdev(0x558c53ab8a80 /var/lib/ceph/osd/ceph-7/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-7/block size 90 GiB 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 1 bluefs mount 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs _open_super 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 20 bluefs _init_alloc 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs _replay 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:36.559+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_add_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0xbf13c6f4) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:20.822383+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:20.822408+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x4c69203a) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:20.822694+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:20.822708+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x21244bef) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:20.823042+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xe0489054) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:20.823681+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0x8372f3f0) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:20.823833+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xfa9ce76b) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x45570161) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:20.824783+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x50e36ed9) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:22.594011+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0x195e9a0c) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:22.594585+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0xae0b9370) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:22.594861+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0x6dbf5b68) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:22.595084+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0xec0b3dd3) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:22.595230+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:22.595560+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0x443daee2) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x2ab2536) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 5666427f-6c4a-4a42-91ea-9a9db9a11ef5 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _replay done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd130000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd120000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd110000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 init_rm_free done 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family L: 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family M: 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) column family P: 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options using custom Env 0x558c546ab440 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43600 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 16 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43600 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 102 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.env: 0x558c546ab440 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.info_log: 0x558c546ad520 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.write_buffer_manager: 0x558c54835e30 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs mkdir db 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs mkdir db 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:20.822366+0000 allocated 0 extents []) with 0x558c53a3e650 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/CURRENT file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43600 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:20.823057+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43600 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 16 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43600 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558c53a3e6a0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x558c53a72910 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43600 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 102 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs open_for_read h 0x558c53a43c30 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs _read h 0x558c53a43c30 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read got 1383 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs _read_random h 0x558c53a43c30 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random got 53 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs _read_random h 0x558c53a43c30 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random got 85 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs _read_random h 0x558c53a43c30 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random got 696 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs _read_random h 0x558c53a43c30 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random got 28 2019-12-07T19:33:36.560+0000 7f8f1299e700 10 bluefs _read_random h 0x558c53a43c30 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:36.560+0000 7f8f1299e700 20 bluefs _read_random got 74 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs mkdir db 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216561986, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43b80 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43b80 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _read got 288 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_write db/000007.sst 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs open_for_write mapping db/000007.sst vsel_hint 0x1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs open_for_write h 0x558c54ea4280 on file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.562025+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 ignoring, length 1238 < min_flush_size 524288 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _fsync 0x558c54ea4280 file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.562025+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 0x0~4d6 to file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.562025+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c54ea4280 pos 0x0 0x0~4d6 to file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.562025+0000 allocated 0 extents []) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _allocate len 0x4d6 from 1 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd100000~10000/10000,10000,0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _flush_range dirty_seq = 17 (was clean) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x0 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _flush_range caching tail of 0x4d6 and padding block with 0xb2a 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c54ea4280 pos now 0x4d6 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea4280 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea4280 done in 0.000000 2019-12-07T19:33:36.560+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _fsync file metadata was dirty (17) on file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]), flushing log 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_and_sync_log txn(seq 17 len 0x56 crc 0x1a60a49a) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c53be5a40 0x10000~1000 to file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c53be5a40 pos 0x10000 0x10000~1000 to file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x10000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x11000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000016 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log log_seq_stable 17 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _close_writer 0x558c54ea4280 type 2 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_read db/000007.sst (random) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43b80 on file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43b80 0x194~342 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) prefetch 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read left 0xe6c len 0x342 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read got 834 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43b80 0x4a1~35 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random left 0xb5f 0x4a1~35 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random got 53 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43b80 0x44c~55 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random left 0xbb4 0x44c~55 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random got 85 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43b80 0x193~2b9 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random left 0xe6d 0x193~2b9 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random got 697 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43b80 0x177~1c from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random left 0xe89 0x177~1c 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random got 28 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43b80 0x12d~4a from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random left 0xed3 0x12d~4a 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _read_random got 74 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216562448, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 1238, "table_properties": {"data_size": 301, "index_size": 28, "filter_size": 69, "raw_key_size": 144, "raw_average_key_size": 24, "raw_value_size": 128, "raw_average_value_size": 21, "num_data_blocks": 1, "num_entries": 6, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 8 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write db/MANIFEST-000008 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs open_for_write mapping db/MANIFEST-000008 vsel_hint 0x1 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write h 0x558c54ea4280 on file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.562482+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 ignoring, length 35 < min_flush_size 524288 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 ignoring, length 87 < min_flush_size 524288 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _fsync 0x558c54ea4280 file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.562482+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4280 0x0~8f to file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.562482+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c54ea4280 pos 0x0 0x0~8f to file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.562482+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _allocate len 0x8f from 1 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd140000~10000/10000,10000,0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range dirty_seq = 18 (was clean) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.562511+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacd140000~10000 x_off 0x0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range caching tail of 0x8f and padding block with 0xf71 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c54ea4280 pos now 0x8f 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea4280 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea4280 done in 0.000000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _fsync file metadata was dirty (18) on file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.562511+0000 allocated 10000 extents [1:0xacd140000~10000]), flushing log 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.562511+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_and_sync_log txn(seq 18 len 0x5b crc 0x65921b64) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c53be5a40 0x11000~1000 to file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c53be5a40 pos 0x11000 0x11000~1000 to file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x11000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x12000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000017 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log log_seq_stable 18 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.562511+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write db/000008.dbtmp 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs open_for_write mapping db/000008.dbtmp vsel_hint 0x1 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write h 0x558c54ea43c0 on file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.562702+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _fsync 0x558c54ea43c0 file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.562702+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea43c0 0x0~10 to file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.562702+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c54ea43c0 pos 0x0 0x0~10 to file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.562702+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd150000~10000/10000,10000,0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range dirty_seq = 19 (was clean) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.562714+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacd150000~10000 x_off 0x0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x10 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _fsync file metadata was dirty (19) on file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.562714+0000 allocated 10000 extents [1:0xacd150000~10000]), flushing log 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.562714+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_and_sync_log txn(seq 19 len 0x57 crc 0x1f3d14de) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c53be5a40 0x12000~1000 to file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c53be5a40 pos 0x12000 0x12000~1000 to file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x12000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x13000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000016 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log log_seq_stable 19 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.562714+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _close_writer 0x558c54ea43c0 type 0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs rename db/000008.dbtmp -> db/CURRENT 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs rename dir db (0x558c547431a0) file CURRENT already exists, unlinking 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _drop_link had refs 1 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _drop_link destroying file(ino 10 size 0x10 mtime 2019-12-07T19:33:22.594877+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs rename db/CURRENT file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.562714+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216562879, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write db.wal/000009.log 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs open_for_write mapping db.wal/000009.log vsel_hint 0 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write h 0x558c54ea43c0 on file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.562887+0000 allocated 0 extents []) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs unlink db.wal/000006.log 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _drop_link had refs 1 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _drop_link destroying file(ino 11 size 0x120 mtime 2019-12-07T19:33:22.596997+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs unlink db/MANIFEST-000005 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _drop_link had refs 1 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _drop_link destroying file(ino 9 size 0x66 mtime 2019-12-07T19:33:22.594614+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs sync_metadata 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_and_sync_log txn(seq 20 len 0xc8 crc 0x1d7c9635) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush 0x558c53be5a40 0x13000~1000 to file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c53be5a40 pos 0x13000 0x13000~1000 to file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x13000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x14000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000014 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log log_seq_stable 20 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 release 0xacd0d0000~10000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 release 0xacd120000~20000 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 release done 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs sync_metadata done in 0.000084 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs _should_compact_log current 0x14000 expected 3000 ratio 6.66667 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write db/OPTIONS-000010.dbtmp 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 20 bluefs open_for_write mapping db/OPTIONS-000010.dbtmp vsel_hint 0x1 2019-12-07T19:33:36.561+0000 7f8f2a397bc0 10 bluefs open_for_write h 0x558c54ea4500 on file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.563041+0000 allocated 0 extents []) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4500 ignoring, length 4949 < min_flush_size 524288 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _fsync 0x558c54ea4500 file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.563041+0000 allocated 0 extents []) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4500 0x0~1355 to file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.563041+0000 allocated 0 extents []) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c54ea4500 pos 0x0 0x0~1355 to file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.563041+0000 allocated 0 extents []) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _allocate len 0x1355 from 1 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range dirty_seq = 21 (was clean) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.563373+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range caching tail of 0x355 and padding block with 0xcab 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c54ea4500 pos now 0x1355 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea4500 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c54ea4500 done in 0.000000 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _fsync file metadata was dirty (21) on file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.563373+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.563373+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush_and_sync_log txn(seq 21 len 0x60 crc 0x77fcc5ee) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush 0x558c53be5a40 0x14000~1000 to file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush_range 0x558c53be5a40 pos 0x14000 0x14000~1000 to file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range file now file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x14000 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x15000 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000028 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log log_seq_stable 21 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.563373+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4500 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _flush 0x558c54ea4500 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _close_writer 0x558c54ea4500 type 0 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs open_for_read db/OPTIONS-000010.dbtmp (sequential) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs open_for_read h 0x558c53a43ad0 on file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.563373+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _read h 0x558c53a43ad0 0x0~2000 from file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.563373+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1355 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read left 0x2000 len 0x1355 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read got 4949 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs rename db/OPTIONS-000010.dbtmp -> db/OPTIONS-000011 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs rename db/OPTIONS-000011 file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.563373+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs unlink db/OPTIONS-000005 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _drop_link had refs 1 on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _drop_link destroying file(ino 7 size 0x1354 mtime 2019-12-07T19:33:20.824185+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs readdir db 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs stat db/000007.sst 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs stat db/000007.sst file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 4 rocksdb: DB pointer 0x558c53ab4400 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _upgrade_super from 3, latest 3 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) _upgrade_super done 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _open_collections 2019-12-07T19:33:36.562+0000 7f8f1319f700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:36.562+0000 7f8f1319f700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 2/0 2.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Sum 2/0 2.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 2/0 2.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Sum 2/0 2.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43b80 0x0~12d from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.562125+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read_random left 0x1000 0x0~12d 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read_random got 301 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluefs _read_random h 0x558c53a43c30 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:22.594153+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluefs _read_random got 447 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _open_collections opened meta 0x558c53a70960 cnode(bits 0) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach meta fresh osr 0x558c54ea4b40 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x233/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_start 2019-12-07T19:33:36.562+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread start 2019-12-07T19:33:36.562+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_replay start 2019-12-07T19:33:36.562+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread start 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_replay draining osr 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_register_zombie 0x558c54ea4b40 meta 2019-12-07T19:33:36.562+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all osr_set 0x558c54ea4b40 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all drain 0x558c54ea4b40 2019-12-07T19:33:36.562+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:36.562+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:36.562+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:36.562+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all reaping empty zombie osr 0x558c54ea4b40 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_drain_all done 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_replay completed 0 events 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) is_journal_rotational 0 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 2 osd.7 0 journal looks like ssd 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 2 osd.7 0 boot 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 osd.7 0 configured osd_max_object_name[space]_len looks ok 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:7b3f43c4:::osd_superblock:0# 0x0~0 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:7b3f43c4:::osd_superblock:0# key 0x7f7fffffffffffffff7b3f43c4216f73'd_superblock!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.562+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r 0 v.len 46 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~233 Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore.blob(0x558c54ea2700) get_ref init 0x1000, 1000 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~233 size 0x233 (563) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558c54ea25b0 sbid 0x0)) need 0x0~233 cache has 0x[] 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558c54ea25b0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~233]} 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 1760 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:36.563+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263467 2019-12-07T19:33:36.563+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1199 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558c54ea25b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~233]} 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 _add level 1 near 0 on buffer(0x558c53a69ef0 space 0x558c54ea25c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 = 563 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 osd.7 0 read_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e0 [0,0] lci=[0,0]) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 osd.7 0 get_map 0 - return initial 0x558c53a4f700 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) exists meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c0371625:::snapmapper:0# key 0x7f7fffffffffffffffc03716'%!snapmapper!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 osd.7 0 init creating/touching snapmapper object 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54836f00 seq 1 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c0371625:::snapmapper:0# key 0x7f7fffffffffffffffc03716'%!snapmapper!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1026 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 4205 (1 ios * 4000 + 205 bytes) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes 0x558c548f2280 shared_blobs 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore.extentmap(0x558c548f2388) update #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore.extentmap(0x558c548f2388) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c0371625:::snapmapper:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[] released 0x[] 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) exists meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c1a3fc6e:::purged_snaps:0# key 0x7f7fffffffffffffffc1a3fc'n!purged_snaps!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 osd.7 0 init creating/touching purged_snaps object 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f7500 seq 2 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c1a3fc6e:::purged_snaps:0# key 0x7f7fffffffffffffffc1a3fc'n!purged_snaps!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1027 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread new_nid_max 2050 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7500 cost 4207 (1 ios * 4000 + 207 bytes) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7500 onodes 0x558c548f2500 shared_blobs 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore.extentmap(0x558c548f2608) update #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore.extentmap(0x558c548f2608) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread new_blobid_max 20480 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c1a3fc6e:::purged_snaps:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7500 allocated 0x[] released 0x[] 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 prepare 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7500 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 io_done 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 open_all_classes 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 open_all_classes found journal 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399535 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 166 < min_flush_size 524288 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _get_class adding new class name journal 0x558c53a7b588 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 166 < min_flush_size 524288 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _load_class journal from /usr/lib64/rados-classes/libcls_journal.so 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.562887+0000 allocated 0 extents []) 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x0~a6 to file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.562887+0000 allocated 0 extents []) 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x0 0x0~a6 to file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.562887+0000 allocated 0 extents []) 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _allocate len 0xa6 from 0 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _allocate len 0xa6 from 1 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd120000~10000/10000,10000,0 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 22 (was clean) 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x0 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa6 and padding block with 0xf5a 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xa6 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (22) on file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]), flushing log 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 22 len 0x86 crc 0x895c847f) 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x15000~1000 to file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x15000 0x15000~1000 to file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_class journal status 3 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x15000 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.create flags 3 0x7f8f18793130 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_order flags 1 0x7f8f18792bf0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_splay_width flags 1 0x7f8f18792b90 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_pool_id flags 1 0x7f8f18792440 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_minimum_set flags 1 0x7f8f18792e80 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.set_minimum_set flags 3 0x7f8f18793010 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_active_set flags 1 0x7f8f18792e10 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.set_active_set flags 3 0x7f8f18792ef0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_client flags 1 0x7f8f18793d90 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.client_register flags 3 0x7f8f18795890 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.client_update_data flags 3 0x7f8f187940e0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.client_update_state flags 3 0x7f8f18794660 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.client_unregister flags 3 0x7f8f18798360 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.client_commit flags 3 0x7f8f18794a70 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.client_list flags 1 0x7f8f18795590 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_next_tag_tid flags 1 0x7f8f18792da0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.get_tag flags 1 0x7f8f18798ae0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.tag_create flags 3 0x7f8f18798d80 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.tag_list flags 1 0x7f8f18796570 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.guard_append flags 3 0x7f8f18793490 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method journal.append flags 3 0x7f8f187935d0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _load_class journal success 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 open_all_classes found version 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _get_class adding new class name version 0x558c53a7b108 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _load_class version from /usr/lib64/rados-classes/libcls_version.so 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x16000 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000018 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 22 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 release 0xacd0f0000~10000 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 release done 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _should_compact_log current 0x16000 expected 3000 ratio 7.33333 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread nid_max now 2050 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread blobid_max now 20480 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000426831s (0.000000199s flush + 0.000426632s kv commit) 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54836f00> 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:36.563+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 125 < min_flush_size 524288 2019-12-07T19:33:36.563+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 finishing 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 125 < min_flush_size 524288 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 0x558c548f2280 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7500 kv_submitted 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xa6~7d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.563+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xa6 0xa6~7d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.563+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xa6 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa6 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x123 and padding block with 0xedd 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_class version status 3 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method version.set flags 3 0x7f8f18577d30 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method version.inc flags 3 0x7f8f18578740 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method version.inc_conds flags 3 0x7f8f18578740 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method version.read flags 1 0x7f8f18577bb0 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 register_cxx_method version.check_conds flags 1 0x7f8f18578590 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _load_class version success 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 open_all_classes found log 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _get_class adding new class name log 0x558c53a7b228 2019-12-07T19:33:36.563+0000 7f8f2a397bc0 10 _load_class log from /usr/lib64/rados-classes/libcls_log.so 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x123 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:36.563+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000083853s (0.000000087s flush + 0.000083766s kv commit) 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:36.563+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7500> 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.563+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 kv_submitted 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7500 2019-12-07T19:33:36.563+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 finishing 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7500 onodes 0x558c548f2500 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7500 done 2019-12-07T19:33:36.563+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54ea4b40 q now empty 2019-12-07T19:33:36.564+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7500 [] 2019-12-07T19:33:36.564+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:36.564+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish empty zombie osr 0x558c54ea4b40 already reaped 2019-12-07T19:33:36.564+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_class log status 3 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method log.add flags 3 0x7f8f18360c40 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method log.list flags 1 0x7f8f1835f4a0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method log.trim flags 3 0x7f8f1835e9e0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method log.info flags 1 0x7f8f1835e4e0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 _load_class log success 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 open_all_classes found rbd 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 _get_class adding new class name rbd 0x558c53a7b348 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 _load_class rbd from /usr/lib64/rados-classes/libcls_rbd.so 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_class rbd status 3 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.create flags 3 0x7f8f18107c90 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_features flags 1 0x7f8f180fdfc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_features flags 3 0x7f8f180fe1a0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_size flags 1 0x7f8f18110150 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_size flags 3 0x7f8f18106200 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_snapcontext flags 1 0x7f8f18107300 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_object_prefix flags 1 0x7f8f180ff3b0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_data_pool flags 1 0x7f8f180ff520 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_snapshot_name flags 1 0x7f8f18111a20 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_snapshot_timestamp flags 1 0x7f8f18111bf0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snapshot_get flags 1 0x7f8f18112ae0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snapshot_add flags 3 0x7f8f18112ec0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snapshot_remove flags 3 0x7f8f18112430 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snapshot_rename flags 3 0x7f8f18111da0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snapshot_trash_add flags 3 0x7f8f181170d0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_all_features flags 1 0x7f8f180f0f70 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_parent flags 1 0x7f8f18111150 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_parent flags 3 0x7f8f18105dc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.remove_parent flags 3 0x7f8f18110110 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.parent_get flags 1 0x7f8f18106dc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.parent_overlap_get flags 1 0x7f8f18111620 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.parent_attach flags 3 0x7f8f18105ff0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.parent_detach flags 3 0x7f8f18110130 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_protection_status flags 3 0x7f8f18110700 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_protection_status flags 1 0x7f8f181104c0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_stripe_unit_count flags 1 0x7f8f180fe540 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_stripe_unit_count flags 3 0x7f8f180fe7c0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_create_timestamp flags 1 0x7f8f180fec40 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_access_timestamp flags 1 0x7f8f180fee50 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_modify_timestamp flags 1 0x7f8f180ff060 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_flags flags 1 0x7f8f181109a0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_flags flags 3 0x7f8f18110c60 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.op_features_get flags 1 0x7f8f180ff270 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.op_features_set flags 3 0x7f8f180fc850 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.metadata_list flags 1 0x7f8f18109a00 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.metadata_set flags 3 0x7f8f181173d0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.metadata_remove flags 3 0x7f8f180f64e0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.metadata_get flags 1 0x7f8f180f6640 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snapshot_get_limit flags 1 0x7f8f180ffc20 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snapshot_set_limit flags 3 0x7f8f18100d20 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.child_attach flags 3 0x7f8f18116310 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.child_detach flags 3 0x7f8f18115710 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.children_list flags 1 0x7f8f18116030 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.migration_set flags 3 0x7f8f180fd070 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.migration_set_state flags 3 0x7f8f180fd730 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.migration_get flags 1 0x7f8f180fd920 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.migration_remove flags 3 0x7f8f180fdfa0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_modify_timestamp flags 3 0x7f8f180ff8f0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_access_timestamp flags 3 0x7f8f180ff800 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.add_child flags 3 0x7f8f18118d90 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.remove_child flags 3 0x7f8f18119170 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_children flags 1 0x7f8f181195d0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.get_id flags 1 0x7f8f180f2d20 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.set_id flags 3 0x7f8f180f2f30 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_get_id flags 1 0x7f8f180f58a0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_get_name flags 1 0x7f8f180f5eb0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_list flags 1 0x7f8f18102970 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_add_image flags 3 0x7f8f18108ec0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_remove_image flags 3 0x7f8f180f5dc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_rename_image flags 3 0x7f8f18108d80 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_state_assert flags 1 0x7f8f180ff9e0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.dir_state_set flags 3 0x7f8f18101130 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.object_map_load flags 1 0x7f8f1810a690 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.object_map_save flags 3 0x7f8f1810aba0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.object_map_resize flags 3 0x7f8f1810be60 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.object_map_update flags 3 0x7f8f1810b410 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.object_map_snap_add flags 3 0x7f8f1810a760 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.object_map_snap_remove flags 3 0x7f8f1810ad40 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snap_list flags 1 0x7f8f180ffd60 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snap_add flags 3 0x7f8f18100000 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snap_remove flags 3 0x7f8f180f3110 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.snap_rename flags 3 0x7f8f180f3650 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_uuid_get flags 1 0x7f8f181049d0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_uuid_set flags 3 0x7f8f18107a20 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_mode_get flags 1 0x7f8f180f1d60 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_mode_set flags 3 0x7f8f18105070 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_peer_ping flags 3 0x7f8f18116fc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_peer_list flags 1 0x7f8f18101ab0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_peer_add flags 3 0x7f8f18104dd0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_peer_remove flags 3 0x7f8f180f6e10 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_peer_set_client flags 3 0x7f8f180f9580 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_peer_set_cluster flags 3 0x7f8f18101d50 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_peer_set_direction flags 3 0x7f8f180f6c10 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_list flags 1 0x7f8f180fb230 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_get_image_id flags 1 0x7f8f180f7880 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_get flags 1 0x7f8f180f70e0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_set flags 3 0x7f8f180f7780 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_remove flags 3 0x7f8f18104550 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_status_set flags 11 0x7f8f181056c0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_status_remove flags 3 0x7f8f18104610 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_status_get flags 1 0x7f8f18102360 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_status_list flags 1 0x7f8f1810d210 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_status_get_summary flags 1 0x7f8f1810d970 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_status_remove_down flags 3 0x7f8f180fb210 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_instance_get flags 1 0x7f8f180f93c0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_instance_list flags 1 0x7f8f1810e3e0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_instances_list flags 1 0x7f8f181027b0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_instances_add flags 11 0x7f8f180f7c20 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_instances_remove flags 3 0x7f8f180f7dc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_map_list flags 1 0x7f8f180fa300 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_map_update flags 2 0x7f8f180f7ed0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.mirror_image_map_remove flags 2 0x7f8f180f81b0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_dir_list flags 1 0x7f8f181031d0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_dir_add flags 3 0x7f8f18109420 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_dir_remove flags 3 0x7f8f180f63f0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_dir_rename flags 3 0x7f8f18109560 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_image_remove flags 3 0x7f8f180f1330 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_image_list flags 1 0x7f8f180fbb80 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_image_set flags 3 0x7f8f180f1dc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.image_group_add flags 3 0x7f8f18100550 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.image_group_remove flags 3 0x7f8f181008b0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.image_group_get flags 1 0x7f8f18100b60 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_snap_set flags 3 0x7f8f18114cf0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_snap_remove flags 3 0x7f8f18113f50 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_snap_get_by_id flags 1 0x7f8f18114080 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.group_snap_list flags 1 0x7f8f181149e0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.trash_add flags 3 0x7f8f181096a0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.trash_remove flags 3 0x7f8f180f8340 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.trash_list flags 1 0x7f8f1810e870 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.trash_get flags 1 0x7f8f180f85b0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.trash_state_set flags 3 0x7f8f180f87d0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.namespace_add flags 3 0x7f8f180f8be0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.namespace_remove flags 3 0x7f8f180f8dc0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.namespace_list flags 1 0x7f8f180fa4a0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.copyup flags 3 0x7f8f180f1130 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.sparse_copyup flags 3 0x7f8f181180b0 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.assert_snapc_seq flags 3 0x7f8f180f1010 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 register_cxx_method rbd.sparsify flags 3 0x7f8f180f2410 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 _load_class rbd success 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 open_all_classes found sdk 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 0 _get_class not permitted to load sdk 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 open_all_classes found rgw 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 _get_class adding new class name rgw 0x558c53a7b468 2019-12-07T19:33:36.564+0000 7f8f2a397bc0 10 _load_class rgw from /usr/lib64/rados-classes/libcls_rgw.so 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_class rgw status 3 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_init_index flags 3 0x7f8f17df4600 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_set_tag_timeout flags 3 0x7f8f17dfc340 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_list flags 1 0x7f8f17e02af0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_check_index flags 1 0x7f8f17e045f0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_rebuild_index flags 3 0x7f8f17e04470 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_update_stats flags 3 0x7f8f17dfd2c0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_prepare_op flags 3 0x7f8f17e05000 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_complete_op flags 3 0x7f8f17e05520 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_link_olh flags 3 0x7f8f17e07080 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_unlink_instance flags 3 0x7f8f17e06570 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_read_olh_log flags 1 0x7f8f17dff0f0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_trim_olh_log flags 3 0x7f8f17dfeae0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bucket_clear_olh flags 3 0x7f8f17dfe600 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.obj_remove flags 3 0x7f8f17e01c90 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.obj_store_pg_ver flags 2 0x7f8f17df7600 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.obj_check_attrs_prefix flags 1 0x7f8f17df73c0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.obj_check_mtime flags 1 0x7f8f17df7140 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bi_get flags 1 0x7f8f17df90b0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bi_put flags 3 0x7f8f17df6fc0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bi_list flags 1 0x7f8f17dffa10 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bi_log_list flags 1 0x7f8f17e07d60 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bi_log_trim flags 3 0x7f8f17df9600 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.dir_suggest_changes flags 3 0x7f8f17dfd570 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bi_log_resync flags 3 0x7f8f17dfcea0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.bi_log_stop flags 3 0x7f8f17dfcb60 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.user_usage_log_add flags 3 0x7f8f17dfb330 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.user_usage_log_read flags 1 0x7f8f17dfad90 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.user_usage_log_trim flags 3 0x7f8f17dfb1b0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.usage_log_clear flags 2 0x7f8f17df34d0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.gc_set_entry flags 3 0x7f8f17df8c90 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.gc_defer_entry flags 3 0x7f8f17df8b70 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.gc_list flags 1 0x7f8f17df7a80 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.gc_remove flags 3 0x7f8f17e04d20 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.lc_get_entry flags 1 0x7f8f17df6d20 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.lc_set_entry flags 3 0x7f8f17df7870 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.lc_rm_entry flags 3 0x7f8f17df77b0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.lc_get_next_entry flags 1 0x7f8f17df6a30 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.lc_put_head flags 3 0x7f8f17df6860 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.lc_get_head flags 1 0x7f8f17df6610 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.lc_list_entries flags 1 0x7f8f17df6070 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.reshard_add flags 3 0x7f8f17df5ea0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.reshard_list flags 1 0x7f8f17df58e0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.reshard_get flags 1 0x7f8f17df56b0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.reshard_remove flags 3 0x7f8f17df54b0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.set_bucket_resharding flags 3 0x7f8f17dfc9a0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.clear_bucket_resharding flags 3 0x7f8f17dfc840 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.guard_bucket_resharding flags 1 0x7f8f17dfc6f0 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 register_cxx_method rgw.get_bucket_resharding flags 1 0x7f8f17dfc490 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 _load_class rgw success 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 open_all_classes found kvs 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 0 _get_class not permitted to load kvs 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 open_all_classes found refcount 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 _get_class adding new class name refcount 0x558c53a7ada8 2019-12-07T19:33:36.566+0000 7f8f2a397bc0 10 _load_class refcount from /usr/lib64/rados-classes/libcls_refcount.so 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_class refcount status 3 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method refcount.get flags 3 0x7f8f17aa42e0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method refcount.put flags 3 0x7f8f17aa4a10 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method refcount.set flags 3 0x7f8f17aa3d90 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method refcount.read flags 1 0x7f8f17aa5280 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class refcount success 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 open_all_classes found user 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _get_class adding new class name user 0x558c53a7aec8 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class user from /usr/lib64/rados-classes/libcls_user.so 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_class user status 3 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method user.set_buckets_info flags 3 0x7f8f177e3800 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method user.complete_stats_sync flags 3 0x7f8f177dfd50 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method user.remove_bucket flags 3 0x7f8f177df070 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method user.list_buckets flags 1 0x7f8f177e1930 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method user.get_header flags 1 0x7f8f177deb60 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method user.reset_user_stats flags 3 0x7f8f177e0360 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class user success 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 open_all_classes found hello 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _get_class adding new class name hello 0x558c53a7bb28 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class hello from /usr/lib64/rados-classes/libcls_hello.so 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 0 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/15.0.0-8038-g503c226/rpm/el7/BUILD/ceph-15.0.0-8038-g503c226/src/cls/hello/cls_hello.cc:313: loading cls_hello 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_class hello status 3 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.say_hello flags 1 0x7f8f175c64d0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.record_hello flags 10 0x7f8f175c7360 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.write_return_data flags 2 0x7f8f175c6670 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.write_too_much_return_data flags 2 0x7f8f175c6550 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.replay flags 1 0x7f8f175c64a0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.turn_it_to_11 flags 11 0x7f8f175c69a0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.bad_reader flags 2 0x7f8f175c6490 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method hello.bad_writer flags 1 0x7f8f175c6480 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class hello success 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 open_all_classes found otp 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _get_class adding new class name otp 0x558c53a7bd68 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class otp from /usr/lib64/rados-classes/libcls_otp.so 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_class otp status 3 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method otp.otp_set flags 3 0x7f8f173a94c0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method otp.otp_get flags 1 0x7f8f173aadd0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method otp.otp_check flags 3 0x7f8f173a7d50 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method otp.otp_get_result flags 1 0x7f8f173a86d0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method otp.otp_remove flags 3 0x7f8f173ac380 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method otp.get_current_time flags 1 0x7f8f173a7740 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class otp success 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 open_all_classes found cas 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _get_class adding new class name cas 0x558c53a7be88 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class cas from /usr/lib64/rados-classes/libcls_cas.so 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_class cas status 3 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method cas.chunk_get flags 3 0x7f8f16f78570 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method cas.chunk_put flags 3 0x7f8f16f79230 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method cas.chunk_set flags 3 0x7f8f16f79b60 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method cas.chunk_read flags 1 0x7f8f16f78b30 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method cas.cas_write_or_get flags 3 0x7f8f16f78cc0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 register_cxx_method cas.has_chunk flags 1 0x7f8f16f77aa0 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class cas success 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 open_all_classes found rgw_gc 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _get_class adding new class name rgw_gc 0x558c55158148 2019-12-07T19:33:36.567+0000 7f8f2a397bc0 10 _load_class rgw_gc from /usr/lib64/rados-classes/libcls_rgw_gc.so 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_class rgw_gc status 3 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_init flags 3 0x7f8f16d36790 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_enqueue flags 3 0x7f8f16d36e60 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_list_entries flags 1 0x7f8f16d37f70 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_remove_entries flags 3 0x7f8f16d3d950 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_update_entry flags 3 0x7f8f16d3aba0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class rgw_gc success 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 open_all_classes found cephfs 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _get_class adding new class name cephfs 0x558c55158268 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class cephfs from /usr/lib64/rados-classes/libcls_cephfs.so 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/15.0.0-8038-g503c226/rpm/el7/BUILD/ceph-15.0.0-8038-g503c226/src/cls/cephfs/cls_cephfs.cc:198: loading cephfs 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_class cephfs status 3 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method cephfs.accumulate_inode_metadata flags 3 0x7f8f16b15900 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class cephfs success 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 open_all_classes found numops 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _get_class adding new class name numops 0x558c551584a8 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class numops from /usr/lib64/rados-classes/libcls_numops.so 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_class numops status 3 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method numops.add flags 3 0x7f8f168ffba0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method numops.mul flags 3 0x7f8f168ff170 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class numops success 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 open_all_classes found lua 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 _get_class not permitted to load lua 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 open_all_classes found queue 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 _get_class not permitted to load queue 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 open_all_classes found lock 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _get_class adding new class name lock 0x558c551585c8 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class lock from /usr/lib64/rados-classes/libcls_lock.so 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_class lock status 3 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method lock.lock flags 11 0x7f8f166e49c0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method lock.unlock flags 11 0x7f8f166e20a0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method lock.break_lock flags 3 0x7f8f166e18d0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method lock.get_info flags 1 0x7f8f166e3a50 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method lock.list_locks flags 1 0x7f8f166dfb00 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method lock.assert_locked flags 9 0x7f8f166e0a50 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method lock.set_cookie flags 11 0x7f8f166e2740 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class lock success 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 open_all_classes found timeindex 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _get_class adding new class name timeindex 0x558c55158808 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class timeindex from /usr/lib64/rados-classes/libcls_timeindex.so 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_class timeindex status 3 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method timeindex.add flags 3 0x7f8f164c8080 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method timeindex.list flags 1 0x7f8f164c6fe0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 register_cxx_method timeindex.trim flags 3 0x7f8f164c66b0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 _load_class timeindex success 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 osd.7 0 crush map has features 288232575208783872, adjusting msgr requires for clients 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 osd.7 0 crush map has features 288232575208783872 was 8705, adjusting msgr requires for mons 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 osd.7 0 crush map has features 288232575208783872, adjusting msgr requires for osds 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 osd.7 0 check_osdmap_features enabling on-disk ERASURE CODES compat feature 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 osd.7 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e0 [0,0] lci=[0,0]) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f7800 seq 3 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x233 (563) bytes fadvise_flags 0x0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 _rm buffer(0x558c53a69ef0 space 0x558c54ea25c8 0x0~1000 clean) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~233 Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x481) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 0 blobs 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~233: 0x0~233 Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24e 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 4803 (1 ios * 4000 + 803 bytes) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes 0x558c53a91900 shared_blobs 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[] released 0x[] 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 osd.7 0 create_recoverystate_perf 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4323 < min_flush_size 524288 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 osd.7 0 clear_temp_objects 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 osd.7 0 load_pgs 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:46e858f0:::pg_num_history:head# 0x0~0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:46e858f0:::pg_num_history:head# key 0x7f7fffffffffffffff46e858f0217067'_num_history!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:46e858f0:::pg_num_history:head# 0x0~0 = -2 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 osd.7 0 load_pgs pg_num_history pg_num_history(e0 pg_nums {} deleted_pools ) 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 osd.7 0 load_pgs ignoring unrecognized meta 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 0 osd.7 0 load_pgs opened 0 pgs 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 2 osd.7 0 superblock: I am osd.7 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 10 osd.7 0 create_logger 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4346 < min_flush_size 524288 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4346 < min_flush_size 524288 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x123~10fa to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x123 0x123~10fa to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x123 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x123 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x21d and padding block with 0xde3 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x121d 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:36.568+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:36.568+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000167292s (0.000000084s flush + 0.000167208s kv commit) 2019-12-07T19:33:36.568+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:36.568+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:36.568+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7800> 2019-12-07T19:33:36.568+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.568+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:33:36.568+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:33:36.568+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f7800 osr 0x558c54ea4b40 2019-12-07T19:33:36.568+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) prepare_write seq 1 0x2000~1000 crc 6ae57bb4 2019-12-07T19:33:36.569+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 Processor -- start 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 -1 osd.7 0 log_to_monitors {default=true} 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 Processor -- start 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 Processor -- start 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 Processor -- start 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 Processor -- start 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 Processor -- start 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 Processor -- start 2019-12-07T19:33:36.569+0000 7f8f0918b700 10 osd.7 0 agent_entry start 2019-12-07T19:33:36.569+0000 7f8f0918b700 20 osd.7 0 agent_entry empty queue 2019-12-07T19:33:36.569+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:36.569+0000 7f8f0097a700 5 osd.7 0 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 --2- [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c5476d000 0x558c53aa4580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.569+0000 7f8f0097a700 20 osd.7 0 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] --> v1:172.21.15.109:6789/0 -- auth(proto 0 26 bytes epoch 0) v1 -- 0x558c55036000 con 0x558c5476d800 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] --> v1:172.21.15.131:6789/0 -- auth(proto 0 26 bytes epoch 0) v1 -- 0x558c55036240 con 0x558c5476d400 2019-12-07T19:33:36.569+0000 7f8f2a397bc0 1 -- [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_getmap magic: 0 v1 -- 0x558c53a6ec40 con 0x558c5476d000 2019-12-07T19:33:36.569+0000 7f8f24457700 1 --1- [v2:0.0.0.0:6824/36939,v1:0.0.0.0:6825/36939] >> v1:172.21.15.109:6789/0 conn(0x558c5476d800 0x558c54897000 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=1).handle_server_banner_and_identify peer v1:172.21.15.109:6789/0 says I am v1:172.21.15.109:40586/0 (socket says 172.21.15.109:40586) 2019-12-07T19:33:36.569+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] learned_addr learned my addr [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] (peer_addr_for_me v1:172.21.15.109:0/0) 2019-12-07T19:33:36.569+0000 7f8f24457700 10 osd.7 0 new session (outgoing) 0x558c53a85380 con=0x558c53ae5000 addr=v2:172.21.15.109:6824/36939 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 ms_handle_connect con 0x558c5476d800 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 ms_handle_connect on mon 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 requeue_pg_temp 0 + 0 -> 0 2019-12-07T19:33:36.570+0000 7f8f164bb700 20 osd.7 0 _send_ready_to_merge ready_to_merge_source {} not_ready_to_merge_source ready_to_merge_target {} not_ready_to_merge_target {} sent_ready_to_merge_source 2019-12-07T19:33:36.570+0000 7f8f164bb700 20 osd.7 0 send_pg_created 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 requeue_failures 0 + 0 -> 0 2019-12-07T19:33:36.570+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c5476d000 0x558c53aa4580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v1:172.21.15.109:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2411797932 0 0) 0x558c55036000 con 0x558c5476d800 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> v1:172.21.15.109:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x558c55194480 con 0x558c5476d800 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 ms_handle_connect con 0x558c5476d400 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 ms_handle_connect on mon 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 requeue_pg_temp 0 + 0 -> 0 2019-12-07T19:33:36.570+0000 7f8f164bb700 20 osd.7 0 _send_ready_to_merge ready_to_merge_source {} not_ready_to_merge_source ready_to_merge_target {} not_ready_to_merge_target {} sent_ready_to_merge_source 2019-12-07T19:33:36.570+0000 7f8f164bb700 20 osd.7 0 send_pg_created 2019-12-07T19:33:36.570+0000 7f8f164bb700 10 osd.7 0 requeue_failures 0 + 0 -> 0 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> v1:172.21.15.131:6789/0 conn(0x558c5476d400 legacy=0x558c54896800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 57 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> v1:172.21.15.131:6789/0 conn(0x558c5476d400 legacy=0x558c54896800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 --1- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> v1:172.21.15.131:6789/0 conn(0x558c5476d400 0x558c54896800 :-1 s=OPENED pgs=41 cs=1 l=1).handle_message read tag failed 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 --1- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> v1:172.21.15.131:6789/0 conn(0x558c5476d400 0x558c54896800 :-1 s=OPENED pgs=41 cs=1 l=1).fault on lossy channel, failing 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v1:172.21.15.109:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 935+0+0 (unknown 3741322179 0 0) 0x558c55194480 con 0x558c5476d800 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> v1:172.21.15.109:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- 0x558c55036000 con 0x558c5476d800 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v1:172.21.15.109:6789/0 3 ==== mon_map magic: 0 v1 ==== 383+0+0 (unknown 1681456789 0 0) 0x558c53aa8a00 con 0x558c5476d800 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c5476d000 msgr2=0x558c53aa4580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c5476d000 0x558c53aa4580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> v1:172.21.15.131:6789/0 conn(0x558c5476d400 legacy=0x558c54896800 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> v1:172.21.15.109:6789/0 conn(0x558c5476d800 legacy=0x558c54897000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c551b6000 0x558c53aa5b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] conn(0x558c5476d000 0x558c53aa6100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] conn(0x558c53ae4c00 0x558c53aa6680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_getmap magic: 0 v1 -- 0x558c53a6ec40 con 0x558c5476d000 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] -- mon_getmap magic: 0 v1 -- 0x558c53a6fa40 con 0x558c53ae4c00 2019-12-07T19:33:36.570+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_getmap magic: 0 v1 -- 0x558c53a6fc00 con 0x558c551b6000 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] conn(0x558c5476d000 0x558c53aa6100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.570+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] conn(0x558c53ae4c00 0x558c53aa6680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.570+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c551b6000 0x558c53aa5b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c551b6000 msgr2=0x558c53aa5b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c551b6000 0x558c53aa5b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] conn(0x558c53ae4c00 msgr2=0x558c53aa6680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] conn(0x558c53ae4c00 0x558c53aa6680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:36.570+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558c548cd800 con 0x558c5476d000 2019-12-07T19:33:36.571+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- auth(proto 2 2 bytes epoch 0) v1 -- 0x558c55036240 con 0x558c5476d000 2019-12-07T19:33:36.571+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558c551b6000 0x558c53aa5b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rx=0 tx=0).handle_auth_reply_more state changed! 2019-12-07T19:33:36.571+0000 7f8f164bb700 10 osd.7 0 ms_handle_connect con 0x558c5476d000 2019-12-07T19:33:36.571+0000 7f8f164bb700 10 osd.7 0 ms_handle_connect on mon 2019-12-07T19:33:36.571+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] conn(0x558c5476d000 0x558c53aa6100 crc :-1 s=READY pgs=57 cs=0 l=1 rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:36.571+0000 7f8f164bb700 10 osd.7 0 requeue_pg_temp 0 + 0 -> 0 2019-12-07T19:33:36.571+0000 7f8f164bb700 20 osd.7 0 _send_ready_to_merge ready_to_merge_source {} not_ready_to_merge_source ready_to_merge_target {} not_ready_to_merge_target {} sent_ready_to_merge_source 2019-12-07T19:33:36.571+0000 7f8f164bb700 20 osd.7 0 send_pg_created 2019-12-07T19:33:36.571+0000 7f8f164bb700 10 osd.7 0 requeue_failures 0 + 0 -> 0 2019-12-07T19:33:36.571+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 1 ==== mon_map magic: 0 v1 ==== 383+0+0 (crc 0 0 0) 0x558c53aa8200 con 0x558c5476d000 2019-12-07T19:33:36.571+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (crc 0 0 0) 0x558c53a71e00 con 0x558c5476d000 2019-12-07T19:33:36.571+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 3 ==== mon_map magic: 0 v1 ==== 383+0+0 (crc 0 0 0) 0x558c548cd800 con 0x558c5476d000 2019-12-07T19:33:36.571+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (crc 0 0 0) 0x558c55036240 con 0x558c5476d000 2019-12-07T19:33:36.571+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:36.571+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:36.571+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) collect_metadata 2019-12-07T19:33:36.571+0000 7f8f2a397bc0 10 bluestore(/var/lib/ceph/osd/ceph-7) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:36.571+0000 7f8f2a397bc0 1 bluestore(/var/lib/ceph/osd/ceph-7) collect_metadata devices span numa nodes 0 2019-12-07T19:33:36.571+0000 7f8f2a397bc0 10 osd.7 0 mon_cmd_maybe_osd_create cmd: {"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} 2019-12-07T19:33:36.571+0000 7f8f2a397bc0 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_command({"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} v 0) v1 -- 0x558c548cda00 con 0x558c5476d000 2019-12-07T19:33:36.614+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.664+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.714+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.764+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.814+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.864+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.914+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.965+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.015+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.064+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.115+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.165+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.215+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.265+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.315+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.366+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.416+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.466+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.513+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 5 ==== mon_command_ack([{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]=0 set osd(s) 7 to class 'ssd' v11) v1 ==== 131+0+0 (crc 0 0 0) 0x558c551aea00 con 0x558c5476d000 2019-12-07T19:33:37.513+0000 7f8f2a397bc0 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:37.513+0000 7f8f2a397bc0 10 osd.7 0 update_crush_location crush location is "host=smithi109", "root=default" 2019-12-07T19:33:37.513+0000 7f8f2a397bc0 10 osd.7 0 mon_cmd_maybe_osd_create cmd: {"prefix": "osd crush create-or-move", "id": 7, "weight":0.0879, "args": ["host=smithi109", "root=default"]} 2019-12-07T19:33:37.513+0000 7f8f2a397bc0 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_command({"prefix": "osd crush create-or-move", "id": 7, "weight":0.0879, "args": ["host=smithi109", "root=default"]} v 0) v1 -- 0x558c548cdc00 con 0x558c5476d000 2019-12-07T19:33:37.515+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845356676 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.566+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.568+0000 7f8f1fbbe700 10 osd.7 0 tick_without_osd_lock 2019-12-07T19:33:37.568+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:37.591+0000 7f8f21451700 10 osd.7 0 tick 2019-12-07T19:33:37.591+0000 7f8f21451700 10 osd.7 0 do_waiters -- start 2019-12-07T19:33:37.591+0000 7f8f21451700 10 osd.7 0 do_waiters -- finish 2019-12-07T19:33:37.591+0000 7f8f21451700 20 osd.7 0 tick last_purged_snaps_scrub 0.000000 next 60.000000 ... now 2019-12-07T19:33:37.591+0000 7f8f21451700 10 osd.7 0 scrub_purged_snaps 2019-12-07T19:33:37.591+0000 7f8f21451700 0 log_channel(cluster) log [DBG] : purged_snaps scrub starts 2019-12-07T19:33:37.591+0000 7f8f21451700 10 snap_mapper.run 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 0 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 0 2019-12-07T19:33:37.591+0000 7f8f21451700 10 snap_mapper.run end, found 0 stray 2019-12-07T19:33:37.591+0000 7f8f21451700 0 log_channel(cluster) log [DBG] : purged_snaps scrub ok 2019-12-07T19:33:37.591+0000 7f8f21451700 10 osd.7 0 scrub_purged_snaps done queueing pgs, updating superblock 2019-12-07T19:33:37.591+0000 7f8f21451700 10 osd.7 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e0 [0,0] lci=[0,0]) 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54836f00 seq 4 2019-12-07T19:33:37.591+0000 7f8f21451700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54836f00 0 blobs 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 4803 (1 ios * 4000 + 803 bytes) 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes 0x558c53a91900 shared_blobs 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[] released 0x[] 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:33:37.591+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:33:37.591+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:33:37.591+0000 7f8f21451700 10 osd.7 0 scrub_purged_snaps done 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4270 < min_flush_size 524288 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399535 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x121d~10c5 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x121d 0x121d~10c5 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x121d 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x21d 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2e2 and padding block with 0xd1e 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x22e2 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:37.591+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000105508s (0.000000099s flush + 0.000105409s kv commit) 2019-12-07T19:33:37.591+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54836f00> 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:37.591+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54836f00 osr 0x558c54ea4b40 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) _discard drop 1 0x2000~1000 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) prepare_write seq 2 0x2000~1000 crc a9c3144f 2019-12-07T19:33:37.591+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:37.616+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.666+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.716+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.766+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.816+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.867+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.917+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.967+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.017+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.067+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.117+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.167+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.217+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.268+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.318+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.368+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.418+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.468+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.514+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 6 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0879, "args": ["host=smithi109", "root=default"]}]=0 create-or-move updating item name 'osd.7' weight 0.0879 at location {host=smithi109,root=default} to crush map v12) v1 ==== 252+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c5476d000 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7 0 ensuring pgs have consumed prior maps 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 7 osd.7 0 consume_map version 0 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 20 osd.7 0 prune_pg_created 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:0.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:1.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:2.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:3.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:4.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:5.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:6.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7:7.consume_map 0 (was 0) 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7 0 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 0 osd.7 0 done with init, starting boot process 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({mgrmap=0+,osd_pg_creates=0+}) v3 -- 0x558c548cde00 con 0x558c5476d000 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 1 osd.7 0 start_boot 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 10 osd.7 0 start_boot - have maps 0..0 2019-12-07T19:33:38.515+0000 7f8f2a397bc0 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x558c53a6f180 con 0x558c5476d000 2019-12-07T19:33:38.517+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415076 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.517+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 7 ==== mgrmap(e 4) v1 ==== 45131+0+0 (crc 0 0 0) 0x558c548f6000 con 0x558c5476d000 2019-12-07T19:33:38.518+0000 7f8f164bb700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 0x558c53aa6c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:38.518+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 0x558c53aa6c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:38.519+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.7) v3 -- 0x558c55038580 con 0x558c551b6400 2019-12-07T19:33:38.519+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 0x558c53aa6c00 crc :-1 s=READY pgs=23 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:38.519+0000 7f8f164bb700 10 osd.7 0 ms_handle_connect con 0x558c551b6400 2019-12-07T19:33:38.519+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 8 ==== mon_get_version_reply(handle=1 version=12) v2 ==== 24+0+0 (crc 0 0 0) 0x558c53a71c20 con 0x558c5476d000 2019-12-07T19:33:38.519+0000 7f8f1299e700 10 osd.7 0 _preboot _preboot mon has osdmaps 1..12 2019-12-07T19:33:38.519+0000 7f8f1299e700 5 osd.7 0 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:38.519+0000 7f8f1299e700 20 osd.7 0 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:38.519+0000 7f8f1299e700 -1 osd.7 0 waiting for initial osdmap 2019-12-07T19:33:38.519+0000 7f8f1299e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=1}) v3 -- 0x558c551af400 con 0x558c5476d000 2019-12-07T19:33:38.519+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 msgr2=0x558c53aa6c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:33:38.519+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 msgr2=0x558c53aa6c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:38.519+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 0x558c53aa6c00 crc :-1 s=READY pgs=23 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:38.519+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 0x558c53aa6c00 crc :-1 s=READY pgs=23 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:38.519+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] reap_dead start 2019-12-07T19:33:38.519+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 msgr2=0x558c53aa6c00 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:33:38.519+0000 7f8f164bb700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c551b6400 0x558c53aa6c00 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:38.525+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 9 ==== osd_map(1..12 src has 1..12) v4 ==== 5778+0+0 (crc 0 0 0) 0x558c53a78fc0 con 0x558c5476d000 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 OSD::ms_dispatch: osd_map(1..12 src has 1..12) v4 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 do_waiters -- start 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 do_waiters -- finish 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 _dispatch 0x558c53a78fc0 osd_map(1..12 src has 1..12) v4 2019-12-07T19:33:38.525+0000 7f8f164bb700 3 osd.7 0 handle_osd_map epochs [1,12], i have 0, src has [1,12] 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 1 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 1, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 2 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8d9276bf:::osdmap.1:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8d9276bf:::osdmap.1:0# key 0x7f7fffffffffffffff8d9276bf216f73'dmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8d9276bf:::osdmap.1:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 2, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 3 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:829276bf:::osdmap.2:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:829276bf:::osdmap.2:0# key 0x7f7fffffffffffffff829276bf216f73'dmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:829276bf:::osdmap.2:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 3, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 4 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:887276bf:::osdmap.3:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:887276bf:::osdmap.3:0# key 0x7f7fffffffffffffff887276bf216f73'dmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:887276bf:::osdmap.3:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 4, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 5 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:85f276bf:::osdmap.4:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:85f276bf:::osdmap.4:0# key 0x7f7fffffffffffffff85f276bf216f73'dmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:85f276bf:::osdmap.4:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 5, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 6 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8ef276bf:::osdmap.5:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8ef276bf:::osdmap.5:0# key 0x7f7fffffffffffffff8ef276bf216f73'dmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8ef276bf:::osdmap.5:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 6, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 7 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:803276bf:::osdmap.6:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:803276bf:::osdmap.6:0# key 0x7f7fffffffffffffff803276bf216f73'dmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:803276bf:::osdmap.6:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 7, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 8 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8bb276bf:::osdmap.7:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8bb276bf:::osdmap.7:0# key 0x7f7fffffffffffffff8bb276bf216f73'dmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8bb276bf:::osdmap.7:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 8, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 9 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:86b276bf:::osdmap.8:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:86b276bf:::osdmap.8:0# key 0x7f7fffffffffffffff86b276bf216f73'dmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:86b276bf:::osdmap.8:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 9, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 10 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8c4276bf:::osdmap.9:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8c4276bf:::osdmap.9:0# key 0x7f7fffffffffffffff8c4276bf216f73'dmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:8c4276bf:::osdmap.9:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 10, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 11 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3890e826:::osdmap.10:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3890e826:::osdmap.10:0# key 0x7f7fffffffffffffff3890e8'&!osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3890e826:::osdmap.10:0# 0x0~0 = -2 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 osd.7 0 got_full_map 11, nothing requested 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 osd.7 0 handle_osd_map got inc map for epoch 12 2019-12-07T19:33:38.525+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3570e826:::osdmap.11:0# 0x0~0 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3570e826:::osdmap.11:0# key 0x7f7fffffffffffffff3570e8'&!osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.525+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.525+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3570e826:::osdmap.11:0# 0x0~0 = -2 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 osd.7 0 got_full_map 12, nothing requested 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 osd.7 0 handle_osd_map pg_num_history pg_num_history(e12 pg_nums {} deleted_pools ) 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {1={},2={},3={},4={},5={},6={},7={},8={},9={},10={},11={},12={}} 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 osd.7 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e12 [1,12] lci=[0,0]) 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f6000 seq 5 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:60c2fa6d:::inc_osdmap.1:0# key 0x7f7fffffffffffffff60c2fa'm!inc_osdmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1028 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~3b2 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc4e on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523a850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523ad20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523a850 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523ad20 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a850) get_ref 0x0~3b2 Blob(0x558c5523a850 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523ad20 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a850) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~3b2: 0x0~3b2 Blob(0x558c5523a850 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3b2) SharedBlob(0x558c5523ad20 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523ad38 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3b2 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f2b08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8d9276bf:::osdmap.1:0# key 0x7f7fffffffffffffff8d9276bf216f73'dmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1029 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~2c6 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd3a on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523ac40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523abd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523ac40 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523abd0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523ac40) get_ref 0x0~2c6 Blob(0x558c5523ac40 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523abd0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523ac40) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2c6: 0x0~2c6 Blob(0x558c5523ac40 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2c6) SharedBlob(0x558c5523abd0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523abe8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2c6 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f2888) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:6b02fa6d:::inc_osdmap.2:0# key 0x7f7fffffffffffffff6b02fa'm!inc_osdmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1030 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24c 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb4 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523aaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523aa80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523aaf0 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523aa80 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523aaf0) get_ref 0x0~24c Blob(0x558c5523aaf0 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523aa80 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523aaf0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24c: 0x0~24c Blob(0x558c5523aaf0 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24c) SharedBlob(0x558c5523aa80 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523aa98 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24c 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:829276bf:::osdmap.2:0# key 0x7f7fffffffffffffff829276bf216f73'dmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:829276bf:::osdmap.2:0# 0x0~2c6 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1031 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:829276bf:::osdmap.2:0# 0x0~2c6 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~2c6 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd3a on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x6000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x6000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523a620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523a5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523a620 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a5b0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a620) get_ref 0x0~2c6 Blob(0x558c5523a620 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a5b0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a620) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2c6: 0x0~2c6 Blob(0x558c5523a620 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2c6) SharedBlob(0x558c5523a5b0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523a5c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2c6 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3508) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:829276bf:::osdmap.2:0# 0x0~2c6 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:6602fa6d:::inc_osdmap.3:0# key 0x7f7fffffffffffffff6602fa'm!inc_osdmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1032 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x7000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x7000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523a690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523a4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523a690 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a4d0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a690) get_ref 0x0~108 Blob(0x558c5523a690 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a4d0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a690) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523a690 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523a4d0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523a4e8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3788) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:887276bf:::osdmap.3:0# key 0x7f7fffffffffffffff887276bf216f73'dmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:887276bf:::osdmap.3:0# 0x0~339 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1033 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:887276bf:::osdmap.3:0# 0x0~339 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~339 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xcc7 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x8000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x8000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523a700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523a7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523a700 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a7e0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a700) get_ref 0x0~339 Blob(0x558c5523a700 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a7e0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a700) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~339: 0x0~339 Blob(0x558c5523a700 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x339) SharedBlob(0x558c5523a7e0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523a7f8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x339 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3a08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:887276bf:::osdmap.3:0# 0x0~339 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:6c82fa6d:::inc_osdmap.4:0# key 0x7f7fffffffffffffff6c82fa'm!inc_osdmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1034 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x9000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x9000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523a8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523ad90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523a8c0 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523ad90 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a8c0) get_ref 0x0~108 Blob(0x558c5523a8c0 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523ad90 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a8c0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523a8c0 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523ad90 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523ada8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3c88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:85f276bf:::osdmap.4:0# key 0x7f7fffffffffffffff85f276bf216f73'dmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:85f276bf:::osdmap.4:0# 0x0~3ac 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1035 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:85f276bf:::osdmap.4:0# 0x0~3ac - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~3ac 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc54 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523ae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523aee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523ae70 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523aee0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523ae70) get_ref 0x0~3ac Blob(0x558c5523ae70 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523aee0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523ae70) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~3ac: 0x0~3ac Blob(0x558c5523ae70 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3ac) SharedBlob(0x558c5523aee0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523aef8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3ac 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258108) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:85f276bf:::osdmap.4:0# 0x0~3ac = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:6362fa6d:::inc_osdmap.5:0# key 0x7f7fffffffffffffff6362fa'm!inc_osdmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1036 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523afc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523afc0 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b030 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523afc0) get_ref 0x0~108 Blob(0x558c5523afc0 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b030 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523afc0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523afc0 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523b030 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b048 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258388) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8ef276bf:::osdmap.5:0# key 0x7f7fffffffffffffff8ef276bf216f73'dmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8ef276bf:::osdmap.5:0# 0x0~41f 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1037 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:8ef276bf:::osdmap.5:0# 0x0~41f - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~41f 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbe1 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523b110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523b110 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b180 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b110) get_ref 0x0~41f Blob(0x558c5523b110 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b180 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b110) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~41f: 0x0~41f Blob(0x558c5523b110 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x41f) SharedBlob(0x558c5523b180 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b198 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x41f 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258608) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8ef276bf:::osdmap.5:0# 0x0~41f = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:69e2fa6d:::inc_osdmap.6:0# key 0x7f7fffffffffffffff69e2fa'm!inc_osdmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1038 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xd000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xd000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523b260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523b260 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b2d0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b260) get_ref 0x0~108 Blob(0x558c5523b260 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b2d0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b260) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523b260 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523b2d0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b2e8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258888) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:803276bf:::osdmap.6:0# key 0x7f7fffffffffffffff803276bf216f73'dmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:803276bf:::osdmap.6:0# 0x0~492 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1039 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:803276bf:::osdmap.6:0# 0x0~492 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~492 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb6e on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xe000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xe000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523b3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523b3b0 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b420 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b3b0) get_ref 0x0~492 Blob(0x558c5523b3b0 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b420 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b3b0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~492: 0x0~492 Blob(0x558c5523b3b0 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x492) SharedBlob(0x558c5523b420 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b438 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x492 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258b08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:803276bf:::osdmap.6:0# 0x0~492 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:64e2fa6d:::inc_osdmap.7:0# key 0x7f7fffffffffffffff64e2fa'm!inc_osdmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1040 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xf000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xf000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523b500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523b500 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b570 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b500) get_ref 0x0~108 Blob(0x558c5523b500 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b570 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b500) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523b500 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523b570 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b588 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258d88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8bb276bf:::osdmap.7:0# key 0x7f7fffffffffffffff8bb276bf216f73'dmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8bb276bf:::osdmap.7:0# 0x0~505 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1041 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:8bb276bf:::osdmap.7:0# 0x0~505 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~505 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xafb on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x10000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x10000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523b650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523b650 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b6c0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b650) get_ref 0x0~505 Blob(0x558c5523b650 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b6c0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b650) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~505: 0x0~505 Blob(0x558c5523b650 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x505) SharedBlob(0x558c5523b6c0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b6d8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x505 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259008) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8bb276bf:::osdmap.7:0# 0x0~505 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:6f22fa6d:::inc_osdmap.8:0# key 0x7f7fffffffffffffff6f22fa'm!inc_osdmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1042 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x11000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x11000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523b7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523b7a0 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b810 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b7a0) get_ref 0x0~108 Blob(0x558c5523b7a0 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b810 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b7a0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523b7a0 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523b810 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b828 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:86b276bf:::osdmap.8:0# key 0x7f7fffffffffffffff86b276bf216f73'dmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:86b276bf:::osdmap.8:0# 0x0~578 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1043 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:86b276bf:::osdmap.8:0# 0x0~578 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~578 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xa88 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x12000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x12000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523b8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523b960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523b8f0 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b960 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b8f0) get_ref 0x0~578 Blob(0x558c5523b8f0 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523b960 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523b8f0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~578: 0x0~578 Blob(0x558c5523b8f0 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x578) SharedBlob(0x558c5523b960 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523b978 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x578 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259508) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:86b276bf:::osdmap.8:0# 0x0~578 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:61a2fa6d:::inc_osdmap.9:0# key 0x7f7fffffffffffffff61a2fa'm!inc_osdmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1044 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x13000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x13000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523bab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523ba40 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bab0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523ba40) get_ref 0x0~108 Blob(0x558c5523ba40 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bab0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523ba40) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523ba40 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523bab0 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523bac8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259788) dirty_range mark inline shard dirty 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 = 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:8c4276bf:::osdmap.9:0# key 0x7f7fffffffffffffff8c4276bf216f73'dmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.526+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8c4276bf:::osdmap.9:0# 0x0~5eb 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1045 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:8c4276bf:::osdmap.9:0# 0x0~5eb - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~5eb 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xa15 on front/back, now 0x0~1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x14000~1000/1000,1000,0 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x14000~1000] 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523bb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523bc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523bb90 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bc00 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523bb90) get_ref 0x0~5eb Blob(0x558c5523bb90 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bc00 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.blob(0x558c5523bb90) get_ref init 0x1000, 1000 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~5eb: 0x0~5eb Blob(0x558c5523bb90 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x5eb) SharedBlob(0x558c5523bc00 sbid 0x0)) 2019-12-07T19:33:38.526+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523bc18 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5eb 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259a08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:8c4276bf:::osdmap.9:0# 0x0~5eb = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c003972f:::inc_osdmap.10:0# key 0x7f7fffffffffffffffc00397'/!inc_osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c003972f:::inc_osdmap.10:0# 0x0~108 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1046 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c003972f:::inc_osdmap.10:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~108 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x15000~1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x15000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523bd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523bce0 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bd50 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c5523bce0) get_ref 0x0~108 Blob(0x558c5523bce0 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bd50 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c5523bce0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558c5523bce0 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558c5523bd50 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523bd68 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x108 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259c88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c003972f:::inc_osdmap.10:0# 0x0~108 = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3890e826:::osdmap.10:0# key 0x7f7fffffffffffffff3890e8'&!osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3890e826:::osdmap.10:0# 0x0~65e 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1047 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3890e826:::osdmap.10:0# 0x0~65e - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~65e 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x9a2 on front/back, now 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x16000~1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x16000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523be30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523bea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523be30 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bea0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c5523be30) get_ref 0x0~65e Blob(0x558c5523be30 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523bea0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c5523be30) get_ref init 0x1000, 1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~65e: 0x0~65e Blob(0x558c5523be30 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x65e) SharedBlob(0x558c5523bea0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5523beb8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x65e 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276108) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3890e826:::osdmap.10:0# 0x0~65e = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cb83972f:::inc_osdmap.11:0# key 0x7f7fffffffffffffffcb8397'/!inc_osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1048 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~33a 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xcc6 on front/back, now 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x17000~1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x17000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523bf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55278000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523bf80 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278000 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c5523bf80) get_ref 0x0~33a Blob(0x558c5523bf80 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278000 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c5523bf80) get_ref init 0x1000, 1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~33a: 0x0~33a Blob(0x558c5523bf80 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x33a) SharedBlob(0x558c55278000 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278018 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x33a 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276388) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3570e826:::osdmap.11:0# key 0x7f7fffffffffffffff3570e8'&!osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3570e826:::osdmap.11:0# 0x0~74c 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1049 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3570e826:::osdmap.11:0# 0x0~74c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~74c 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x8b4 on front/back, now 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x18000~1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x18000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552780e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55278150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552780e0 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278150 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c552780e0) get_ref 0x0~74c Blob(0x558c552780e0 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278150 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c552780e0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~74c: 0x0~74c Blob(0x558c552780e0 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x74c) SharedBlob(0x558c55278150 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278168 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x74c 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276608) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3570e826:::osdmap.11:0# 0x0~74c = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c683972f:::inc_osdmap.12:0# key 0x7f7fffffffffffffffc68397'/!inc_osdmap.12!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c683972f:::inc_osdmap.12:0# 0x0~496 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1050 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c683972f:::inc_osdmap.12:0# 0x0~496 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~496 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb6a on front/back, now 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x19000~1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x19000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55278230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552782a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55278230 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552782a0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c55278230) get_ref 0x0~496 Blob(0x558c55278230 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552782a0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c55278230) get_ref init 0x1000, 1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~496: 0x0~496 Blob(0x558c55278230 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x496) SharedBlob(0x558c552782a0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c552782b8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x496 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276888) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c683972f:::inc_osdmap.12:0# 0x0~496 = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3e70e826:::osdmap.12:0# key 0x7f7fffffffffffffff3e70e8'&!osdmap.12!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1051 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3e70e826:::osdmap.12:0# 0x0~8a8 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~8a8 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x758 on front/back, now 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1a000~1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1a000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55278380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552783f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55278380 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552783f0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c55278380) get_ref 0x0~8a8 Blob(0x558c55278380 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552783f0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c55278380) get_ref init 0x1000, 1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~8a8: 0x0~8a8 Blob(0x558c55278380 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x8a8) SharedBlob(0x558c552783f0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278408 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x8a8 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276b08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:46e858f0:::pg_num_history:head# key 0x7f7fffffffffffffff46e858f0217067'_num_history!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1052 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~12 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xfee on front/back, now 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1b000~1000/1000,1000,0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1b000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552784d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55278540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x12 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 0 blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6000 cost 129493 (26 ios * 4000 + 25493 bytes) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6000 onodes 0x558c53a91900,0x558c548f2500,0x558c548f2780,0x558c548f2a00,0x558c548f3180,0x558c548f3400,0x558c548f3680,0x558c548f3900,0x558c548f3b80,0x558c55258000,0x558c55258280,0x558c55258500,0x558c55258780,0x558c55258a00,0x558c55258c80,0x558c55258f00,0x558c55259180,0x558c55259400,0x558c55259680,0x558c55259900,0x558c55259b80,0x558c55276000,0x558c55276280,0x558c55276500,0x558c55276780,0x558c55276a00,0x558c55276c80 shared_blobs 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f2608) update #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c1a3fc6e:::purged_snaps:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f2888) update #-1:8d9276bf:::osdmap.1:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f2888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:8d9276bf:::osdmap.1:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f2b08) update #-1:60c2fa6d:::inc_osdmap.1:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f2b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:60c2fa6d:::inc_osdmap.1:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3288) update #-1:6b02fa6d:::inc_osdmap.2:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:6b02fa6d:::inc_osdmap.2:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3508) update #-1:829276bf:::osdmap.2:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:829276bf:::osdmap.2:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3788) update #-1:6602fa6d:::inc_osdmap.3:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3788) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:6602fa6d:::inc_osdmap.3:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3a08) update #-1:887276bf:::osdmap.3:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:887276bf:::osdmap.3:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3c88) update #-1:6c82fa6d:::inc_osdmap.4:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c548f3c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:6c82fa6d:::inc_osdmap.4:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258108) update #-1:85f276bf:::osdmap.4:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:85f276bf:::osdmap.4:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258388) update #-1:6362fa6d:::inc_osdmap.5:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258388) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:6362fa6d:::inc_osdmap.5:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258608) update #-1:8ef276bf:::osdmap.5:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:8ef276bf:::osdmap.5:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258888) update #-1:69e2fa6d:::inc_osdmap.6:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:69e2fa6d:::inc_osdmap.6:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258b08) update #-1:803276bf:::osdmap.6:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:803276bf:::osdmap.6:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258d88) update #-1:64e2fa6d:::inc_osdmap.7:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55258d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:64e2fa6d:::inc_osdmap.7:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259008) update #-1:8bb276bf:::osdmap.7:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:8bb276bf:::osdmap.7:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259288) update #-1:6f22fa6d:::inc_osdmap.8:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:6f22fa6d:::inc_osdmap.8:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259508) update #-1:86b276bf:::osdmap.8:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:86b276bf:::osdmap.8:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259788) update #-1:61a2fa6d:::inc_osdmap.9:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259788) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:61a2fa6d:::inc_osdmap.9:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259a08) update #-1:8c4276bf:::osdmap.9:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:8c4276bf:::osdmap.9:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259c88) update #-1:c003972f:::inc_osdmap.10:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55259c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c003972f:::inc_osdmap.10:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276108) update #-1:3890e826:::osdmap.10:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3890e826:::osdmap.10:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276388) update #-1:cb83972f:::inc_osdmap.11:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276388) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cb83972f:::inc_osdmap.11:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276608) update #-1:3570e826:::osdmap.11:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3570e826:::osdmap.11:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276888) update #-1:c683972f:::inc_osdmap.12:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c683972f:::inc_osdmap.12:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276b08) update #-1:3e70e826:::osdmap.12:0# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3e70e826:::osdmap.12:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6000 allocated 0x[3000~19000] released 0x[] 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 freelist allocate 0x3000~19000 2019-12-07T19:33:38.527+0000 7f8f164bb700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 prepare 2019-12-07T19:33:38.527+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6000 2019-12-07T19:33:38.527+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 aio_wait 2019-12-07T19:33:38.527+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6000 2019-12-07T19:33:38.527+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 io_done 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:38.527+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6836 < min_flush_size 524288 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2780 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2a00 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f3180 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f3400 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f3680 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f3900 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f3b80 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55258000 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55258280 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55258500 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55258780 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55258a00 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55258c80 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55258f00 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55259180 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55259400 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55259680 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55259900 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55259b80 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276000 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276280 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276500 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276780 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276a00 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:38.527+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6859 < min_flush_size 524288 2019-12-07T19:33:38.527+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6859 < min_flush_size 524288 2019-12-07T19:33:38.527+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.527+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x22e2~1acb to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.527+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x22e2 0x22e2~1acb to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x22e2 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2e2 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:38.527+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdad and padding block with 0x253 2019-12-07T19:33:38.528+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3dad 2019-12-07T19:33:38.528+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:38.528+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:38.528+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:38.528+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:38.528+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000175584s (0.000043224s flush + 0.000132360s kv commit) 2019-12-07T19:33:38.528+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6000> 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:38.528+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 kv_submitted 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6000 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f6000 osr 0x558c54ea4b40 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) _discard drop 2 0x2000~1000 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) prepare_write seq 3 0x2000~1000 crc e042d987 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 0 _committed_osd_maps 1..12 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 0 advance to epoch 1 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 1 advance to epoch 2 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 2 advance to epoch 3 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 3 advance to epoch 4 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 4 advance to epoch 5 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 5 advance to epoch 6 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 6 advance to epoch 7 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 7 advance to epoch 8 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 8 advance to epoch 9 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 9 advance to epoch 10 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 10 advance to epoch 11 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 11 advance to epoch 12 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.528+0000 7f8f141a1700 0 osd.7 12 crush map has features 288514050185494528, adjusting msgr requires for clients 2019-12-07T19:33:38.528+0000 7f8f141a1700 0 osd.7 12 crush map has features 288514050185494528 was 288232575208792577, adjusting msgr requires for mons 2019-12-07T19:33:38.528+0000 7f8f141a1700 0 osd.7 12 crush map has features 3314932999778484224, adjusting msgr requires for osds 2019-12-07T19:33:38.528+0000 7f8f141a1700 1 osd.7 12 check_osdmap_features require_osd_release unknown -> octopus 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label got bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 7 meta) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 bluestore(/var/lib/ceph/osd/ceph-7/block) _write_bdev_label path /var/lib/ceph/osd/ceph-7/block label bdev(osd_uuid 0f8b5d06-9920-4212-962e-b5b150d77eaa, size 0x1680000000, btime 2019-12-07T19:33:20.818923+0000, desc main, 8 meta) 2019-12-07T19:33:38.528+0000 7f8f141a1700 7 osd.7 12 consume_map version 12 2019-12-07T19:33:38.528+0000 7f8f141a1700 20 osd.7 12 prune_pg_created 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:0.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:1.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:2.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:3.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:4.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:5.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:6.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7:7.consume_map 12 (was 0) 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 12 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:38.528+0000 7f8f141a1700 10 osd.7 12 _preboot _preboot mon has osdmaps 1..12 2019-12-07T19:33:38.528+0000 7f8f141a1700 5 osd.7 12 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:38.528+0000 7f8f141a1700 20 osd.7 12 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 10 osd.7 12 operator() waiting for peering work to drain 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 10 osd.7 12 _send_boot 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 20 osd.7 12 initial client_addrs [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939], cluster_addrs [v2:0.0.0.0:6826/36939,v1:0.0.0.0:6827/36939], hb_back_addrs [v2:0.0.0.0:6830/36939,v1:0.0.0.0:6831/36939], hb_front_addrs [v2:0.0.0.0:6828/36939,v1:0.0.0.0:6829/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6826/36939,v1:0.0.0.0:6827/36939] set_addr_unknowns [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6826/36939,v1:0.0.0.0:6827/36939] set_addr_unknowns assuming my addr v2:0.0.0.0:6826/36939 matches provided addr v2:172.21.15.109:6824/36939 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6826/36939,v1:0.0.0.0:6827/36939] set_addr_unknowns assuming my addr v1:0.0.0.0:6827/36939 matches provided addr v2:172.21.15.109:6824/36939 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 10 osd.7 12 new session (outgoing) 0x558c53a856c0 con=0x558c53ae5400 addr=v2:172.21.15.109:6826/36939 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] set_addr_unknowns now [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 10 osd.7 12 assuming cluster_addrs match client_addrs [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6830/36939,v1:0.0.0.0:6831/36939] set_addr_unknowns [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6830/36939,v1:0.0.0.0:6831/36939] set_addr_unknowns assuming my addr v2:0.0.0.0:6830/36939 matches provided addr v2:172.21.15.109:6826/36939 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6830/36939,v1:0.0.0.0:6831/36939] set_addr_unknowns assuming my addr v1:0.0.0.0:6831/36939 matches provided addr v2:172.21.15.109:6826/36939 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] set_addr_unknowns now [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 10 osd.7 12 assuming hb_back_addrs match cluster_addrs [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6828/36939,v1:0.0.0.0:6829/36939] set_addr_unknowns [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6828/36939,v1:0.0.0.0:6829/36939] set_addr_unknowns assuming my addr v2:0.0.0.0:6828/36939 matches provided addr v2:172.21.15.109:6824/36939 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:0.0.0.0:6828/36939,v1:0.0.0.0:6829/36939] set_addr_unknowns assuming my addr v1:0.0.0.0:6829/36939 matches provided addr v2:172.21.15.109:6824/36939 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] set_addr_unknowns now [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 10 osd.7 12 assuming hb_front_addrs match client_addrs [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 osd.7 12 set_numa_affinity storage numa node 0 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 osd.7 12 set_numa_affinity public network enp3s0f1 numa node 0 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 osd.7 12 set_numa_affinity cluster network enp3s0f1 numa node 0 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 osd.7 12 objectstore and network numa nodes all match 2019-12-07T19:33:38.528+0000 7f8f1e3bb700 1 osd.7 12 set_numa_affinity setting numa affinity to node 0 cpus 0-7 2019-12-07T19:33:38.529+0000 7f8f1e3bb700 10 osd.7 12 final client_addrs [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939], cluster_addrs [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939], hb_back_addrs [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939], hb_front_addrs [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] 2019-12-07T19:33:38.529+0000 7f8f1e3bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) collect_metadata 2019-12-07T19:33:38.529+0000 7f8f1e3bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.529+0000 7f8f1e3bb700 1 bluestore(/var/lib/ceph/osd/ceph-7) collect_metadata devices span numa nodes 0 2019-12-07T19:33:38.529+0000 7f8f1e3bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) collect_metadata 2019-12-07T19:33:38.529+0000 7f8f1e3bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.529+0000 7f8f1e3bb700 1 bluestore(/var/lib/ceph/osd/ceph-7) collect_metadata devices span numa nodes 0 2019-12-07T19:33:38.530+0000 7f8f1e3bb700 10 osd.7 12 _collect_metadata {arch=x86_64,back_addr=[v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939],back_iface=enp3s0f1,bluefs=1,bluefs_dedicated_db=0,bluefs_dedicated_wal=0,bluefs_single_shared_device=1,bluestore_bdev_access_mode=file,bluestore_bdev_block_size=4096,bluestore_bdev_devices=nvme0n1,bluestore_bdev_driver=KernelDevice,bluestore_bdev_path=/var/lib/ceph/osd/ceph-7/block,bluestore_bdev_rotational=0,bluestore_bdev_size=96636764160,bluestore_bdev_support_discard=1,bluestore_bdev_type=ssd,ceph_release=octopus,ceph_version=ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev),ceph_version_short=15.0.0-8038-g503c226,cpu=Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz,default_device_class=ssd,device_ids=nvme0n1=INTEL_SSDPEDMD400G4_PHFT6204016X400BGN,devices=nvme0n1,distro=rhel,distro_description=Employee SKU,distro_version=7.7,front_addr=[v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939],front_iface=enp3s0f1,hb_back_addr=[v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939],hb_front_addr=[v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939],hostname=smithi109,journal_rotational=0,kernel_description=#1 SMP Wed Sep 25 09:42:57 EDT 2019,kernel_version=3.10.0-1062.4.1.el7.x86_64,mem_swap_kb=0,mem_total_kb=32664132,network_numa_node=0,network_numa_nodes=0,numa_node=0,numa_node_cpus=0-7,objectstore_numa_node=0,objectstore_numa_nodes=0,os=Linux,osd_data=/var/lib/ceph/osd/ceph-7,osd_objectstore=bluestore,rotational=0} 2019-12-07T19:33:38.530+0000 7f8f1e3bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_boot(osd.7 booted 0 features 4540138292836696063 v12) v7 -- 0x558c5489e700 con 0x558c5476d000 2019-12-07T19:33:38.534+0000 7f8f1fbbe700 10 osd.7 12 tick_without_osd_lock 2019-12-07T19:33:38.534+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffe4000/0x40000000/0x1680000000, data 0x544a/0x1a000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:38.568+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.618+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.623+0000 7f8f21451700 10 osd.7 12 tick 2019-12-07T19:33:38.623+0000 7f8f21451700 1 osd.7 12 tick checking mon for new map 2019-12-07T19:33:38.623+0000 7f8f21451700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=13}) v3 -- 0x558c551af800 con 0x558c5476d000 2019-12-07T19:33:38.623+0000 7f8f21451700 10 osd.7 12 do_waiters -- start 2019-12-07T19:33:38.623+0000 7f8f21451700 10 osd.7 12 do_waiters -- finish 2019-12-07T19:33:38.623+0000 7f8f21451700 20 osd.7 12 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:38.669+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.719+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.769+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.819+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.869+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.919+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.969+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.020+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.070+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.120+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.170+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.220+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.270+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.320+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.371+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.421+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.471+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.516+0000 7f8f1fbbe700 10 osd.7 12 tick_without_osd_lock 2019-12-07T19:33:39.517+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffe4000/0x40000000/0x1680000000, data 0x544a/0x1a000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:39.519+0000 7f8f1219d700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.519+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.7) v3 -- 0x558c550382c0 con 0x558c5476d800 2019-12-07T19:33:39.519+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.519+0000 7f8f164bb700 10 osd.7 12 ms_handle_connect con 0x558c5476d800 2019-12-07T19:33:39.519+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 crc :-1 s=READY pgs=31 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.520+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 msgr2=0x558c53aa7180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:33:39.520+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mgr.4099 v2:172.21.15.131:6800/36137 1 ==== mgrconfigure(period=5, threshold=5) v3 ==== 12+0+0 (crc 0 0 0) 0x558c53aa8400 con 0x558c5476d800 2019-12-07T19:33:39.520+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 msgr2=0x558c53aa7180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:39.520+0000 7f8f164bb700 10 osd.7 12 setting 0 queries 2019-12-07T19:33:39.520+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 crc :-1 s=READY pgs=31 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:39.520+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 crc :-1 s=READY pgs=31 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.520+0000 7f8f164bb700 20 osd.7 12 reports for 0 queries 2019-12-07T19:33:39.520+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +109-0 packed 1198 daemon_metrics=2) v8 -- 0x558c5489e380 con 0x558c5476d800 2019-12-07T19:33:39.520+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558c5489e380 con 0x558c5476d800 2019-12-07T19:33:39.520+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 msgr2=0x558c53aa7180 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:33:39.520+0000 7f8f164bb700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.521+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415823 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 10 ==== osd_map(13..13 src has 1..13) v4 ==== 3272+0+0 (crc 0 0 0) 0x558c55274900 con 0x558c5476d000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 osd.7 12 OSD::ms_dispatch: osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 osd.7 12 do_waiters -- start 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 osd.7 12 do_waiters -- finish 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 osd.7 12 _dispatch 0x558c55274900 osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.521+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558c53aa8400 con 0x558c5476d000 2019-12-07T19:33:39.521+0000 7f8f164bb700 3 osd.7 12 handle_osd_map epochs [13,13], i have 12, src has [1,13] 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 osd.7 12 handle_osd_map got inc map for epoch 13 2019-12-07T19:33:39.521+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3e70e826:::osdmap.12:0# 0x0~0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~8a8 size 0x8a8 (2216) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278380 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x8a8) SharedBlob(0x558c552783f0 sbid 0x0)) need 0x0~8a8 cache has 0x[0~8a8] 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 = 2216 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 osd.7 12 add_map_bl 12 2216 bytes 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 osd.7 12 got_full_map 13, nothing requested 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 osd.7 12 handle_osd_map pg_num_history pg_num_history(e13 pg_nums {} deleted_pools ) 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {13={}} 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 osd.7 12 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e13 [1,13] lci=[0,0]) 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f6300 seq 6 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cc63972f:::inc_osdmap.13:0# key 0x7f7fffffffffffffffcc6397'/!inc_osdmap.13!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1053 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~c9c 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x364 on front/back, now 0x0~1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 1 blobs 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1c000~1000/1000,1000,0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1c000~1000] 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c546773b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c546773b0 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677570 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c546773b0) get_ref 0x0~c9c Blob(0x558c546773b0 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677570 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c546773b0) get_ref init 0x1000, 1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~c9c: 0x0~c9c Blob(0x558c546773b0 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0xc9c) SharedBlob(0x558c54677570 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54677588 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xc9c 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55277008) dirty_range mark inline shard dirty 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c = 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:30f0e826:::osdmap.13:0# key 0x7f7fffffffffffffff30f0e8'&!osdmap.13!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1054 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:30f0e826:::osdmap.13:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~168 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 2 blobs 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1d000~2000/1000,2000,0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1d000~2000] 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c546778f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c546778f0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677ab0 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c546778f0) get_ref 0x0~1000 Blob(0x558c546778f0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677ab0 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c546778f0) get_ref init 0x1000, 1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c546778f0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54677ab0 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54677ac8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54677c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54677c70 blob([!~1000,0x1e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677e30 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c54677c70) get_ref 0x1000~168 Blob(0x558c54677c70 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c54677e30 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c54677c70) get_ref init 0x2000, 1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x558c54677c70 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558c54677e30 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54677e48 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1168 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55277288) dirty_range mark inline shard dirty 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 = 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~12 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 0 blobs 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:39.521+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.521+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:39.521+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 0 blobs 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:39.522+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:39.522+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 25145 (4 ios * 4000 + 9145 bytes) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes 0x558c53a91900,0x558c55276c80,0x558c55276f00,0x558c55277180 shared_blobs 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55277008) update #-1:cc63972f:::inc_osdmap.13:0# 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55277008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cc63972f:::inc_osdmap.13:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55277288) update #-1:30f0e826:::osdmap.13:0# 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55277288) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:30f0e826:::osdmap.13:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[1c000~3000] released 0x[] 2019-12-07T19:33:39.522+0000 7f8f164bb700 10 freelist allocate 0x1c000~3000 2019-12-07T19:33:39.522+0000 7f8f164bb700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:39.522+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:33:39.522+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6300 2019-12-07T19:33:39.523+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c53ae4c00 0x558c53aa6680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.523+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c53ae4c00 0x558c53aa6680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.523+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 aio_wait 2019-12-07T19:33:39.523+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:33:39.523+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8788 < min_flush_size 524288 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276f00 had 1 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55277180 had 1 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8811 < min_flush_size 524288 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8811 < min_flush_size 524288 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3dad~226b to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3dad 0x3dad~226b to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x3dad 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdad 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x18 and padding block with 0xfe8 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6018 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:39.523+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000250550s (0.000055924s flush + 0.000194626s kv commit) 2019-12-07T19:33:39.523+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300> 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:39.523+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f6300 osr 0x558c54ea4b40 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7 12 _committed_osd_maps 13..13 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) prepare_write seq 4 0x1b000~1000 crc 2ba63da0 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7 12 advance to epoch 13 (<= last 13 <= newest_map 13) 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) _discard drop 3 0x2000~1000 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54ea4640) prepare_write seq 4 0x2000~1000 crc 4c53cca6 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7 13 up_epoch is 13 2019-12-07T19:33:39.523+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7 13 boot_epoch is 13 2019-12-07T19:33:39.523+0000 7f8f141a1700 1 osd.7 13 state: booting -> active 2019-12-07T19:33:39.523+0000 7f8f141a1700 7 osd.7 13 consume_map version 13 2019-12-07T19:33:39.523+0000 7f8f141a1700 20 osd.7 13 prune_pg_created 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:0.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:1.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:2.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:3.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:4.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:5.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:6.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7:7.consume_map 13 (was 12) 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7 13 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:39.523+0000 7f8f141a1700 10 osd.7 13 maybe_update_heartbeat_peers updating 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 adding neighbor peer osd.0 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c5476d400 0x558c53aa6c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 0x558c53aa4580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 _add_heartbeat_peer: new peer osd.0 v2:172.21.15.131:6830/37079 v2:172.21.15.131:6826/37079 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 adding neighbor peer osd.1 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c551b6400 0x558c53aa5b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 0x558c552b2000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 _add_heartbeat_peer: new peer osd.1 v2:172.21.15.131:6831/37078 v2:172.21.15.131:6827/37078 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 adding neighbor peer osd.6 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c551b7400 0x558c552b2b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 _add_heartbeat_peer: new peer osd.6 v2:172.21.15.109:6822/36937 v2:172.21.15.109:6820/36937 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 adding random peer osd.2 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x558c551b7800 0x558c552b3080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 11 ==== osd_map(13..13 src has 1..13) v4 ==== 4500+0+0 (crc 0 0 0) 0x558c55274b40 con 0x558c5476d000 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x558c551b7c00 0x558c552b3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f164bb700 20 osd.7 13 OSD::ms_dispatch: osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 _add_heartbeat_peer: new peer osd.2 v2:172.21.15.131:6808/37077 v2:172.21.15.131:6806/37077 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 adding random peer osd.3 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x558c552c2000 0x558c552b3b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 0x558c552b4100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 _add_heartbeat_peer: new peer osd.3 v2:172.21.15.131:6816/37076 v2:172.21.15.131:6814/37076 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 adding random peer osd.4 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x558c552c2800 0x558c552b4680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x558c552c2c00 0x558c552b4c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 _add_heartbeat_peer: new peer osd.4 v2:172.21.15.109:6814/36938 v2:172.21.15.109:6812/36938 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 adding random peer osd.5 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 0x558c552b5180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 1 --2- >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c552c3400 0x558c552b5700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 _add_heartbeat_peer: new peer osd.5 v2:172.21.15.109:6806/36936 v2:172.21.15.109:6804/36936 2019-12-07T19:33:39.524+0000 7f8f141a1700 10 osd.7 13 maybe_update_heartbeat_peers 7 peers, extras 0,1,2,3,4,5,6 2019-12-07T19:33:39.524+0000 7f8f141a1700 7 osd.7 13 activate_map version 13 2019-12-07T19:33:39.524+0000 7f8f0918b700 20 osd.7 13 agent_entry empty queue 2019-12-07T19:33:39.524+0000 7f8f164bb700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:39.524+0000 7f8f164bb700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:39.524+0000 7f8f164bb700 20 osd.7 13 _dispatch 0x558c55274b40 osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.524+0000 7f8f164bb700 3 osd.7 13 handle_osd_map epochs [13,13], i have 13, src has [1,13] 2019-12-07T19:33:39.524+0000 7f8f164bb700 10 osd.7 13 no new maps here, dropping 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c5476d400 0x558c53aa6c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 0x558c53aa4580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d0000 0x558c552d6000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d0400 0x558c552d6580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c552d0800 0x558c552d6b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c552d0c00 0x558c552d7080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d0400 0x558c552d6580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c552d0800 0x558c552d6b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d0000 0x558c552d6000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 0x558c552b5180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 0x558c552b5180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.109:6806/36936 says I am v2:172.21.15.109:47098/0 (socket says 172.21.15.109:47098) 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 learned_addr learned my addr 172.21.15.109:0/36939 (peer_addr_for_me v2:172.21.15.109:0/0) 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c552d0c00 0x558c552d7080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c552bd860 con 0x558c552d0400 entity osd.6 addr 2019-12-07T19:33:39.533+0000 7f8f24457700 1 --2- >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x558c552c2c00 0x558c552b4c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c552bd860 osd.6 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x558c552c2800 0x558c552b4680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f24457700 1 --2- >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x558c552c2c00 0x558c552b4c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.109:6812/36938 says I am v2:172.21.15.109:33506/0 (socket says 172.21.15.109:33506) 2019-12-07T19:33:39.533+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 learned_addr learned my addr 172.21.15.109:0/36939 (peer_addr_for_me v2:172.21.15.109:0/0) 2019-12-07T19:33:39.533+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication new session 0x558c552bda00 con 0x558c552d0000 entity osd.5 addr 2019-12-07T19:33:39.533+0000 7f8f24c58700 10 osd.7 13 ms_handle_authentication new session 0x558c552bdd40 con 0x558c552d0c00 entity osd.4 addr 2019-12-07T19:33:39.533+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication session 0x558c552bda00 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.533+0000 7f8f24c58700 10 osd.7 13 ms_handle_authentication session 0x558c552bdd40 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c552c3400 0x558c552b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36937 conn(0x558c552d0400 0x558c552d6580 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0a000 con 0x558c552d0800 entity osd.5 addr 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d1000 0x558c552d7600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.533+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0a000 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.533+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d1000 0x558c552d7600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 0x558c552b5180 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.533+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0000 0x558c552d6000 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.533+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0800 0x558c552d6b00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f8f24c58700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0a340 con 0x558c552d1000 entity osd.4 addr 2019-12-07T19:33:39.534+0000 7f8f24c58700 10 osd.7 13 ms_handle_authentication session 0x558c54c0a340 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c552d0c00 0x558c552d7080 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c552c3400 0x558c552b5700 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x558c552c2800 0x558c552b4680 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c552d1000 0x558c552d7600 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d1800 0x558c552d7b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d1400 0x558c552d8100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c552d1c00 0x558c552d8680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x558c552c2c00 0x558c552b4c00 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c18000 0x558c552d8c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication new session 0x558c552bdba0 con 0x558c53ae4c00 entity osd.6 addr 2019-12-07T19:33:39.534+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication session 0x558c552bdba0 osd.6 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.534+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c551b7400 0x558c552b2b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 0x558c552b4100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f23c56700 0 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 msgr2=0x558c552b4100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2019-12-07T19:33:39.535+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 0x558c552b2580 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d1400 0x558c552d8100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 msgr2=0x558c552b4100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send send error: (32) Broken pipe 2019-12-07T19:33:39.535+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c552d1800 0x558c552d7b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 0x558c552b4100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).write auth request write failed r=-32 ((32) Broken pipe) 2019-12-07T19:33:39.535+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 0x558c552b4100 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:33:39.535+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c552d1c00 0x558c552d8680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x558c551b7c00 0x558c552b3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c18000 0x558c552d8c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x558c551b7800 0x558c552b3080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x558c552c2000 0x558c552b3b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36937 conn(0x558c53ae4c00 0x558c53aa6680 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f8f24c58700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0a4e0 con 0x558c552d1800 entity osd.2 addr 2019-12-07T19:33:39.535+0000 7f8f24c58700 10 osd.7 13 ms_handle_authentication session 0x558c54c0a4e0 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0a680 con 0x558c552d1400 entity osd.3 addr 2019-12-07T19:33:39.535+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication session 0x558c54c0a680 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0a1a0 con 0x558c54c18000 entity osd.3 addr 2019-12-07T19:33:39.535+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0a1a0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x558c552c2000 0x558c552b3b80 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0a820 con 0x558c552d1c00 entity osd.2 addr 2019-12-07T19:33:39.535+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication session 0x558c54c0a820 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37077 conn(0x558c552d1800 0x558c552d7b80 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x558c551b7800 0x558c552b3080 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c551b7400 0x558c552b2b00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37076 conn(0x558c552d1400 0x558c552d8100 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c18400 0x558c552d9180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c18800 0x558c552d9700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.536+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37076 conn(0x558c54c18000 0x558c552d8c00 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37077 conn(0x558c552d1c00 0x558c552d8680 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c18c00 0x558c54c36000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.536+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x558c551b7c00 0x558c552b3600 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c18c00 0x558c54c36000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c18400 0x558c552d9180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c19000 0x558c54c36580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.536+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0ab60 con 0x558c54c18c00 entity osd.0 addr 2019-12-07T19:33:39.536+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0ab60 osd.0 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c551b6400 0x558c53aa5b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 0x558c552b2000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c19000 0x558c54c36580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c18800 0x558c552d9700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 0x558c53aa4580 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18c00 0x558c54c36000 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0ad00 con 0x558c54c19000 entity osd.1 addr 2019-12-07T19:33:39.537+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0ad00 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c551b6400 0x558c53aa5b80 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c5476d400 0x558c53aa6c00 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 0x558c552b2000 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37078 conn(0x558c54c19000 0x558c54c36580 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0aea0 con 0x558c54c18800 entity osd.1 addr 2019-12-07T19:33:39.537+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0aea0 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37078 conn(0x558c54c18800 0x558c552d9700 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0b040 con 0x558c54c18400 entity osd.0 addr 2019-12-07T19:33:39.538+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0b040 osd.0 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.538+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18400 0x558c552d9180 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.571+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:33:39.571+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:33:39.571+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 4 0x2000~1000 2019-12-07T19:33:39.571+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 4c53cca6 2019-12-07T19:33:39.571+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 4 0x1b000~1000 2019-12-07T19:33:39.571+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 2ba63da0 2019-12-07T19:33:39.571+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.571+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:33:39.571+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:33:39.574+0000 7f8f21451700 10 osd.7 13 tick 2019-12-07T19:33:39.574+0000 7f8f21451700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:39.574+0000 7f8f21451700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:39.574+0000 7f8f21451700 20 osd.7 13 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:39.621+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.671+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.722+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.736+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 0x558c552b4100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.736+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558c552c2400 0x558c552b4100 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.772+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.822+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.872+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.922+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.972+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.022+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.072+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.123+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.173+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.223+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.273+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.323+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.373+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.423+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.474+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.519+0000 7f8f1219d700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:40.519+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.7) v3 -- 0x558c55038580 con 0x558c5476d800 2019-12-07T19:33:40.519+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:40.520+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558c5476d800 0x558c53aa7180 crc :-1 s=READY pgs=39 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:40.520+0000 7f8f164bb700 10 osd.7 13 ms_handle_connect con 0x558c5476d800 2019-12-07T19:33:40.520+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mgr.4099 v2:172.21.15.131:6800/36137 1 ==== mgrconfigure(period=5, threshold=5) v3 ==== 12+0+0 (crc 0 0 0) 0x558c54c1e800 con 0x558c5476d800 2019-12-07T19:33:40.520+0000 7f8f164bb700 10 osd.7 13 setting 0 queries 2019-12-07T19:33:40.520+0000 7f8f164bb700 20 osd.7 13 reports for 0 queries 2019-12-07T19:33:40.520+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +109-0 packed 1198) v8 -- 0x558c5489e380 con 0x558c5476d800 2019-12-07T19:33:40.520+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558c5489e700 con 0x558c5476d800 2019-12-07T19:33:40.524+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.541+0000 7f8f21451700 10 osd.7 13 tick 2019-12-07T19:33:40.541+0000 7f8f21451700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:40.541+0000 7f8f21451700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:40.541+0000 7f8f21451700 20 osd.7 13 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 10 osd.7 13 tick_without_osd_lock 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffe1000/0x40000000/0x1680000000, data 0x724e/0x1d000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 25 osd.7 13 heartbeat_check we haven't sent ping to osd.0 yet, skipping 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 25 osd.7 13 heartbeat_check we haven't sent ping to osd.1 yet, skipping 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 25 osd.7 13 heartbeat_check we haven't sent ping to osd.2 yet, skipping 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 25 osd.7 13 heartbeat_check we haven't sent ping to osd.3 yet, skipping 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 25 osd.7 13 heartbeat_check we haven't sent ping to osd.4 yet, skipping 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 25 osd.7 13 heartbeat_check we haven't sent ping to osd.5 yet, skipping 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 25 osd.7 13 heartbeat_check we haven't sent ping to osd.6 yet, skipping 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 20 osd.7 13 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 10 osd.7 13 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 20 osd.7 13 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 10 osd.7 13 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 20 osd.7 13 send_beacon sending 2019-12-07T19:33:40.555+0000 7f8f1fbbe700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_beacon(pgs [] lec 13 last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 v13) v2 -- 0x558c54c1f800 con 0x558c5476d000 2019-12-07T19:33:40.574+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.624+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.674+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.724+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.774+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.824+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.875+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.925+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.975+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.025+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.075+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.125+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.175+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.225+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.235+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.707221878s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1e600 con 0x558c552d1c00 2019-12-07T19:33:41.235+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.035848334s,]) 2019-12-07T19:33:41.235+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.671373544s delta_ub 18446744073.673703282s) v5 -- 0x558c54c1fa00 con 0x558c552d1c00 2019-12-07T19:33:41.235+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.236+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.707221878s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c551afe00 con 0x558c552d1800 2019-12-07T19:33:41.236+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.035478404s,]) 2019-12-07T19:33:41.236+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.671743474s delta_ub 18446744073.674073212s) v5 -- 0x558c54c58000 con 0x558c552d1800 2019-12-07T19:33:41.236+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.236+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.236+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.237+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0a9c0 con 0x558c54c19400 entity osd.2 addr 2019-12-07T19:33:41.237+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication session 0x558c54c0a9c0 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.237+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c19400 msgr2=0x558c54c36b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 91 2019-12-07T19:33:41.237+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c19400 msgr2=0x558c54c36b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.237+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c19400 0x558c54c36b00 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.237+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c19400 0x558c54c36b00 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.237+0000 7f8f1199c700 2 osd.7 13 ms_handle_reset con 0x558c54c19400 session 0x558c54c0a9c0 2019-12-07T19:33:41.237+0000 7f8f1199c700 10 osd.7 13 ms_handle_connect con 0x558c54c19800 2019-12-07T19:33:41.237+0000 7f8f24c58700 10 osd.7 13 new session (outgoing) 0x558c54c0b6c0 con=0x558c54c19800 addr=v2:172.21.15.131:6804/37077 2019-12-07T19:33:41.237+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 crc :-1 s=READY pgs=11 cs=0 l=0 rx=0 tx=0).ready entity=osd.2 client_cookie=a3b801957c02aad7 server_cookie=22a5b791d0d72b65 in_seq=0 out_seq=0 2019-12-07T19:33:41.240+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.706869389s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c552d0800 2019-12-07T19:33:41.240+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.706869389s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d0000 2019-12-07T19:33:41.240+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030575067s,]) 2019-12-07T19:33:41.240+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.676294322s delta_ub 18446744073.678976549s) v5 -- 0x558c54c1e600 con 0x558c552d0000 2019-12-07T19:33:41.240+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.240+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030488008s,]) 2019-12-07T19:33:41.240+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.676381381s delta_ub 18446744073.679063608s) v5 -- 0x558c54c1f800 con 0x558c552d0800 2019-12-07T19:33:41.241+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.241+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a000 0x558c54c37080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.241+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a000 0x558c54c37080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.241+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 crc :-1 s=START_CONNECT pgs=0 cs=0 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:41.241+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0bba0 con 0x558c54c5a000 entity osd.5 addr 2019-12-07T19:33:41.241+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication session 0x558c54c0bba0 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.242+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c5a000 0x558c54c37080 crc :-1 s=SESSION_ACCEPTING pgs=3 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558c54c19c00 socket by this connection's socket 2019-12-07T19:33:41.242+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c5a000 0x558c54c37080 crc :-1 s=SESSION_ACCEPTING pgs=3 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.242+0000 7f8f1199c700 2 osd.7 13 ms_handle_reset con 0x558c54c5a000 session 0x558c54c0bba0 2019-12-07T19:33:41.242+0000 7f8f24457700 10 osd.7 13 new session (incoming)0x558c54c0bd40 con=0x558c54c19c00 addr=v2:172.21.15.109:6802/36936 must have raced with connect 2019-12-07T19:33:41.242+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=READY pgs=3 cs=0 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=e020c6588efc571e server_cookie=4bffdf0ee71c7be4 in_seq=0 out_seq=0 2019-12-07T19:33:41.250+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.707537742s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c551aee00 con 0x558c54c18000 2019-12-07T19:33:41.250+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.707537742s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d1400 2019-12-07T19:33:41.250+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021694968s,]) 2019-12-07T19:33:41.250+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.685842774s delta_ub 18446744073.687856648s) v5 -- 0x558c54c1f800 con 0x558c54c18000 2019-12-07T19:33:41.250+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.250+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021572472s,]) 2019-12-07T19:33:41.250+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.685965270s delta_ub 18446744073.687979144s) v5 -- 0x558c54c1fa00 con 0x558c552d1400 2019-12-07T19:33:41.250+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.706948734s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c54c19000 2019-12-07T19:33:41.250+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.020648098s,]) 2019-12-07T19:33:41.250+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.686300636s delta_ub 18446744073.688903518s) v5 -- 0x558c54c1f800 con 0x558c54c19000 2019-12-07T19:33:41.250+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558c54c64000 0x558c54c39700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.251+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.706948734s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c551aee00 con 0x558c54c18800 2019-12-07T19:33:41.251+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.020168576s,]) 2019-12-07T19:33:41.251+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.686780158s delta_ub 18446744073.689383040s) v5 -- 0x558c54c1f800 con 0x558c54c18800 2019-12-07T19:33:41.251+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.251+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a400 0x558c54c37600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.251+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a400 0x558c54c37600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.251+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a800 0x558c54c37b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.251+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a400 msgr2=0x558c54c37600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 94 2019-12-07T19:33:41.251+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a400 msgr2=0x558c54c37600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.251+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a400 0x558c54c37600 unknown :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.251+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a400 0x558c54c37600 unknown :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.251+0000 7f8f24457700 10 osd.7 13 new session (outgoing) 0x558c54c0a9c0 con=0x558c54c5bc00 addr=v2:172.21.15.131:6812/37076 2019-12-07T19:33:41.251+0000 7f8f1199c700 10 osd.7 13 ms_handle_connect con 0x558c54c5bc00 2019-12-07T19:33:41.251+0000 7f8f1199c700 2 osd.7 13 ms_handle_reset con 0x558c54c5a400 session 0 2019-12-07T19:33:41.251+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 crc :-1 s=READY pgs=11 cs=0 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=64577724c341cd38 server_cookie=ea40093cfb40a50d in_seq=0 out_seq=0 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c551aee00 con 0x558c54c18400 2019-12-07T19:33:41.252+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019007813s,]) 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.687745086s delta_ub 18446744073.690543803s) v5 -- 0x558c54c1f800 con 0x558c54c18400 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 0x558c54c6e000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.252+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558c54c64000 0x558c54c39700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a800 0x558c54c37b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c54c18c00 2019-12-07T19:33:41.252+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018808560s,]) 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.687944339s delta_ub 18446744073.690743056s) v5 -- 0x558c54c1f800 con 0x558c54c18c00 2019-12-07T19:33:41.252+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5ac00 0x558c54c6f080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.252+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0b1e0 con 0x558c54c5a800 entity osd.1 addr 2019-12-07T19:33:41.252+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0b1e0 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a800 msgr2=0x558c54c37b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 95 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a800 msgr2=0x558c54c37b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.252+0000 7f8f24457700 10 osd.7 13 new session (outgoing) 0x558c54c0bba0 con=0x558c54c64000 addr=v2:172.21.15.131:6823/37078 2019-12-07T19:33:41.252+0000 7f8f1199c700 10 osd.7 13 ms_handle_connect con 0x558c54c64000 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a800 0x558c54c37b80 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5a800 0x558c54c37b80 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.252+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558c54c64000 0x558c54c39700 crc :-1 s=READY pgs=15 cs=0 l=0 rx=0 tx=0).ready entity=osd.1 client_cookie=aae4c9f3797cb75f server_cookie=f12781276c86556b in_seq=0 out_seq=0 2019-12-07T19:33:41.252+0000 7f8f1199c700 2 osd.7 13 ms_handle_reset con 0x558c54c5a800 session 0x558c54c0b1e0 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5ac00 0x558c54c6f080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 0x558c54c6e000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c0b380 con 0x558c54c5ac00 entity osd.0 addr 2019-12-07T19:33:41.252+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c0b380 osd.0 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c5ac00 0x558c54c6f080 crc :-1 s=SESSION_ACCEPTING pgs=12 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, this connection loses to existing=0x558c54c64400 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c5ac00 msgr2=0x558c54c6f080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 96 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c5ac00 msgr2=0x558c54c6f080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c5ac00 0x558c54c6f080 crc :-1 s=SESSION_ACCEPTING pgs=12 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c5ac00 0x558c54c6f080 crc :-1 s=SESSION_ACCEPTING pgs=12 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.253+0000 7f8f1199c700 2 osd.7 13 ms_handle_reset con 0x558c54c5ac00 session 0x558c54c0b380 2019-12-07T19:33:41.253+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] reap_dead start 2019-12-07T19:33:41.253+0000 7f8f23c56700 10 osd.7 13 new session (outgoing) 0x558c54c0b520 con=0x558c54c64400 addr=v2:172.21.15.131:6822/37079 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 0x558c54c6e000 crc :-1 s=READY pgs=16 cs=0 l=0 rx=0 tx=0).ready entity=osd.0 client_cookie=613b45576f9f3c20 server_cookie=9a36ce1d1f5316c2 in_seq=0 out_seq=0 2019-12-07T19:33:41.253+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.6 172.21.15.109:0/36937 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1e600 con 0x558c53ae4c00 2019-12-07T19:33:41.253+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018703558s,]) 2019-12-07T19:33:41.253+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.688961995s delta_ub 18446744073.690848058s) v5 -- 0x558c54c1fa00 con 0x558c53ae4c00 2019-12-07T19:33:41.253+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.253+0000 7f8f1199c700 10 osd.7 13 ms_handle_connect con 0x558c54c64400 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.6 172.21.15.109:0/36937 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c552d0400 2019-12-07T19:33:41.253+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018329523s,]) 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.689336030s delta_ub 18446744073.691222093s) v5 -- 0x558c551aee00 con 0x558c552d0400 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.253+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5b000 0x558c54c6f600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.253+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5b000 0x558c54c6f600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.254+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication new session 0x558c54c5c000 con 0x558c54c5b000 entity osd.6 addr 2019-12-07T19:33:41.254+0000 7f8f23c56700 10 osd.7 13 ms_handle_authentication session 0x558c54c5c000 osd.6 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.254+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 0x558c54c36b00 crc :-1 s=START_CONNECT pgs=0 cs=0 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:41.254+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c5b000 0x558c54c6f600 crc :-1 s=SESSION_ACCEPTING pgs=9 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558c54c19400 socket by this connection's socket 2019-12-07T19:33:41.254+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c5b000 0x558c54c6f600 crc :-1 s=SESSION_ACCEPTING pgs=9 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.254+0000 7f8f1199c700 2 osd.7 13 ms_handle_reset con 0x558c54c5b000 session 0x558c54c5c000 2019-12-07T19:33:41.254+0000 7f8f23c56700 10 osd.7 13 new session (incoming)0x558c54c5c1a0 con=0x558c54c19400 addr=v2:172.21.15.109:6818/36937 must have raced with connect 2019-12-07T19:33:41.254+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=READY pgs=9 cs=0 l=0 rx=0 tx=0).ready entity=osd.6 client_cookie=920af010d0047943 server_cookie=aa504281cd3e39d2 in_seq=0 out_seq=0 2019-12-07T19:33:41.257+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.706911153s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552d0c00 2019-12-07T19:33:41.257+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013462320s,]) 2019-12-07T19:33:41.257+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.693448833s delta_ub 18446744073.696089296s) v5 -- 0x558c54c58000 con 0x558c552d0c00 2019-12-07T19:33:41.257+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.257+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.706911153s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558c551afe00 con 0x558c552d1000 2019-12-07T19:33:41.257+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013346605s,]) 2019-12-07T19:33:41.257+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.693564548s delta_ub 18446744073.696205011s) v5 -- 0x558c54c58000 con 0x558c552d1000 2019-12-07T19:33:41.258+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5b800 0x558c54c70100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.258+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c5b800 0x558c54c70100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.258+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication new session 0x558c54c5c000 con 0x558c54c5b800 entity osd.4 addr 2019-12-07T19:33:41.258+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.258+0000 7f8f24457700 10 osd.7 13 ms_handle_authentication session 0x558c54c5c000 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.258+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b800 0x558c54c70100 crc :-1 s=SESSION_ACCEPTING pgs=9 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558c54c5b400 socket by this connection's socket 2019-12-07T19:33:41.258+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b800 0x558c54c70100 crc :-1 s=SESSION_ACCEPTING pgs=9 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.258+0000 7f8f1199c700 2 osd.7 13 ms_handle_reset con 0x558c54c5b800 session 0x558c54c5c000 2019-12-07T19:33:41.258+0000 7f8f24457700 10 osd.7 13 new session (incoming)0x558c54c5c820 con=0x558c54c5b400 addr=v2:172.21.15.109:6810/36938 must have raced with connect 2019-12-07T19:33:41.258+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=READY pgs=9 cs=0 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=f8982639b78e8be6 server_cookie=84aece1fce04073f in_seq=0 out_seq=0 2019-12-07T19:33:41.270+0000 7f8f0097a700 5 osd.7 13 heartbeat osd_stat(store_statfs(0x163ffe1000/0x40000000/0x1680000000, data 0x724e/0x1d000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,6] op hist []) 2019-12-07T19:33:41.270+0000 7f8f0097a700 20 osd.7 13 check_full_status cur ratio 1.31395e-06, physical ratio 1.31395e-06, new state none 2019-12-07T19:33:41.270+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.690743056s) v5 -- 0x558c54c58800 con 0x558c5476d400 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.690743056s) v5 -- 0x558c54c58a00 con 0x558c551b6800 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.689383040s) v5 -- 0x558c54c58c00 con 0x558c551b6400 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.689383040s) v5 -- 0x558c54c58e00 con 0x558c551b6000 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.674073212s) v5 -- 0x558c54c59000 con 0x558c551b7800 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.674073212s) v5 -- 0x558c54c59200 con 0x558c551b7c00 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.687979144s) v5 -- 0x558c54c58600 con 0x558c552c2000 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.687979144s) v5 -- 0x558c54c58400 con 0x558c552c2400 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.696205011s) v5 -- 0x558c54c58200 con 0x558c552c2800 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.696205011s) v5 -- 0x558c54c58000 con 0x558c552c2c00 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.679063608s) v5 -- 0x558c54c59c00 con 0x558c552c3000 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.679063608s) v5 -- 0x558c54c59e00 con 0x558c552c3400 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.691222093s) v5 -- 0x558c54c1f000 con 0x558c551b7000 2019-12-07T19:33:41.271+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.691222093s) v5 -- 0x558c54c1ee00 con 0x558c551b7400 2019-12-07T19:33:41.271+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.720656806s delta_ub 0.014036259s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c552c2800 2019-12-07T19:33:41.271+0000 7f8f24c58700 25 osd.7 13 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.271563+0000 last_rx_front 0.000000 2019-12-07T19:33:41.271+0000 7f8f24c58700 10 osd.7 13 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.271+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013653874s,0.014036259s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (unknown 0 0 0) 0x558c53a79440 con 0x558c54c19c00 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 osd.7 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 osd.7 13 _dispatch 0x558c53a79440 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.271+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6822/36937 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.725990718s delta_ub 0.019370171s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c551b7000 2019-12-07T19:33:41.271+0000 7f8f24c58700 25 osd.7 13 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.271607+0000 last_rx_front 0.000000 2019-12-07T19:33:41.271+0000 7f8f24c58700 10 osd.7 13 maybe_share_map con 0x558c54c19400 v2:172.21.15.109:6818/36937 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.271+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018944336s,0.019370171s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 3 osd.7 13 handle_osd_map epochs [14,14], i have 13, src has [1,14] 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 osd.7 13 handle_osd_map got inc map for epoch 14 2019-12-07T19:33:41.271+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.737887841s delta_ub 0.031267294s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c552c3400 2019-12-07T19:33:41.271+0000 7f8f24c58700 25 osd.7 13 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.271670+0000 2019-12-07T19:33:41.271+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.720803657s delta_ub 0.014183110s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1fe00 con 0x558c552c2c00 2019-12-07T19:33:41.271+0000 7f8f24c58700 10 osd.7 13 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.271+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:30f0e826:::osdmap.13:0# 0x0~0 2019-12-07T19:33:41.271+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030778050s,0.031267294s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546778f0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54677ab0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54677c70 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558c54677e30 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:41.271+0000 7f8f24457700 25 osd.7 13 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271563+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.271691+0000 2019-12-07T19:33:41.271+0000 7f8f24457700 25 osd.7 13 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:41.271181+0000) and older pending ping(s) 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 = 4456 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 osd.7 13 add_map_bl 13 4456 bytes 2019-12-07T19:33:41.271+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013672957s,0.014183110s]) 2019-12-07T19:33:41.271+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.743253235s delta_ub 0.036632688s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c551b7c00 2019-12-07T19:33:41.271+0000 7f8f24c58700 25 osd.7 13 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.271726+0000 2019-12-07T19:33:41.271+0000 7f8f24c58700 10 osd.7 13 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.271+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036087925s,0.036632688s]) 2019-12-07T19:33:41.271+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.729006398s delta_ub 0.022385851s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1fe00 con 0x558c552c2000 2019-12-07T19:33:41.271+0000 7f8f24457700 25 osd.7 13 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.271744+0000 last_rx_front 0.000000 2019-12-07T19:33:41.271+0000 7f8f24457700 10 osd.7 13 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.271+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021822373s,0.022385851s]) 2019-12-07T19:33:41.271+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.726313677s delta_ub 0.019693130s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c551b6800 2019-12-07T19:33:41.271+0000 7f8f24c58700 25 osd.7 13 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.271763+0000 2019-12-07T19:33:41.271+0000 7f8f24c58700 10 osd.7 13 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.271+0000 7f8f24c58700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019111236s,0.019693130s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 osd.7 13 got_full_map 14, nothing requested 2019-12-07T19:33:41.271+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.737785422s delta_ub 0.031164875s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c552c3000 2019-12-07T19:33:41.271+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6820/36937 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.726135541s delta_ub 0.019514994s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c551b7400 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 osd.7 13 handle_osd_map pg_num_history pg_num_history(e14 pg_nums {} deleted_pools ) 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {14={}} 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.271804+0000 last_rx_front 2019-12-07T19:33:41.271670+0000 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:41.271181+0000) and older pending ping(s) 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 osd.7 13 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e14 [1,14] lci=[13,14]) 2019-12-07T19:33:41.271+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030541325s,0.031164875s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f6900 seq 7 2019-12-07T19:33:41.271+0000 7f8f24457700 25 osd.7 13 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271607+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.271828+0000 2019-12-07T19:33:41.271+0000 7f8f24457700 25 osd.7 13 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:41.271181+0000) and older pending ping(s) 2019-12-07T19:33:41.271+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018867664s,0.019514994s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c3e3972f:::inc_osdmap.14:0# key 0x7f7fffffffffffffffc3e397'/!inc_osdmap.14!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:41.271+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.729127832s delta_ub 0.022507285s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c552c2400 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:41.271+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.728575314s delta_ub 0.021954767s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c551b6400 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271744+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.271858+0000 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:41.271181+0000) and older pending ping(s) 2019-12-07T19:33:41.271+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1055 2019-12-07T19:33:41.271+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021829761s,0.022507285s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~218 2019-12-07T19:33:41.271+0000 7f8f24457700 25 osd.7 13 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.271872+0000 last_rx_front 0.000000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xde8 on front/back, now 0x0~1000 2019-12-07T19:33:41.271+0000 7f8f24457700 10 osd.7 13 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.271+0000 7f8f24457700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021263958s,0.021954767s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 1 blobs 2019-12-07T19:33:41.271+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.726157852s delta_ub 0.019537305s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c5476d400 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1f000~1000/1000,1000,0 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.271885+0000 last_rx_front 2019-12-07T19:33:41.271763+0000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1f000~1000] 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:41.271181+0000) and older pending ping(s) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:41.271+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018832812s,0.019537305s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279a40 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279ab0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c55279a40) get_ref 0x0~218 Blob(0x558c55279a40 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279ab0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c55279a40) get_ref init 0x1000, 1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~218: 0x0~218 Blob(0x558c55279a40 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x218) SharedBlob(0x558c55279ab0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55279ac8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x218 2019-12-07T19:33:41.271+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.728405014s delta_ub 0.021784467s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c551b6000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277508) dirty_range mark inline shard dirty 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271872+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.271908+0000 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:41.271181+0000) and older pending ping(s) 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 = 0 2019-12-07T19:33:41.271+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021057715s,0.021784467s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3b30e826:::osdmap.14:0# key 0x7f7fffffffffffffff3b30e8'&!osdmap.14!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:41.271+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1056 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3b30e826:::osdmap.14:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~168 2019-12-07T19:33:41.271+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.743395964s delta_ub 0.036775417s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c551b7800 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.271937+0000 last_rx_front 2019-12-07T19:33:41.271726+0000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:41.271+0000 7f8f23c56700 25 osd.7 13 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:41.271181+0000) and older pending ping(s) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 2 blobs 2019-12-07T19:33:41.271+0000 7f8f23c56700 20 osd.7 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036019686s,0.036775417s]) 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x20000~2000/1000,2000,0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x20000~2000] 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279b90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279c00 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c55279b90) get_ref 0x0~1000 Blob(0x558c55279b90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279c00 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c55279b90) get_ref init 0x1000, 1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55279b90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55279c00 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55279c18 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279c70 blob([!~1000,0x21000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279ce0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c55279c70) get_ref 0x1000~168 Blob(0x558c55279c70 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c55279ce0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c55279c70) get_ref init 0x2000, 1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x558c55279c70 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558c55279ce0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55279cf8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1168 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277788) dirty_range mark inline shard dirty 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 = 0 2019-12-07T19:33:41.271+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~12 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 0 blobs 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:41.271+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:41.271+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:41.271+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 0 blobs 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6900 cost 22453 (4 ios * 4000 + 6453 bytes) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6900 onodes 0x558c53a91900,0x558c55276c80,0x558c55277400,0x558c55277680 shared_blobs 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277508) update #-1:c3e3972f:::inc_osdmap.14:0# 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c3e3972f:::inc_osdmap.14:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277788) update #-1:3b30e826:::osdmap.14:0# 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277788) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3b30e826:::osdmap.14:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6900 allocated 0x[1f000~3000] released 0x[] 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 freelist allocate 0x1f000~3000 2019-12-07T19:33:41.271+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 prepare 2019-12-07T19:33:41.271+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6900 2019-12-07T19:33:41.272+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (unknown 0 0 0) 0x558c55275440 con 0x558c54c5b400 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 _dispatch 0x558c55275440 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 3 osd.7 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 no new maps here, dropping 2019-12-07T19:33:41.272+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (crc 0 0 0) 0x558c55274fc0 con 0x558c54c19800 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 _dispatch 0x558c55274fc0 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 3 osd.7 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 no new maps here, dropping 2019-12-07T19:33:41.272+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (crc 0 0 0) 0x558c55036000 con 0x558c54c5bc00 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 _dispatch 0x558c55036000 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 3 osd.7 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 no new maps here, dropping 2019-12-07T19:33:41.272+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (crc 0 0 0) 0x558c55275680 con 0x558c54c64000 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 _dispatch 0x558c55275680 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 3 osd.7 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 no new maps here, dropping 2019-12-07T19:33:41.272+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (crc 0 0 0) 0x558c55275b00 con 0x558c54c64400 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 _dispatch 0x558c55275b00 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 3 osd.7 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 no new maps here, dropping 2019-12-07T19:33:41.272+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.6 v2:172.21.15.109:6818/36937 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (unknown 0 0 0) 0x558c55274d80 con 0x558c54c19400 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- start 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 do_waiters -- finish 2019-12-07T19:33:41.272+0000 7f8f1199c700 20 osd.7 13 _dispatch 0x558c55274d80 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.272+0000 7f8f1199c700 3 osd.7 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.272+0000 7f8f1199c700 10 osd.7 13 no new maps here, dropping 2019-12-07T19:33:41.272+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 aio_wait 2019-12-07T19:33:41.272+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6900 2019-12-07T19:33:41.272+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 io_done 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8168 < min_flush_size 524288 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8799 < min_flush_size 524288 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55277400 had 1 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55277680 had 1 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8870 < min_flush_size 524288 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8870 < min_flush_size 524288 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6018~22a6 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6018 0x6018~22a6 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x6018 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x18 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2be and padding block with 0xd42 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x82be 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:41.272+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000221358s (0.000052337s flush + 0.000169021s kv commit) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:41.272+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6900> 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c54ea4640> 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 kv_submitted 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6900 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f6900 osr 0x558c54ea4b40 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525e640) prepare_write seq 5 0x1b000~1000 crc 4c6c8a7 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7 13 _committed_osd_maps 14..14 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525e640) prepare_write seq 5 0x2000~1000 crc 205149ee 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7 13 advance to epoch 14 (<= last 14 <= newest_map 14) 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 deferred_cleanup 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 0x558c53a91900 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 deferred_cleanup 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:41.272+0000 7f8f141a1700 7 osd.7 14 consume_map version 14 2019-12-07T19:33:41.272+0000 7f8f141a1700 20 osd.7 14 prune_pg_created 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 deferred_cleanup 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:0.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 0x558c53a91900 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:1.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:2.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 deferred_cleanup 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:3.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:4.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:5.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:6.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7:7.consume_map 14 (was 13) 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:41.272+0000 7f8f141a1700 10 osd.7 14 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 deferred_cleanup 2019-12-07T19:33:41.272+0000 7f8f141a1700 7 osd.7 14 activate_map version 14 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6000 onodes 0x558c53a91900,0x558c548f2500,0x558c548f2780,0x558c548f2a00,0x558c548f3180,0x558c548f3400,0x558c548f3680,0x558c548f3900,0x558c548f3b80,0x558c55258000,0x558c55258280,0x558c55258500,0x558c55258780,0x558c55258a00,0x558c55258c80,0x558c55258f00,0x558c55259180,0x558c55259400,0x558c55259680,0x558c55259900,0x558c55259b80,0x558c55276000,0x558c55276280,0x558c55276500,0x558c55276780,0x558c55276a00,0x558c55276c80 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523a4e8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8a20 space 0x558c5523a4e8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523a5c8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8990 space 0x558c5523a5c8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523a7f8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8ab0 space 0x558c5523a7f8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f0918b700 20 osd.7 14 agent_entry empty queue 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523aa98 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8900 space 0x558c5523aa98 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523abe8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8870 space 0x558c5523abe8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523ad38 in 0x558c53a59340) _finish_write discard buffer(0x558c551a87e0 space 0x558c5523ad38 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523ada8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8b40 space 0x558c5523ada8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523aef8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8bd0 space 0x558c5523aef8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b048 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8c60 space 0x558c5523b048 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b198 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8cf0 space 0x558c5523b198 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b2e8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8d80 space 0x558c5523b2e8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b438 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8e10 space 0x558c5523b438 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b588 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8ea0 space 0x558c5523b588 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b6d8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8f30 space 0x558c5523b6d8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b828 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8fc0 space 0x558c5523b828 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523b978 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9050 space 0x558c5523b978 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523bac8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a90e0 space 0x558c5523bac8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523bc18 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9170 space 0x558c5523bc18 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523bd68 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9200 space 0x558c5523bd68 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523beb8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9290 space 0x558c5523beb8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55278018 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9320 space 0x558c55278018 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55278168 in 0x558c53a59340) _finish_write discard buffer(0x558c551a93b0 space 0x558c55278168 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552782b8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9440 space 0x558c552782b8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55278408 in 0x558c53a59340) _finish_write discard buffer(0x558c551a94d0 space 0x558c55278408 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 done 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 deferred_cleanup 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6000 [] 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 deferred_cleanup 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 0x558c53a91900,0x558c55276c80,0x558c55276f00,0x558c55277180 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677588 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9680 space 0x558c54677588 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677ac8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9710 space 0x558c54677ac8 0x0~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677e48 in 0x558c53a59340) _finish_write discard buffer(0x558c551a97a0 space 0x558c54677e48 0x1000~1000 writing nocache) 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 deferred_queued 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:33:41.272+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:41.272+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:41.276+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.326+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.376+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.426+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.476+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.526+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.544+0000 7f8f21451700 10 osd.7 14 tick 2019-12-07T19:33:41.544+0000 7f8f21451700 10 osd.7 14 do_waiters -- start 2019-12-07T19:33:41.544+0000 7f8f21451700 10 osd.7 14 do_waiters -- finish 2019-12-07T19:33:41.544+0000 7f8f21451700 20 osd.7 14 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:41.576+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258771 2019-12-07T19:33:41.576+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 10 osd.7 14 tick_without_osd_lock 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffde000/0x40000000/0x1680000000, data 0x85ce/0x20000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 25 osd.7 14 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271885+0000 last_rx_front 2019-12-07T19:33:41.271763+0000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 25 osd.7 14 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271872+0000 last_rx_front 2019-12-07T19:33:41.271908+0000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 25 osd.7 14 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271937+0000 last_rx_front 2019-12-07T19:33:41.271726+0000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 25 osd.7 14 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271744+0000 last_rx_front 2019-12-07T19:33:41.271858+0000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 25 osd.7 14 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271563+0000 last_rx_front 2019-12-07T19:33:41.271691+0000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 25 osd.7 14 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271804+0000 last_rx_front 2019-12-07T19:33:41.271670+0000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 25 osd.7 14 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271607+0000 last_rx_front 2019-12-07T19:33:41.271828+0000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 20 osd.7 14 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 10 osd.7 14 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 20 osd.7 14 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:41.590+0000 7f8f1fbbe700 10 osd.7 14 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:41.626+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.676+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.727+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.741+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.207320920s delta_ub 0.031267294s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1fe00 con 0x558c552d0000 2019-12-07T19:33:41.741+0000 7f8f24457700 20 osd.7 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030433715s,0.031267294s]) 2019-12-07T19:33:41.741+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.176887205s delta_ub 18446744073.679117901s) v5 -- 0x558c54c1fc00 con 0x558c552d0000 2019-12-07T19:33:41.741+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.207320920s delta_ub 0.031267294s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c552d0800 2019-12-07T19:33:41.741+0000 7f8f23c56700 20 osd.7 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030389255s,0.031267294s]) 2019-12-07T19:33:41.741+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.176931665s delta_ub 18446744073.679162361s) v5 -- 0x558c54c58600 con 0x558c552d0800 2019-12-07T19:33:41.777+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.827+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.877+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.927+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.977+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.027+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.077+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.128+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.178+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.228+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.278+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.328+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.807392265s delta_ub 0.014183110s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552d0c00 2019-12-07T19:33:42.358+0000 7f8f24c58700 20 osd.7 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013483574s,0.014183110s]) 2019-12-07T19:33:42.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.793908691s delta_ub 18446744073.696068042s) v5 -- 0x558c54c1f000 con 0x558c552d0c00 2019-12-07T19:33:42.358+0000 7f8f24c58700 10 osd.7 14 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:42.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.807392265s delta_ub 0.014183110s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c552d1000 2019-12-07T19:33:42.358+0000 7f8f24c58700 20 osd.7 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013375903s,0.014183110s]) 2019-12-07T19:33:42.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.794016362s delta_ub 18446744073.696175713s) v5 -- 0x558c54c1f000 con 0x558c552d1000 2019-12-07T19:33:42.358+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 2 ==== osd_map(15..15 src has 1..15) v4 ==== 380+0+0 (unknown 0 0 0) 0x558c55036000 con 0x558c54c5b400 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 osd.7 14 OSD::ms_dispatch: osd_map(15..15 src has 1..15) v4 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 osd.7 14 do_waiters -- start 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 osd.7 14 do_waiters -- finish 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 osd.7 14 _dispatch 0x558c55036000 osd_map(15..15 src has 1..15) v4 2019-12-07T19:33:42.358+0000 7f8f1199c700 3 osd.7 14 handle_osd_map epochs [15,15], i have 14, src has [1,15] 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 osd.7 14 handle_osd_map got inc map for epoch 15 2019-12-07T19:33:42.358+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3b30e826:::osdmap.14:0# 0x0~0 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279b90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55279c00 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279c70 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558c55279ce0 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 = 4456 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 osd.7 14 add_map_bl 14 4456 bytes 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 osd.7 14 got_full_map 15, nothing requested 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 osd.7 14 handle_osd_map pg_num_history pg_num_history(e15 pg_nums {} deleted_pools ) 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {15={}} 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 osd.7 14 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e15 [1,15] lci=[13,15]) 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f7800 seq 8 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c923972f:::inc_osdmap.15:0# key 0x7f7fffffffffffffffc92397'/!inc_osdmap.15!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:42.358+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c923972f:::inc_osdmap.15:0# 0x0~150 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1057 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c923972f:::inc_osdmap.15:0# 0x0~150 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~150 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xeb0 on front/back, now 0x0~1000 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 1 blobs 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:42.358+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x22000~1000/1000,1000,0 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x22000~1000] 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54ea3c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54ea3c70 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3730 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea3c70) get_ref 0x0~150 Blob(0x558c54ea3c70 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3730 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea3c70) get_ref init 0x1000, 1000 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~150: 0x0~150 Blob(0x558c54ea3c70 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x150) SharedBlob(0x558c54ea3730 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea3748 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x150 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.358+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277a08) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c923972f:::inc_osdmap.15:0# 0x0~150 = 0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3630e826:::osdmap.15:0# key 0x7f7fffffffffffffff3630e8'&!osdmap.15!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:42.359+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3630e826:::osdmap.15:0# 0x0~1168 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1058 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3630e826:::osdmap.15:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~168 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 2 blobs 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x23000~2000/1000,2000,0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x23000~2000] 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552788c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552788c0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279f10 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c552788c0) get_ref 0x0~1000 Blob(0x558c552788c0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279f10 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c552788c0) get_ref init 0x1000, 1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c552788c0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55279f10 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55279f28 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279ea0 blob([!~1000,0x24000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279e30 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c55279ea0) get_ref 0x1000~168 Blob(0x558c55279ea0 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c55279e30 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c55279ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x558c55279ea0 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558c55279e30 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55279e48 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1168 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277c88) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3630e826:::osdmap.15:0# 0x0~1168 = 0 2019-12-07T19:33:42.359+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~12 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 0 blobs 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:42.359+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:42.359+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:42.359+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 0 blobs 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 22253 (4 ios * 4000 + 6253 bytes) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes 0x558c53a91900,0x558c55276c80,0x558c55277900,0x558c55277b80 shared_blobs 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277a08) update #-1:c923972f:::inc_osdmap.15:0# 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277a08) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c923972f:::inc_osdmap.15:0# is 46 (22 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277c88) update #-1:3630e826:::osdmap.15:0# 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55277c88) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3630e826:::osdmap.15:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[22000~3000] released 0x[] 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 freelist allocate 0x22000~3000 2019-12-07T19:33:42.359+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:33:42.359+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f7800 2019-12-07T19:33:42.359+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 aio_wait 2019-12-07T19:33:42.359+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:33:42.359+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8787 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55277900 had 1 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55277b80 had 1 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8810 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8810 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x82be~226a to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x82be 0x82be~226a to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x82be 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2be 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x528 and padding block with 0xad8 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xa528 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:42.359+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000210448s (0.000056524s flush + 0.000153924s kv commit) 2019-12-07T19:33:42.359+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7800> 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:42.359+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f7800 osr 0x558c54ea4b40 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525e640) _discard drop 5 0x1b000~1000 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7 14 _committed_osd_maps 15..15 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525e640) prepare_write seq 6 0x1b000~1000 crc 1e19645a 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7 14 advance to epoch 15 (<= last 15 <= newest_map 15) 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525e640) _discard drop 5 0x2000~1000 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525e640) prepare_write seq 6 0x2000~1000 crc 9b6a5dd3 2019-12-07T19:33:42.359+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:42.359+0000 7f8f141a1700 7 osd.7 15 consume_map version 15 2019-12-07T19:33:42.359+0000 7f8f141a1700 20 osd.7 15 prune_pg_created 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:0.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:1.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:2.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:3.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:4.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:5.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:6.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7:7.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f8f141a1700 10 osd.7 15 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:42.359+0000 7f8f141a1700 7 osd.7 15 activate_map version 15 2019-12-07T19:33:42.359+0000 7f8f0918b700 20 osd.7 15 agent_entry empty queue 2019-12-07T19:33:42.378+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.428+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.478+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.529+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.544+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:42.544+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:42.544+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:42.544+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:42.579+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:33:42.579+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:33:42.579+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 6 0x2000~1000 2019-12-07T19:33:42.579+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 9b6a5dd3 2019-12-07T19:33:42.579+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 6 0x1b000~1000 2019-12-07T19:33:42.579+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 1e19645a 2019-12-07T19:33:42.579+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.579+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:33:42.579+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:33:42.629+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271885+0000 last_rx_front 2019-12-07T19:33:41.271763+0000 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271872+0000 last_rx_front 2019-12-07T19:33:41.271908+0000 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271937+0000 last_rx_front 2019-12-07T19:33:41.271726+0000 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271744+0000 last_rx_front 2019-12-07T19:33:41.271858+0000 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271563+0000 last_rx_front 2019-12-07T19:33:41.271691+0000 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271804+0000 last_rx_front 2019-12-07T19:33:41.271670+0000 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271607+0000 last_rx_front 2019-12-07T19:33:41.271828+0000 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 20 osd.7 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:42.637+0000 7f8f1fbbe700 20 osd.7 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:42.638+0000 7f8f1fbbe700 20 osd.7 15 scrub_load_below_threshold loadavg per cpu 0.0725 < max 5 = yes 2019-12-07T19:33:42.638+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub load_is_low=1 2019-12-07T19:33:42.638+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub done 2019-12-07T19:33:42.638+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:42.638+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:42.638+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:42.679+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.729+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.779+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.829+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.307750221s delta_ub 0.014167929s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552d0c00 2019-12-07T19:33:42.858+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013598565s,0.014167929s]) 2019-12-07T19:33:42.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.294151656s delta_ub 18446744073.695953051s) v5 -- 0x558c54c1f000 con 0x558c552d0c00 2019-12-07T19:33:42.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.307750221s delta_ub 0.014167929s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552d1000 2019-12-07T19:33:42.858+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013565099s,0.014167929s]) 2019-12-07T19:33:42.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.294185122s delta_ub 18446744073.695986517s) v5 -- 0x558c54c59e00 con 0x558c552d1000 2019-12-07T19:33:42.879+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.930+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.980+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.030+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.080+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.130+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.180+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.230+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.280+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.330+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.381+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.431+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.481+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.531+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.540+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:43.540+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:43.540+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:43.540+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:43.581+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271885+0000 last_rx_front 2019-12-07T19:33:41.271763+0000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271872+0000 last_rx_front 2019-12-07T19:33:41.271908+0000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271937+0000 last_rx_front 2019-12-07T19:33:41.271726+0000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271744+0000 last_rx_front 2019-12-07T19:33:41.271858+0000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271563+0000 last_rx_front 2019-12-07T19:33:41.271691+0000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271804+0000 last_rx_front 2019-12-07T19:33:41.271670+0000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271607+0000 last_rx_front 2019-12-07T19:33:41.271828+0000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:43.626+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:43.631+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.682+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.732+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.782+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.832+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.882+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.932+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.982+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.032+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.041+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.507816348s delta_ub 0.031024245s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c552d0800 2019-12-07T19:33:44.041+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030625482s,0.031024245s]) 2019-12-07T19:33:44.041+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.507816348s delta_ub 0.031024245s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1fe00 con 0x558c552d0000 2019-12-07T19:33:44.041+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.477190866s delta_ub 18446744073.678926134s) v5 -- 0x558c54c58600 con 0x558c552d0800 2019-12-07T19:33:44.041+0000 7f8f23c56700 10 osd.7 15 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.041+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030598202s,0.031024245s]) 2019-12-07T19:33:44.041+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.477218146s delta_ub 18446744073.678953414s) v5 -- 0x558c54c1fc00 con 0x558c552d0000 2019-12-07T19:33:44.082+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.132+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.183+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.233+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.283+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.333+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.383+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.433+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.483+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.533+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.556+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:44.556+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:44.556+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:44.556+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:44.583+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.633+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271885+0000 last_rx_front 2019-12-07T19:33:41.271763+0000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271872+0000 last_rx_front 2019-12-07T19:33:41.271908+0000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271937+0000 last_rx_front 2019-12-07T19:33:41.271726+0000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271744+0000 last_rx_front 2019-12-07T19:33:41.271858+0000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271563+0000 last_rx_front 2019-12-07T19:33:41.271691+0000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271804+0000 last_rx_front 2019-12-07T19:33:41.271670+0000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:41.271181+0000 last_rx_back 2019-12-07T19:33:41.271607+0000 last_rx_front 2019-12-07T19:33:41.271828+0000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:44.648+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:44.684+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.734+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.019693130s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c54c18c00 2019-12-07T19:33:44.752+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019014675s,0.019693130s]) 2019-12-07T19:33:44.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.188117821s delta_ub 18446744073.690536941s) v5 -- 0x558c54c58600 con 0x558c54c18c00 2019-12-07T19:33:44.752+0000 7f8f23c56700 10 osd.7 15 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.019693130s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c18400 2019-12-07T19:33:44.752+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018857640s,0.019693130s]) 2019-12-07T19:33:44.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.188274856s delta_ub 18446744073.690693976s) v5 -- 0x558c54c58600 con 0x558c54c18400 2019-12-07T19:33:44.784+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.834+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.884+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.934+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.984+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.034+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.084+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.134+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.185+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.235+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.285+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.335+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.350+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.807969424s delta_ub 0.022507285s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d1400 2019-12-07T19:33:45.350+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021886005s,0.022507285s]) 2019-12-07T19:33:45.350+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.786083419s delta_ub 18446744073.687665611s) v5 -- 0x558c54c1fc00 con 0x558c552d1400 2019-12-07T19:33:45.350+0000 7f8f24457700 10 osd.7 15 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.350+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.807969424s delta_ub 0.022507285s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c54c18000 2019-12-07T19:33:45.350+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021825940s,0.022507285s]) 2019-12-07T19:33:45.350+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.786143484s delta_ub 18446744073.687725676s) v5 -- 0x558c54c58600 con 0x558c54c18000 2019-12-07T19:33:45.350+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.807380414s delta_ub 0.021954767s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c19000 2019-12-07T19:33:45.350+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021185971s,0.021954767s]) 2019-12-07T19:33:45.350+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.786194443s delta_ub 18446744073.688365645s) v5 -- 0x558c54c58600 con 0x558c54c19000 2019-12-07T19:33:45.350+0000 7f8f23c56700 10 osd.7 15 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.350+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.807380414s delta_ub 0.021954767s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c18800 2019-12-07T19:33:45.350+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021153761s,0.021954767s]) 2019-12-07T19:33:45.350+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.786226653s delta_ub 18446744073.688397855s) v5 -- 0x558c54c58000 con 0x558c54c18800 2019-12-07T19:33:45.371+0000 7f8f0097a700 5 osd.7 15 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,6] op hist []) 2019-12-07T19:33:45.371+0000 7f8f0097a700 20 osd.7 15 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690693976s) v5 -- 0x558c54c1ec00 con 0x558c5476d400 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690693976s) v5 -- 0x558c54c59200 con 0x558c551b6800 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.688397855s) v5 -- 0x558c54c58200 con 0x558c551b6400 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.688397855s) v5 -- 0x558c548cda00 con 0x558c551b6000 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.673531930s) v5 -- 0x558c55270000 con 0x558c551b7800 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.673531930s) v5 -- 0x558c55270200 con 0x558c551b7c00 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.687725676s) v5 -- 0x558c55270400 con 0x558c552c2000 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.687725676s) v5 -- 0x558c55270600 con 0x558c552c2400 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.695986517s) v5 -- 0x558c55270800 con 0x558c552c2800 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.695986517s) v5 -- 0x558c55270a00 con 0x558c552c2c00 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.678953414s) v5 -- 0x558c55270c00 con 0x558c552c3000 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.678953414s) v5 -- 0x558c54c1f200 con 0x558c552c3400 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690683952s) v5 -- 0x558c54c1f400 con 0x558c551b7000 2019-12-07T19:33:45.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690683952s) v5 -- 0x558c54c59400 con 0x558c551b7400 2019-12-07T19:33:45.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.820928463s delta_ub 0.013933203s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552c2800 2019-12-07T19:33:45.371+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271563+0000 -> 2019-12-07T19:33:45.371764+0000 last_rx_front 2019-12-07T19:33:41.271691+0000 2019-12-07T19:33:45.371+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013724804s,0.013933203s]) 2019-12-07T19:33:45.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.838052932s delta_ub 0.031057672s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552c3400 2019-12-07T19:33:45.371+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.820986642s delta_ub 0.013991382s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c552c2c00 2019-12-07T19:33:45.371+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271804+0000 last_rx_front 2019-12-07T19:33:41.271670+0000 -> 2019-12-07T19:33:45.371808+0000 2019-12-07T19:33:45.371+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030804999s,0.031057672s]) 2019-12-07T19:33:45.371+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371764+0000 last_rx_front 2019-12-07T19:33:41.271691+0000 -> 2019-12-07T19:33:45.371821+0000 2019-12-07T19:33:45.371+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:45.371556+0000) and older pending ping(s) 2019-12-07T19:33:45.371+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013726235s,0.013991382s]) 2019-12-07T19:33:45.371+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.829330176s delta_ub 0.022334916s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c552c2000 2019-12-07T19:33:45.371+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271744+0000 -> 2019-12-07T19:33:45.371855+0000 last_rx_front 2019-12-07T19:33:41.271858+0000 2019-12-07T19:33:45.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6822/36937 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826250921s delta_ub 0.019255661s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c551b7000 2019-12-07T19:33:45.371+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022035200s,0.022334916s]) 2019-12-07T19:33:45.371+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271607+0000 -> 2019-12-07T19:33:45.371863+0000 last_rx_front 2019-12-07T19:33:41.271828+0000 2019-12-07T19:33:45.371+0000 7f8f24c58700 10 osd.7 15 maybe_share_map con 0x558c54c19400 v2:172.21.15.109:6818/36937 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.371+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018948768s,0.019255661s]) 2019-12-07T19:33:45.371+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.828781102s delta_ub 0.021785842s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c551b6400 2019-12-07T19:33:45.371+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271872+0000 -> 2019-12-07T19:33:45.371875+0000 last_rx_front 2019-12-07T19:33:41.271908+0000 2019-12-07T19:33:45.371+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021466328s,0.021785842s]) 2019-12-07T19:33:45.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.843515886s delta_ub 0.036520626s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c551b7c00 2019-12-07T19:33:45.371+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6820/36937 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826328817s delta_ub 0.019333557s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c551b7400 2019-12-07T19:33:45.371+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271937+0000 last_rx_front 2019-12-07T19:33:41.271726+0000 -> 2019-12-07T19:33:45.371887+0000 2019-12-07T19:33:45.371+0000 7f8f24c58700 10 osd.7 15 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.371+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036188918s,0.036520626s]) 2019-12-07T19:33:45.371+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371863+0000 last_rx_front 2019-12-07T19:33:41.271828+0000 -> 2019-12-07T19:33:45.371896+0000 2019-12-07T19:33:45.371+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:45.371556+0000) and older pending ping(s) 2019-12-07T19:33:45.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826478726s delta_ub 0.019483466s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c551b6800 2019-12-07T19:33:45.371+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018992950s,0.019333557s]) 2019-12-07T19:33:45.371+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271885+0000 last_rx_front 2019-12-07T19:33:41.271763+0000 -> 2019-12-07T19:33:45.371910+0000 2019-12-07T19:33:45.371+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.838077955s delta_ub 0.031082695s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c552c3000 2019-12-07T19:33:45.371+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019128822s,0.019483466s]) 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271804+0000 -> 2019-12-07T19:33:45.371926+0000 last_rx_front 2019-12-07T19:33:45.371808+0000 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:45.371556+0000) and older pending ping(s) 2019-12-07T19:33:45.371+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030712687s,0.031082695s]) 2019-12-07T19:33:45.371+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.828707474s delta_ub 0.021712214s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c551b6000 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371875+0000 last_rx_front 2019-12-07T19:33:41.271908+0000 -> 2019-12-07T19:33:45.371950+0000 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:45.371556+0000) and older pending ping(s) 2019-12-07T19:33:45.371+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021317679s,0.021712214s]) 2019-12-07T19:33:45.371+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.829303780s delta_ub 0.022308520s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c552c2400 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371855+0000 last_rx_front 2019-12-07T19:33:41.271858+0000 -> 2019-12-07T19:33:45.371967+0000 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:45.371556+0000) and older pending ping(s) 2019-12-07T19:33:45.371+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021896994s,0.022308520s]) 2019-12-07T19:33:45.371+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826523956s delta_ub 0.019528696s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c5476d400 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271885+0000 -> 2019-12-07T19:33:45.371985+0000 last_rx_front 2019-12-07T19:33:45.371910+0000 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:45.371556+0000) and older pending ping(s) 2019-12-07T19:33:45.371+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019098824s,0.019528696s]) 2019-12-07T19:33:45.371+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.843554936s delta_ub 0.036559676s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ea00 con 0x558c551b7800 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:41.271937+0000 -> 2019-12-07T19:33:45.372003+0000 last_rx_front 2019-12-07T19:33:45.371887+0000 2019-12-07T19:33:45.371+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:45.371556+0000) and older pending ping(s) 2019-12-07T19:33:45.371+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036112262s,0.036559676s]) 2019-12-07T19:33:45.385+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.435+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.485+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.521+0000 7f8f1219d700 20 osd.7 15 reports for 0 queries 2019-12-07T19:33:45.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c5489e380 con 0x558c5476d800 2019-12-07T19:33:45.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558c5489e700 con 0x558c5476d800 2019-12-07T19:33:45.535+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.585+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:45.585+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.592+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:45.592+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:45.592+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:45.592+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:45.636+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.686+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371985+0000 last_rx_front 2019-12-07T19:33:45.371910+0000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371875+0000 last_rx_front 2019-12-07T19:33:45.371950+0000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.372003+0000 last_rx_front 2019-12-07T19:33:45.371887+0000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371855+0000 last_rx_front 2019-12-07T19:33:45.371967+0000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371764+0000 last_rx_front 2019-12-07T19:33:45.371821+0000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371926+0000 last_rx_front 2019-12-07T19:33:45.371808+0000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371863+0000 last_rx_front 2019-12-07T19:33:45.371896+0000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:45.696+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:45.736+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.786+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.821+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c64800 0x558c54c70680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:45.821+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c64800 0x558c54c70680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:45.821+0000 7f8f23c56700 10 osd.7 15 ms_handle_authentication new session 0x558c54c5c340 con 0x558c54c64800 entity client.admin addr 2019-12-07T19:33:45.821+0000 7f8f23c56700 10 osd.7 15 ms_handle_authentication session 0x558c54c5c340 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:45.821+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1086357697 conn(0x558c54c64800 0x558c54c70680 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4333 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:45.821+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4333 172.21.15.131:0/1086357697 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c53a71860 con 0x558c54c64800 2019-12-07T19:33:45.821+0000 7f8f164bb700 20 osd.7 15 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:45.821+0000 7f8f164bb700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:45.821+0000 7f8f164bb700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:45.821+0000 7f8f164bb700 20 osd.7 15 _dispatch 0x558c53a71860 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:45.822+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1086357697 -- command_reply(tid 1: 0 ) v1 -- 0x558c53a6ec40 con 0x558c54c64800 2019-12-07T19:33:45.836+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.842+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4333 172.21.15.131:0/1086357697 2 ==== command(tid 2: {"prefix": "flush_pg_stats"}) v1 ==== 52+0+0 (crc 0 0 0) 0x558c53a71c20 con 0x558c54c64800 2019-12-07T19:33:45.842+0000 7f8f164bb700 20 osd.7 15 OSD::ms_dispatch: command(tid 2: {"prefix": "flush_pg_stats"}) v1 2019-12-07T19:33:45.842+0000 7f8f164bb700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:45.842+0000 7f8f164bb700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:45.842+0000 7f8f164bb700 20 osd.7 15 _dispatch 0x558c53a71c20 command(tid 2: {"prefix": "flush_pg_stats"}) v1 2019-12-07T19:33:45.842+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558c5489f500 con 0x558c5476d800 2019-12-07T19:33:45.842+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1086357697 -- command_reply(tid 2: 0 ) v1 -- 0x558c53a6f180 con 0x558c54c64800 2019-12-07T19:33:45.844+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1086357697 conn(0x558c54c64800 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:45.844+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1086357697 conn(0x558c54c64800 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:45.844+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1086357697 conn(0x558c54c64800 0x558c54c70680 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:45.844+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1086357697 conn(0x558c54c64800 0x558c54c70680 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:45.844+0000 7f8f164bb700 2 osd.7 15 ms_handle_reset con 0x558c54c64800 session 0x558c54c5c340 2019-12-07T19:33:45.886+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.936+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.6 172.21.15.109:0/36937 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.019333557s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c552d0400 2019-12-07T19:33:45.953+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018807090s,0.019333557s]) 2019-12-07T19:33:45.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.389243407s delta_ub 18446744073.690744526s) v5 -- 0x558c54c1ea00 con 0x558c552d0400 2019-12-07T19:33:45.953+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.6 172.21.15.109:0/36937 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.019333557s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c53ae4c00 2019-12-07T19:33:45.953+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018776168s,0.019333557s]) 2019-12-07T19:33:45.953+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.389274329s delta_ub 18446744073.690775448s) v5 -- 0x558c54c58800 con 0x558c53ae4c00 2019-12-07T19:33:45.986+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.036+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.087+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.137+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.187+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.237+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.287+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.337+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.808045243s delta_ub 0.013991382s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552d0c00 2019-12-07T19:33:46.358+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013542914s,0.013991382s]) 2019-12-07T19:33:46.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.794502329s delta_ub 18446744073.696008702s) v5 -- 0x558c54c1f400 con 0x558c552d0c00 2019-12-07T19:33:46.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.808045243s delta_ub 0.013991382s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5ee00 con 0x558c552d1000 2019-12-07T19:33:46.358+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013504787s,0.013991382s]) 2019-12-07T19:33:46.358+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.794540456s delta_ub 18446744073.696046829s) v5 -- 0x558c54c1f200 con 0x558c552d1000 2019-12-07T19:33:46.387+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.437+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.450+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.908320123s delta_ub 0.022334916s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d1400 2019-12-07T19:33:46.450+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021861396s,0.022334916s]) 2019-12-07T19:33:46.450+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.886458727s delta_ub 18446744073.687690220s) v5 -- 0x558c54c58800 con 0x558c552d1400 2019-12-07T19:33:46.450+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.908320123s delta_ub 0.022334916s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c18000 2019-12-07T19:33:46.450+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021769652s,0.022334916s]) 2019-12-07T19:33:46.450+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.886550471s delta_ub 18446744073.687781964s) v5 -- 0x558c54c1ea00 con 0x558c54c18000 2019-12-07T19:33:46.487+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.536+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 10.007789311s delta_ub 0.036559676s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c552d1c00 2019-12-07T19:33:46.536+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 10.007789311s delta_ub 0.036559676s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c552d1800 2019-12-07T19:33:46.536+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036062978s,0.036559676s]) 2019-12-07T19:33:46.536+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 9.971726333s delta_ub 18446744073.673488638s) v5 -- 0x558c54c58800 con 0x558c552d1c00 2019-12-07T19:33:46.536+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036041269s,0.036559676s]) 2019-12-07T19:33:46.536+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 9.971748042s delta_ub 18446744073.673510347s) v5 -- 0x558c54c1f200 con 0x558c552d1800 2019-12-07T19:33:46.537+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.547+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:46.547+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:46.547+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:46.547+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:46.570+0000 7f8f154b9700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- log(2 entries from seq 1 at 2019-12-07T19:33:37.591419+0000) v1 -- 0x558c5524c000 con 0x558c5476d000 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:46.588+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258772 2019-12-07T19:33:46.588+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.638+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.688+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.738+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371985+0000 last_rx_front 2019-12-07T19:33:45.371910+0000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371875+0000 last_rx_front 2019-12-07T19:33:45.371950+0000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.372003+0000 last_rx_front 2019-12-07T19:33:45.371887+0000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371855+0000 last_rx_front 2019-12-07T19:33:45.371967+0000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371764+0000 last_rx_front 2019-12-07T19:33:45.371821+0000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371926+0000 last_rx_front 2019-12-07T19:33:45.371808+0000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371863+0000 last_rx_front 2019-12-07T19:33:45.371896+0000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:46.745+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:46.788+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.838+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.888+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.939+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.989+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.039+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.089+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.139+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.189+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.218+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 12 ==== log(last 2) v1 ==== 28+0+0 (crc 0 0 0) 0x558c53a70f00 con 0x558c5476d000 2019-12-07T19:33:47.239+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.289+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.339+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.390+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.440+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.490+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.540+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.572+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:47.572+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:47.572+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:47.572+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:47.590+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.640+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.690+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371985+0000 last_rx_front 2019-12-07T19:33:45.371910+0000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371875+0000 last_rx_front 2019-12-07T19:33:45.371950+0000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.372003+0000 last_rx_front 2019-12-07T19:33:45.371887+0000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371855+0000 last_rx_front 2019-12-07T19:33:45.371967+0000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371764+0000 last_rx_front 2019-12-07T19:33:45.371821+0000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371926+0000 last_rx_front 2019-12-07T19:33:45.371808+0000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371863+0000 last_rx_front 2019-12-07T19:33:45.371896+0000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:47.700+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:47.740+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.790+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.841+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.891+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.941+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.991+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.041+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.091+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.141+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.151+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.608668265s delta_ub 0.022204754s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c54c18000 2019-12-07T19:33:48.151+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021808892s,0.022204754s]) 2019-12-07T19:33:48.151+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.586859373s delta_ub 18446744073.687742724s) v5 -- 0x558c54c1ea00 con 0x558c54c18000 2019-12-07T19:33:48.151+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.608668265s delta_ub 0.022204754s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d1400 2019-12-07T19:33:48.151+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021730725s,0.022204754s]) 2019-12-07T19:33:48.151+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.586937540s delta_ub 18446744073.687820891s) v5 -- 0x558c54c58800 con 0x558c552d1400 2019-12-07T19:33:48.191+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.241+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.291+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.342+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.392+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.442+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.492+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.531+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:48.531+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:48.531+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:48.531+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:48.542+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.592+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:48.592+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.642+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371985+0000 last_rx_front 2019-12-07T19:33:45.371910+0000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371875+0000 last_rx_front 2019-12-07T19:33:45.371950+0000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.372003+0000 last_rx_front 2019-12-07T19:33:45.371887+0000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371855+0000 last_rx_front 2019-12-07T19:33:45.371967+0000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371764+0000 last_rx_front 2019-12-07T19:33:45.371821+0000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371926+0000 last_rx_front 2019-12-07T19:33:45.371808+0000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:45.371556+0000 last_rx_back 2019-12-07T19:33:45.371863+0000 last_rx_front 2019-12-07T19:33:45.371896+0000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:48.687+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:48.693+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.743+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.793+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.843+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.852+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.019528696s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c18400 2019-12-07T19:33:48.852+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018951956s,0.019528696s]) 2019-12-07T19:33:48.852+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.288469078s delta_ub 18446744073.690599660s) v5 -- 0x558c54c1ea00 con 0x558c54c18400 2019-12-07T19:33:48.852+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.019528696s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c54c18c00 2019-12-07T19:33:48.852+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018808532s,0.019528696s]) 2019-12-07T19:33:48.853+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.288612502s delta_ub 18446744073.690743084s) v5 -- 0x558c54c1ea00 con 0x558c54c18c00 2019-12-07T19:33:48.871+0000 7f8f0097a700 5 osd.7 15 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,6] op hist []) 2019-12-07T19:33:48.871+0000 7f8f0097a700 20 osd.7 15 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690743084s) v5 -- 0x558c54c59600 con 0x558c5476d400 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690743084s) v5 -- 0x558c54c59800 con 0x558c551b6800 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.688233937s) v5 -- 0x558c54c59a00 con 0x558c551b6400 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.688233937s) v5 -- 0x558c54c58600 con 0x558c551b6000 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.673510347s) v5 -- 0x558c54c58000 con 0x558c551b7800 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.673510347s) v5 -- 0x558c55270a00 con 0x558c551b7c00 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.687820891s) v5 -- 0x558c55270400 con 0x558c552c2000 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.687820891s) v5 -- 0x558c55270000 con 0x558c552c2400 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.696046829s) v5 -- 0x558c548cda00 con 0x558c552c2800 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.696046829s) v5 -- 0x558c54c1fa00 con 0x558c552c2c00 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.678838929s) v5 -- 0x558c54c1ee00 con 0x558c552c3000 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.678838929s) v5 -- 0x558c54c1fc00 con 0x558c552c3400 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690775448s) v5 -- 0x558c54c1fe00 con 0x558c551b7000 2019-12-07T19:33:48.871+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690775448s) v5 -- 0x558c54c1ec00 con 0x558c551b7400 2019-12-07T19:33:48.871+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.321317194s delta_ub 0.013987622s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c552c2800 2019-12-07T19:33:48.871+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.371764+0000 -> 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:45.371821+0000 2019-12-07T19:33:48.871+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013708680s,0.013987622s]) 2019-12-07T19:33:48.872+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.343882694s delta_ub 0.036553122s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c551b7c00 2019-12-07T19:33:48.872+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.338450184s delta_ub 0.031120612s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c552c3000 2019-12-07T19:33:48.872+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.372003+0000 last_rx_front 2019-12-07T19:33:45.371887+0000 -> 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:48.872+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.329681101s delta_ub 0.022351529s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552c2000 2019-12-07T19:33:48.872+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036228318s,0.036553122s]) 2019-12-07T19:33:48.872+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.371855+0000 -> 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:45.371967+0000 2019-12-07T19:33:48.872+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021994822s,0.022351529s]) 2019-12-07T19:33:48.872+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326894526s delta_ub 0.019564954s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c551b6800 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.371926+0000 -> 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:45.371808+0000 2019-12-07T19:33:48.872+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030742571s,0.031120612s]) 2019-12-07T19:33:48.872+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.371985+0000 last_rx_front 2019-12-07T19:33:45.371910+0000 -> 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:48.872+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019173050s,0.019564954s]) 2019-12-07T19:33:48.872+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.321387880s delta_ub 0.014058308s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552c2c00 2019-12-07T19:33:48.872+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.329702894s delta_ub 0.022373322s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c552c2400 2019-12-07T19:33:48.872+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:45.371821+0000 -> 2019-12-07T19:33:48.872295+0000 2019-12-07T19:33:48.872+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6822/36937 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326655872s delta_ub 0.019326300s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c551b7000 2019-12-07T19:33:48.872+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:48.871890+0000) and older pending ping(s) 2019-12-07T19:33:48.872+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013653345s,0.014058308s]) 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:45.371967+0000 -> 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:48.871890+0000) and older pending ping(s) 2019-12-07T19:33:48.872+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021942176s,0.022373322s]) 2019-12-07T19:33:48.872+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.329125985s delta_ub 0.021796413s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c551b6400 2019-12-07T19:33:48.872+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.371863+0000 -> 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:45.371896+0000 2019-12-07T19:33:48.872+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018879515s,0.019326300s]) 2019-12-07T19:33:48.872+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326851678s delta_ub 0.019522106s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c5476d400 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.371985+0000 -> 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:48.871890+0000) and older pending ping(s) 2019-12-07T19:33:48.872+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019063830s,0.019522106s]) 2019-12-07T19:33:48.872+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.338529461s delta_ub 0.031199889s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c552c3400 2019-12-07T19:33:48.872+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:45.371808+0000 -> 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:48.872+0000 7f8f24c58700 25 osd.7 15 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:48.871890+0000) and older pending ping(s) 2019-12-07T19:33:48.872+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.329083408s delta_ub 0.021753836s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c551b6000 2019-12-07T19:33:48.872+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030730370s,0.031199889s]) 2019-12-07T19:33:48.872+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.371875+0000 -> 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:45.371950+0000 2019-12-07T19:33:48.872+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021312840s,0.021796413s]) 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:45.371950+0000 -> 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:48.871890+0000) and older pending ping(s) 2019-12-07T19:33:48.872+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021256736s,0.021753836s]) 2019-12-07T19:33:48.872+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6820/36937 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326741970s delta_ub 0.019412398s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c551b7400 2019-12-07T19:33:48.872+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.343954254s delta_ub 0.036624682s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c551b7800 2019-12-07T19:33:48.872+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:45.371896+0000 -> 2019-12-07T19:33:48.872410+0000 2019-12-07T19:33:48.872+0000 7f8f24457700 25 osd.7 15 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:48.871890+0000) and older pending ping(s) 2019-12-07T19:33:48.872+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018892140s,0.019412398s]) 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:45.372003+0000 -> 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:48.872+0000 7f8f23c56700 25 osd.7 15 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:48.871890+0000) and older pending ping(s) 2019-12-07T19:33:48.872+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036085130s,0.036624682s]) 2019-12-07T19:33:48.893+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.943+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.993+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.043+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.093+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.143+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.194+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.244+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.708361738s delta_ub 0.014058308s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c552d0c00 2019-12-07T19:33:49.259+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013549975s,0.014058308s]) 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.694811763s delta_ub 18446744073.696001641s) v5 -- 0x558c54c58800 con 0x558c552d0c00 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c552d1000 msgr2=0x558c552d7600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 80 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c552d1000 msgr2=0x558c552d7600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c552d1000 0x558c552d7600 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c552d1000 0x558c552d7600 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:49.259+0000 7f8f0e996700 20 osd.7 15 heartbeat_reset con 0x558c552d1000 s 0x558c54c0a340 2019-12-07T19:33:49.259+0000 7f8f0e996700 10 osd.7 15 heartbeat_reset closing (old) failed hb con 0x558c552d1000 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c552d0c00 msgr2=0x558c552d7080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 75 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c552d0c00 msgr2=0x558c552d7080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c552d0c00 0x558c552d7080 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c552d0c00 0x558c552d7080 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c64c00 0x558c54c70c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c64c00 0x558c54c70c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c65000 0x558c54c71180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c65000 0x558c54c71180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:49.259+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication new session 0x558c54c0b860 con 0x558c54c64c00 entity osd.4 addr 2019-12-07T19:33:49.259+0000 7f8f0d994700 20 osd.7 15 heartbeat_reset con 0x558c552d0c00 s 0x558c552bdd40 2019-12-07T19:33:49.259+0000 7f8f0d994700 10 osd.7 15 heartbeat_reset closing (old) failed hb con 0x558c552d0c00 2019-12-07T19:33:49.259+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication session 0x558c54c0b860 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:49.259+0000 7f8f24c58700 0 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c64c00 msgr2=0x558c54c70c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c64c00 msgr2=0x558c54c70c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 75 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c64c00 msgr2=0x558c54c70c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c64c00 0x558c54c70c00 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c64c00 0x558c54c70c00 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:49.259+0000 7f8f0d994700 20 osd.7 15 heartbeat_reset con 0x558c54c64c00 s 0x558c54c0b860 2019-12-07T19:33:49.259+0000 7f8f0d994700 10 osd.7 15 heartbeat_reset closing (old) failed hb con 0x558c54c64c00 2019-12-07T19:33:49.259+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication new session 0x558c54c0ba00 con 0x558c54c65000 entity osd.4 addr 2019-12-07T19:33:49.259+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication session 0x558c54c0ba00 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:49.259+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65000 0x558c54c71180 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:49.294+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.344+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.394+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.444+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.460+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c65400 0x558c54c6f080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:49.460+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c65400 0x558c54c6f080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:49.460+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication new session 0x558c54c5d1e0 con 0x558c54c65400 entity osd.4 addr 2019-12-07T19:33:49.460+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication session 0x558c54c5d1e0 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:49.460+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65400 0x558c54c6f080 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:49.492+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:49.492+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:49.492+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:49.492+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:49.494+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.544+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.594+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.645+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:48.872295+0000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:48.872410+0000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 20 osd.7 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 20 osd.7 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 20 osd.7 15 scrub_load_below_threshold loadavg per cpu 0.06125 < max 5 = yes 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub load_is_low=1 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub done 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:49.645+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:49.695+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.745+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.795+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.845+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.851+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.309014358s delta_ub 0.022373322s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c54c18000 2019-12-07T19:33:49.851+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.309014358s delta_ub 0.022373322s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552d1400 2019-12-07T19:33:49.851+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021825340s,0.022373322s]) 2019-12-07T19:33:49.851+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.287189018s delta_ub 18446744073.687726276s) v5 -- 0x558c54c1ee00 con 0x558c54c18000 2019-12-07T19:33:49.851+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021739998s,0.022373322s]) 2019-12-07T19:33:49.851+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.287274360s delta_ub 18446744073.687811618s) v5 -- 0x558c55270200 con 0x558c552d1400 2019-12-07T19:33:49.895+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.941+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.408188910s delta_ub 0.031199889s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c552d0800 2019-12-07T19:33:49.941+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030616446s,0.031199889s]) 2019-12-07T19:33:49.941+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.377572464s delta_ub 18446744073.678935170s) v5 -- 0x558c54c1ee00 con 0x558c552d0800 2019-12-07T19:33:49.941+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.408188910s delta_ub 0.031199889s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d0000 2019-12-07T19:33:49.941+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030592740s,0.031199889s]) 2019-12-07T19:33:49.941+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.377596170s delta_ub 18446744073.678958876s) v5 -- 0x558c55270200 con 0x558c552d0000 2019-12-07T19:33:49.945+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.995+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.045+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.095+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.145+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.196+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.246+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.296+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.346+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.396+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.446+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.496+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.497+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:50.497+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:50.497+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:50.497+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:50.521+0000 7f8f1219d700 20 osd.7 15 reports for 0 queries 2019-12-07T19:33:50.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c5489ee00 con 0x558c5476d800 2019-12-07T19:33:50.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558c5489e380 con 0x558c5476d800 2019-12-07T19:33:50.547+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.597+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:48.872295+0000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:48.872410+0000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 20 osd.7 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 20 osd.7 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 20 osd.7 15 scrub_load_below_threshold loadavg per cpu 0.06125 < max 5 = yes 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub load_is_low=1 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub done 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:50.610+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:50.647+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.650+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.107717608s delta_ub 0.021796413s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c19000 2019-12-07T19:33:50.650+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021221981s,0.021796413s]) 2019-12-07T19:33:50.650+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.086495627s delta_ub 18446744073.688329635s) v5 -- 0x558c54c1ee00 con 0x558c54c19000 2019-12-07T19:33:50.650+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.107717608s delta_ub 0.021796413s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c54c18800 2019-12-07T19:33:50.651+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021069220s,0.021796413s]) 2019-12-07T19:33:50.651+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.086648388s delta_ub 18446744073.688482396s) v5 -- 0x558c54c1ee00 con 0x558c54c18800 2019-12-07T19:33:50.697+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.747+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.797+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.847+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.897+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.948+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.998+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.048+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.098+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.148+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.153+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.019564954s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c18400 2019-12-07T19:33:51.153+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018998192s,0.019564954s]) 2019-12-07T19:33:51.153+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.588694208s delta_ub 18446744073.690553424s) v5 -- 0x558c54c1ee00 con 0x558c54c18400 2019-12-07T19:33:51.153+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.019564954s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c54c18c00 2019-12-07T19:33:51.153+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018753986s,0.019564954s]) 2019-12-07T19:33:51.153+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.588938414s delta_ub 18446744073.690797630s) v5 -- 0x558c54c1ee00 con 0x558c54c18c00 2019-12-07T19:33:51.198+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.236+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.708091194s delta_ub 0.036624682s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aee00 con 0x558c552d1800 2019-12-07T19:33:51.236+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036083741s,0.036624682s]) 2019-12-07T19:33:51.236+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.672007453s delta_ub 18446744073.673467875s) v5 -- 0x558c551afe00 con 0x558c552d1800 2019-12-07T19:33:51.236+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.708091194s delta_ub 0.036624682s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552d1c00 2019-12-07T19:33:51.236+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036041156s,0.036624682s]) 2019-12-07T19:33:51.236+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.672050038s delta_ub 18446744073.673510460s) v5 -- 0x558c55270200 con 0x558c552d1c00 2019-12-07T19:33:51.248+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.298+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.348+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.398+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.449+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.449+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:51.449+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:51.449+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:51.449+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:51.499+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.549+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:48.872295+0000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:48.872410+0000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:51.564+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:51.599+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258772 2019-12-07T19:33:51.599+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:51.599+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.649+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.699+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.749+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.798+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.848+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.852+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.6 172.21.15.109:0/36937 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.019412398s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c53ae4c00 2019-12-07T19:33:51.852+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018833921s,0.019412398s]) 2019-12-07T19:33:51.852+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.289554355s delta_ub 18446744073.690717695s) v5 -- 0x558c55270200 con 0x558c53ae4c00 2019-12-07T19:33:51.852+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.6 172.21.15.109:0/36937 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.019412398s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c552d0400 2019-12-07T19:33:51.852+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018795425s,0.019412398s]) 2019-12-07T19:33:51.852+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.289592851s delta_ub 18446744073.690756191s) v5 -- 0x558c54c1ee00 con 0x558c552d0400 2019-12-07T19:33:51.898+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.949+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.999+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.049+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.099+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.149+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.199+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.249+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.299+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.335+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.808438677s delta_ub 0.036498762s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552d1c00 2019-12-07T19:33:52.335+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036043735s,0.036498762s]) 2019-12-07T19:33:52.335+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.772394942s delta_ub 18446744073.673507881s) v5 -- 0x558c55270200 con 0x558c552d1c00 2019-12-07T19:33:52.335+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.808438677s delta_ub 0.036498762s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5fe00 con 0x558c552d1800 2019-12-07T19:33:52.335+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.035971262s,0.036498762s]) 2019-12-07T19:33:52.335+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.772467415s delta_ub 18446744073.673580354s) v5 -- 0x558c551afe00 con 0x558c552d1800 2019-12-07T19:33:52.349+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.399+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.450+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.482+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:52.482+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:52.482+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:52.482+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:52.500+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.550+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:48.872295+0000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:48.872410+0000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:52.572+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:52.600+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.650+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.700+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.750+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.750+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 6 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.209356665s delta_ub 0.022303619s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270c00 con 0x558c54c18000 2019-12-07T19:33:52.750+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021873233s,0.022303619s]) 2019-12-07T19:33:52.750+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.187483432s delta_ub 18446744073.687678383s) v5 -- 0x558c54c1ee00 con 0x558c54c18000 2019-12-07T19:33:52.750+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 6 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.209356665s delta_ub 0.022303619s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d1400 2019-12-07T19:33:52.750+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021771542s,0.022303619s]) 2019-12-07T19:33:52.750+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.187585123s delta_ub 18446744073.687780074s) v5 -- 0x558c55270200 con 0x558c552d1400 2019-12-07T19:33:52.800+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.841+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.308647276s delta_ub 0.031118449s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c552d0800 2019-12-07T19:33:52.841+0000 7f8f23c56700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030641049s,0.031118449s]) 2019-12-07T19:33:52.841+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.278006227s delta_ub 18446744073.678910567s) v5 -- 0x558c54c1ee00 con 0x558c552d0800 2019-12-07T19:33:52.841+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.308647276s delta_ub 0.031118449s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552d0000 2019-12-07T19:33:52.841+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030622463s,0.031118449s]) 2019-12-07T19:33:52.841+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.278024813s delta_ub 18446744073.678929153s) v5 -- 0x558c55270200 con 0x558c552d0000 2019-12-07T19:33:52.841+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0800 msgr2=0x558c552d6b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 74 2019-12-07T19:33:52.841+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0800 msgr2=0x558c552d6b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:52.841+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0800 0x558c552d6b00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:52.841+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0800 0x558c552d6b00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:52.841+0000 7f8f0d994700 20 osd.7 15 heartbeat_reset con 0x558c552d0800 s 0x558c54c0a000 2019-12-07T19:33:52.841+0000 7f8f0d994700 10 osd.7 15 heartbeat_reset closing (old) failed hb con 0x558c552d0800 2019-12-07T19:33:52.841+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0000 msgr2=0x558c552d6000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 67 2019-12-07T19:33:52.841+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0000 msgr2=0x558c552d6000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:52.841+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0000 0x558c552d6000 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:52.841+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36936 conn(0x558c552d0000 0x558c552d6000 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:52.841+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c5ac00 0x558c54c71700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:52.841+0000 7f8f0e996700 20 osd.7 15 heartbeat_reset con 0x558c552d0000 s 0x558c552bda00 2019-12-07T19:33:52.841+0000 7f8f0e996700 10 osd.7 15 heartbeat_reset closing (old) failed hb con 0x558c552d0000 2019-12-07T19:33:52.841+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a800 0x558c54c38c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:52.841+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c54c5ac00 0x558c54c71700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:52.841+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a800 0x558c54c38c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:52.841+0000 7f8f23c56700 10 osd.7 15 ms_handle_authentication new session 0x558c54c5cd00 con 0x558c54c5ac00 entity osd.5 addr 2019-12-07T19:33:52.841+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication new session 0x558c54c5d380 con 0x558c54c5a800 entity osd.5 addr 2019-12-07T19:33:52.841+0000 7f8f23c56700 10 osd.7 15 ms_handle_authentication session 0x558c54c5cd00 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:52.841+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication session 0x558c54c5d380 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:52.841+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a800 msgr2=0x558c54c38c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:52.841+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a800 msgr2=0x558c54c38c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:52.841+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a800 0x558c54c38c00 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:52.841+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a800 0x558c54c38c00 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:52.842+0000 7f8f0e996700 20 osd.7 15 heartbeat_reset con 0x558c54c5a800 s 0x558c54c5d380 2019-12-07T19:33:52.842+0000 7f8f0e996700 10 osd.7 15 heartbeat_reset closing (old) failed hb con 0x558c54c5a800 2019-12-07T19:33:52.842+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36936 conn(0x558c54c5ac00 0x558c54c71700 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:52.851+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.901+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.951+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.001+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.043+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a400 0x558c54c37b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:53.043+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c54c5a400 0x558c54c37b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:53.043+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication new session 0x558c54c0b860 con 0x558c54c5a400 entity osd.5 addr 2019-12-07T19:33:53.043+0000 7f8f24c58700 10 osd.7 15 ms_handle_authentication session 0x558c54c0b860 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:53.043+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36936 conn(0x558c54c5a400 0x558c54c37b80 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:53.051+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.101+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.151+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.202+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.252+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.302+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.352+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.384+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c5a000 0x558c54c37600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:53.385+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c5a000 0x558c54c37600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:53.385+0000 7f8f23c56700 10 osd.7 15 ms_handle_authentication new session 0x558c54c5c4e0 con 0x558c54c5a000 entity client.admin addr 2019-12-07T19:33:53.385+0000 7f8f23c56700 10 osd.7 15 ms_handle_authentication session 0x558c54c5c4e0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:53.385+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/481722811 conn(0x558c54c5a000 0x558c54c37600 crc :-1 s=READY pgs=14 cs=0 l=1 rx=0 tx=0).ready entity=client.4295 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:53.385+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4295 172.21.15.131:0/481722811 1 ==== command(tid 15: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c53a71860 con 0x558c54c5a000 2019-12-07T19:33:53.385+0000 7f8f164bb700 20 osd.7 15 OSD::ms_dispatch: command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:53.385+0000 7f8f164bb700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:53.385+0000 7f8f164bb700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:53.385+0000 7f8f164bb700 20 osd.7 15 _dispatch 0x558c53a71860 command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:53.385+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/481722811 -- command_reply(tid 15: 0 ) v1 -- 0x558c53a6ee00 con 0x558c54c5a000 2019-12-07T19:33:53.402+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.409+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4295 172.21.15.131:0/481722811 2 ==== command(tid 16: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c53a71c20 con 0x558c54c5a000 2019-12-07T19:33:53.409+0000 7f8f164bb700 20 osd.7 15 OSD::ms_dispatch: command(tid 16: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:53.409+0000 7f8f164bb700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:53.409+0000 7f8f164bb700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:53.409+0000 7f8f164bb700 20 osd.7 15 _dispatch 0x558c53a71c20 command(tid 16: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:53.411+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/481722811 -- command_reply(tid 16: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c53a6ec40 con 0x558c54c5a000 2019-12-07T19:33:53.413+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/481722811 conn(0x558c54c5a000 msgr2=0x558c54c37600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:53.413+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/481722811 conn(0x558c54c5a000 msgr2=0x558c54c37600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:53.413+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/481722811 conn(0x558c54c5a000 0x558c54c37600 crc :-1 s=READY pgs=14 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:53.413+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/481722811 conn(0x558c54c5a000 0x558c54c37600 crc :-1 s=READY pgs=14 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:53.413+0000 7f8f164bb700 2 osd.7 15 ms_handle_reset con 0x558c54c5a000 session 0x558c54c5c4e0 2019-12-07T19:33:53.445+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:53.445+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:53.445+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:53.445+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:53.452+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.502+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.552+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:48.872295+0000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:48.872410+0000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 20 osd.7 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:53.556+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:53.602+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.652+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.703+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.752+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.803+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.853+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.903+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.953+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.003+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.053+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.104+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.154+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.204+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.254+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.304+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.354+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.404+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.407+0000 7f8f21451700 10 osd.7 15 tick 2019-12-07T19:33:54.407+0000 7f8f21451700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:54.407+0000 7f8f21451700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:54.407+0000 7f8f21451700 20 osd.7 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:54.454+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.505+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.555+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 10 osd.7 15 tick_without_osd_lock 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:48.872295+0000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 25 osd.7 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:48.871890+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:48.872410+0000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 20 osd.7 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 20 osd.7 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 20 osd.7 15 scrub_load_below_threshold loadavg per cpu 0.05625 < max 5 = yes 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub load_is_low=1 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 20 osd.7 15 sched_scrub done 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 20 osd.7 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:54.560+0000 7f8f1fbbe700 10 osd.7 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:54.605+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:54.605+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.635+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 5 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.108739871s delta_ub 0.036505120s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5fe00 con 0x558c552d1800 2019-12-07T19:33:54.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 5 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.108739871s delta_ub 0.036505120s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552d1c00 2019-12-07T19:33:54.636+0000 7f8f24c58700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036081458s,0.036505120s]) 2019-12-07T19:33:54.636+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.072658413s delta_ub 18446744073.673470158s) v5 -- 0x558c551afc00 con 0x558c552d1800 2019-12-07T19:33:54.636+0000 7f8f24c58700 10 osd.7 15 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.636+0000 7f8f24457700 20 osd.7 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036051838s,0.036505120s]) 2019-12-07T19:33:54.636+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.072688033s delta_ub 18446744073.673499778s) v5 -- 0x558c53aa8c00 con 0x558c552d1c00 2019-12-07T19:33:54.636+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 2 ==== osd_map(16..17 src has 1..17) v4 ==== 1760+0+0 (crc 0 0 0) 0x558c55275200 con 0x558c54c19800 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 osd.7 15 OSD::ms_dispatch: osd_map(16..17 src has 1..17) v4 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 osd.7 15 do_waiters -- start 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 osd.7 15 do_waiters -- finish 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 osd.7 15 _dispatch 0x558c55275200 osd_map(16..17 src has 1..17) v4 2019-12-07T19:33:54.636+0000 7f8f1199c700 3 osd.7 15 handle_osd_map epochs [16,17], i have 15, src has [1,17] 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 osd.7 15 handle_osd_map got inc map for epoch 16 2019-12-07T19:33:54.636+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3630e826:::osdmap.15:0# 0x0~0 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552788c0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55279f10 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279ea0 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558c55279e30 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3630e826:::osdmap.15:0# 0x0~1168 = 4456 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 osd.7 15 add_map_bl 15 4456 bytes 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 osd.7 15 got_full_map 16, nothing requested 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 osd.7 15 handle_osd_map got inc map for epoch 17 2019-12-07T19:33:54.636+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3cb0e826:::osdmap.16:0# 0x0~0 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3cb0e826:::osdmap.16:0# key 0x7f7fffffffffffffff3cb0e8'&!osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3cb0e826:::osdmap.16:0# 0x0~0 = -2 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 osd.7 15 got_full_map 17, nothing requested 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 osd.7 15 handle_osd_map pg_num_history pg_num_history(e17 pg_nums {} deleted_pools ) 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {16={},17={}} 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 osd.7 15 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e17 [1,17] lci=[13,17]) 2019-12-07T19:33:54.636+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54836f00 seq 9 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c423972f:::inc_osdmap.16:0# key 0x7f7fffffffffffffffc42397'/!inc_osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:54.636+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1059 2019-12-07T19:33:54.636+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~1b9 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xe47 on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54836f00 1 blobs 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x25000~1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x25000~1000] 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55264a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279f80 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264a10 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55279f80) get_ref 0x0~1b9 Blob(0x558c55279f80 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264a10 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55279f80) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1b9: 0x0~1b9 Blob(0x558c55279f80 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1b9) SharedBlob(0x558c55264a10 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55264a28 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1b9 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6108) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3cb0e826:::osdmap.16:0# key 0x7f7fffffffffffffff3cb0e8'&!osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3cb0e826:::osdmap.16:0# 0x0~1235 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1060 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3cb0e826:::osdmap.16:0# 0x0~1235 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~235 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdcb on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54836f00 2 blobs 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x26000~2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x26000~2000] 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55264b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264af0 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264b60 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264af0) get_ref 0x0~1000 Blob(0x558c55264af0 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264b60 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264af0) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55264af0 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55264b60 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55264b78 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55264c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264bd0 blob([!~1000,0x27000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264c40 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264bd0) get_ref 0x1000~235 Blob(0x558c55264bd0 blob([!~1000,0x27000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c55264c40 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~235: 0x1000~235 Blob(0x558c55264bd0 blob([!~1000,0x27000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,235]) SharedBlob(0x558c55264c40 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55264c58 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1235 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6388) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3cb0e826:::osdmap.16:0# 0x0~1235 = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cfa3972f:::inc_osdmap.17:0# key 0x7f7fffffffffffffffcfa397'/!inc_osdmap.17!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1061 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~4f3 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb0d on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54836f00 1 blobs 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x28000~1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x28000~1000] 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55264e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264d90 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264e00 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264d90) get_ref 0x0~4f3 Blob(0x558c55264d90 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264e00 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264d90) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~4f3: 0x0~4f3 Blob(0x558c55264d90 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4f3) SharedBlob(0x558c55264e00 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55264e18 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4f3 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6608) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3340e826:::osdmap.17:0# key 0x7f7fffffffffffffff3340e8'&!osdmap.17!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3340e826:::osdmap.17:0# 0x0~1292 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1062 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3340e826:::osdmap.17:0# 0x0~1292 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~292 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd6e on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54836f00 2 blobs 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x29000~2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x29000~2000] 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55264f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264ee0 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264f50 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264ee0) get_ref 0x0~1000 Blob(0x558c55264ee0 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264f50 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264ee0) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55264ee0 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55264f50 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55264f68 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55265030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264fc0 blob([!~1000,0x2a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265030 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264fc0) get_ref 0x1000~292 Blob(0x558c55264fc0 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c55265030 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c55264fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~292: 0x1000~292 Blob(0x558c55264fc0 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,292]) SharedBlob(0x558c55265030 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55265048 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1292 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6888) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3340e826:::osdmap.17:0# 0x0~1292 = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~12 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54836f00 0 blobs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54836f00 0 blobs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 40864 (7 ios * 4000 + 12864 bytes) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes 0x558c53a91900,0x558c54ce6000,0x558c54ce6280,0x558c54ce6500,0x558c54ce6780,0x558c55276c80 shared_blobs 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6108) update #-1:c423972f:::inc_osdmap.16:0# 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c423972f:::inc_osdmap.16:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6388) update #-1:3cb0e826:::osdmap.16:0# 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6388) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3cb0e826:::osdmap.16:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6608) update #-1:cfa3972f:::inc_osdmap.17:0# 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cfa3972f:::inc_osdmap.17:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6888) update #-1:3340e826:::osdmap.17:0# 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce6888) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3340e826:::osdmap.17:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[25000~6000] released 0x[] 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 freelist allocate 0x25000~6000 2019-12-07T19:33:54.637+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:33:54.637+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54836f00 2019-12-07T19:33:54.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 aio_wait 2019-12-07T19:33:54.637+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:33:54.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9014 < min_flush_size 524288 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce6000 had 1 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce6280 had 1 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce6500 had 1 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce6780 had 1 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9061 < min_flush_size 524288 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9061 < min_flush_size 524288 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xa528~2365 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xa528 0xa528~2365 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xa528 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x528 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x88d and padding block with 0x773 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc88d 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:54.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000232131s (0.000056200s flush + 0.000175931s kv commit) 2019-12-07T19:33:54.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54836f00> 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c5525e640> 2019-12-07T19:33:54.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54836f00 osr 0x558c54ea4b40 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7 15 _committed_osd_maps 16..17 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 7 0x1b000~1000 crc 3e33509d 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7 15 advance to epoch 16 (<= last 17 <= newest_map 17) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 7 0x2000~1000 crc 56a7ddd8 2019-12-07T19:33:54.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 deferred_cleanup 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7 16 advance to epoch 17 (<= last 17 <= newest_map 17) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6900 onodes 0x558c53a91900,0x558c55276c80,0x558c55277400,0x558c55277680 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279ac8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9d40 space 0x558c55279ac8 0x0~1000 writing nocache) 2019-12-07T19:33:54.637+0000 7f8f141a1700 7 osd.7 17 consume_map version 17 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279c18 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9dd0 space 0x558c55279c18 0x0~1000 writing nocache) 2019-12-07T19:33:54.637+0000 7f8f141a1700 20 osd.7 17 prune_pg_created 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:0.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:1.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279cf8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9e60 space 0x558c55279cf8 0x1000~1000 writing nocache) 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:2.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:3.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:4.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:5.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 done 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:6.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 deferred_cleanup 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7:7.consume_map 17 (was 15) 2019-12-07T19:33:54.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6900 [] 2019-12-07T19:33:54.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:54.637+0000 7f8f141a1700 10 osd.7 17 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:54.637+0000 7f8f141a1700 7 osd.7 17 activate_map version 17 2019-12-07T19:33:54.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 deferred_cleanup 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 0x558c53a91900,0x558c55276c80,0x558c55277900,0x558c55277b80 2019-12-07T19:33:54.637+0000 7f8f0918b700 20 osd.7 17 agent_entry empty queue 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54ea3748 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a090 space 0x558c54ea3748 0x0~1000 writing nocache) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279e48 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a1b0 space 0x558c55279e48 0x1000~1000 writing nocache) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279f28 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a120 space 0x558c55279f28 0x0~1000 writing nocache) 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 deferred_queued 2019-12-07T19:33:54.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:33:54.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:54.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:54.655+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.705+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.750+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.208013836s delta_ub 0.021654997s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c54c18800 2019-12-07T19:33:54.750+0000 7f8f23c56700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021306752s,0.021654997s]) 2019-12-07T19:33:54.750+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.186707084s delta_ub 18446744073.688244864s) v5 -- 0x558c54c58c00 con 0x558c54c18800 2019-12-07T19:33:54.750+0000 7f8f23c56700 10 osd.7 17 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.750+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.208013836s delta_ub 0.021654997s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c54c19000 2019-12-07T19:33:54.750+0000 7f8f23c56700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021031125s,0.021654997s]) 2019-12-07T19:33:54.750+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.186982711s delta_ub 18446744073.688520491s) v5 -- 0x558c54c58c00 con 0x558c54c19000 2019-12-07T19:33:54.755+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.771+0000 7f8f0097a700 5 osd.7 17 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,6] op hist []) 2019-12-07T19:33:54.771+0000 7f8f0097a700 20 osd.7 17 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690797630s) v5 -- 0x558c55271000 con 0x558c5476d400 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690797630s) v5 -- 0x558c55271200 con 0x558c551b6800 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.688520491s) v5 -- 0x558c55271400 con 0x558c551b6400 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.688520491s) v5 -- 0x558c55271600 con 0x558c551b6000 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.673499778s) v5 -- 0x558c55271800 con 0x558c551b7800 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.673499778s) v5 -- 0x558c55271a00 con 0x558c551b7c00 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.687780074s) v5 -- 0x558c55271c00 con 0x558c552c2000 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.687780074s) v5 -- 0x558c55271e00 con 0x558c552c2400 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.696001641s) v5 -- 0x558c55250000 con 0x558c552c2800 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.696001641s) v5 -- 0x558c55250200 con 0x558c552c2c00 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.678929153s) v5 -- 0x558c55250400 con 0x558c552c3000 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.678929153s) v5 -- 0x558c55250600 con 0x558c552c3400 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690756191s) v5 -- 0x558c55250800 con 0x558c551b7000 2019-12-07T19:33:54.771+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690756191s) v5 -- 0x558c55250a00 con 0x558c551b7400 2019-12-07T19:33:54.771+0000 7f8f23c56700 0 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 msgr2=0x558c552b5180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:54.771+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.221582178s delta_ub 0.013933015s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c552c2800 2019-12-07T19:33:54.771+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.221613508s delta_ub 0.013964345s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552c2c00 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.238696843s delta_ub 0.031047680s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c552c3000 2019-12-07T19:33:54.771+0000 7f8f24457700 25 osd.7 17 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 last_rx_front 2019-12-07T19:33:48.872295+0000 -> 2019-12-07T19:33:54.772448+0000 2019-12-07T19:33:54.771+0000 7f8f24457700 10 osd.7 17 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.771+0000 7f8f24457700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013726184s,0.013964345s]) 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872169+0000 -> 2019-12-07T19:33:54.772464+0000 last_rx_front 2019-12-07T19:33:54.772448+0000 2019-12-07T19:33:54.771+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6820/36937 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.226899905s delta_ub 0.019250742s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c551b7400 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:54.772210+0000) and older pending ping(s) 2019-12-07T19:33:54.771+0000 7f8f24c58700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013678241s,0.013933015s]) 2019-12-07T19:33:54.771+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.238753176s delta_ub 0.031104013s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c552c3400 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872268+0000 -> 2019-12-07T19:33:54.772502+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 2019-12-07T19:33:54.771+0000 7f8f23c56700 10 osd.7 17 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.771+0000 7f8f23c56700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030755640s,0.031047680s]) 2019-12-07T19:33:54.771+0000 7f8f24457700 25 osd.7 17 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 last_rx_front 2019-12-07T19:33:48.872410+0000 -> 2019-12-07T19:33:54.772519+0000 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 msgr2=0x558c552b5180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 70 2019-12-07T19:33:54.771+0000 7f8f24457700 10 osd.7 17 maybe_share_map con 0x558c54c19400 v2:172.21.15.109:6818/36937 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.771+0000 7f8f24457700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018941358s,0.019250742s]) 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 msgr2=0x558c552b5180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772502+0000 last_rx_front 2019-12-07T19:33:48.872359+0000 -> 2019-12-07T19:33:54.772537+0000 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:54.772210+0000) and older pending ping(s) 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 0x558c552b5180 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:54.771+0000 7f8f24c58700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030776427s,0.031104013s]) 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c552c3000 0x558c552b5180 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:54.771+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.229391936s delta_ub 0.021742773s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c551b6400 2019-12-07T19:33:54.771+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.227192309s delta_ub 0.019543146s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c551b6800 2019-12-07T19:33:54.771+0000 7f8f24457700 25 osd.7 17 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872373+0000 -> 2019-12-07T19:33:54.772556+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 2019-12-07T19:33:54.771+0000 7f8f24457700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021395976s,0.021742773s]) 2019-12-07T19:33:54.771+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.230031738s delta_ub 0.022382575s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552c2000 2019-12-07T19:33:54.771+0000 7f8f0f998700 20 osd.7 17 heartbeat_reset con 0x558c552c3000 s 0x558c552bcd00 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.227158369s delta_ub 0.019509206s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c5476d400 2019-12-07T19:33:54.771+0000 7f8f0f998700 10 osd.7 17 heartbeat_reset failed hb con 0x558c552c3000 for osd.5, reopening 2019-12-07T19:33:54.771+0000 7f8f0f998700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c552c3400 msgr2=0x558c552b5700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:54.771+0000 7f8f0f998700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c552c3400 0x558c552b5700 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:54.771+0000 7f8f0f998700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c54c65800 0x558c54d02000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:54.771+0000 7f8f0f998700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c54c65c00 0x558c54d02580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 last_rx_front 2019-12-07T19:33:48.872282+0000 -> 2019-12-07T19:33:54.772626+0000 2019-12-07T19:33:54.771+0000 7f8f24c58700 10 osd.7 17 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.771+0000 7f8f24c58700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019126208s,0.019543146s]) 2019-12-07T19:33:54.771+0000 7f8f24457700 25 osd.7 17 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872247+0000 -> 2019-12-07T19:33:54.772638+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 2019-12-07T19:33:54.771+0000 7f8f24457700 10 osd.7 17 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.771+0000 7f8f24457700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021954060s,0.022382575s]) 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872348+0000 -> 2019-12-07T19:33:54.772649+0000 last_rx_front 2019-12-07T19:33:54.772626+0000 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:54.772210+0000) and older pending ping(s) 2019-12-07T19:33:54.771+0000 7f8f23c56700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019070199s,0.019509206s]) 2019-12-07T19:33:54.771+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.244231582s delta_ub 0.036582419s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5fe00 con 0x558c551b7c00 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 last_rx_front 2019-12-07T19:33:48.872215+0000 -> 2019-12-07T19:33:54.772667+0000 2019-12-07T19:33:54.771+0000 7f8f24c58700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036125111s,0.036582419s]) 2019-12-07T19:33:54.771+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6822/36937 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.226971768s delta_ub 0.019322605s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5fe00 con 0x558c551b7000 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.244210345s delta_ub 0.036561182s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c551b7800 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872337+0000 -> 2019-12-07T19:33:54.772683+0000 last_rx_front 2019-12-07T19:33:54.772519+0000 2019-12-07T19:33:54.771+0000 7f8f24c58700 25 osd.7 17 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:54.772210+0000) and older pending ping(s) 2019-12-07T19:33:54.771+0000 7f8f24c58700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018848639s,0.019322605s]) 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:48.872429+0000 -> 2019-12-07T19:33:54.772696+0000 last_rx_front 2019-12-07T19:33:54.772667+0000 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:54.772210+0000) and older pending ping(s) 2019-12-07T19:33:54.771+0000 7f8f23c56700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036074596s,0.036561182s]) 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.230012856s delta_ub 0.022363693s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c552c2400 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772638+0000 last_rx_front 2019-12-07T19:33:48.872321+0000 -> 2019-12-07T19:33:54.772726+0000 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:54.772210+0000) and older pending ping(s) 2019-12-07T19:33:54.771+0000 7f8f23c56700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021846865s,0.022363693s]) 2019-12-07T19:33:54.771+0000 7f8f23c56700 0 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 msgr2=0x558c552b2000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 4 ==== osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.229463653s delta_ub 0.021814490s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c551b6000 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772556+0000 last_rx_front 2019-12-07T19:33:48.872387+0000 -> 2019-12-07T19:33:54.772753+0000 2019-12-07T19:33:54.771+0000 7f8f23c56700 25 osd.7 17 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:54.772210+0000) and older pending ping(s) 2019-12-07T19:33:54.771+0000 7f8f23c56700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021270630s,0.021814490s]) 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 msgr2=0x558c552b2000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 65 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 msgr2=0x558c552b2000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 0x558c552b2000 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c551b6000 0x558c552b2000 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:54.771+0000 7f8f1099a700 20 osd.7 17 heartbeat_reset con 0x558c551b6000 s 0x558c53a85d40 2019-12-07T19:33:54.771+0000 7f8f1099a700 10 osd.7 17 heartbeat_reset failed hb con 0x558c551b6000 for osd.1, reopening 2019-12-07T19:33:54.771+0000 7f8f1099a700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c551b6400 msgr2=0x558c53aa5b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:54.771+0000 7f8f1099a700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c551b6400 0x558c53aa5b80 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:54.771+0000 7f8f1099a700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c54c9e000 0x558c54d03080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:54.771+0000 7f8f1099a700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c54c9e400 0x558c54d03600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c54c65800 0x558c54d02000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c54c65c00 0x558c54d02580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:54.771+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c54c9e400 0x558c54d03600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558c54c65800 0x558c54d02000 crc :-1 s=READY pgs=9 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c54c9e000 0x558c54d03080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:54.771+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558c54c65c00 0x558c54d02580 crc :-1 s=READY pgs=9 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:54.772+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558c54c9e400 0x558c54d03600 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:54.772+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558c54c9e000 0x558c54d03080 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:54.805+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.855+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.905+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.955+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.005+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.056+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.106+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.156+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.158+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.608678189s delta_ub 0.013964345s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c54c65400 2019-12-07T19:33:55.158+0000 7f8f24c58700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013618863s,0.013964345s]) 2019-12-07T19:33:55.158+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.595059326s delta_ub 18446744073.695932753s) v5 -- 0x558c55250600 con 0x558c54c65400 2019-12-07T19:33:55.158+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.608678189s delta_ub 0.013964345s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53a5fe00 con 0x558c54c65000 2019-12-07T19:33:55.158+0000 7f8f24c58700 20 osd.7 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013547537s,0.013964345s]) 2019-12-07T19:33:55.158+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.595130652s delta_ub 18446744073.696004079s) v5 -- 0x558c55250600 con 0x558c54c65000 2019-12-07T19:33:55.206+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.256+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.306+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.356+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.380+0000 7f8f21451700 10 osd.7 17 tick 2019-12-07T19:33:55.380+0000 7f8f21451700 10 osd.7 17 do_waiters -- start 2019-12-07T19:33:55.380+0000 7f8f21451700 10 osd.7 17 do_waiters -- finish 2019-12-07T19:33:55.380+0000 7f8f21451700 20 osd.7 17 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:55.406+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.456+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.506+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.520+0000 7f8f1219d700 20 osd.7 17 reports for 0 queries 2019-12-07T19:33:55.520+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c5489e700 con 0x558c5476d800 2019-12-07T19:33:55.520+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558c5489f500 con 0x558c5476d800 2019-12-07T19:33:55.557+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 10 osd.7 17 tick_without_osd_lock 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffd5000/0x40000000/0x1680000000, data 0xc3f9/0x29000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 25 osd.7 17 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772649+0000 last_rx_front 2019-12-07T19:33:54.772626+0000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 25 osd.7 17 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772556+0000 last_rx_front 2019-12-07T19:33:54.772753+0000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 25 osd.7 17 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772696+0000 last_rx_front 2019-12-07T19:33:54.772667+0000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 25 osd.7 17 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772638+0000 last_rx_front 2019-12-07T19:33:54.772726+0000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 25 osd.7 17 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772464+0000 last_rx_front 2019-12-07T19:33:54.772448+0000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 25 osd.7 17 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772502+0000 last_rx_front 2019-12-07T19:33:54.772537+0000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 25 osd.7 17 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772683+0000 last_rx_front 2019-12-07T19:33:54.772519+0000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 20 osd.7 17 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 20 osd.7 17 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 20 osd.7 17 scrub_load_below_threshold loadavg per cpu 0.05625 < max 5 = yes 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 20 osd.7 17 sched_scrub load_is_low=1 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 20 osd.7 17 sched_scrub done 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 10 osd.7 17 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 20 osd.7 17 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:55.560+0000 7f8f1fbbe700 10 osd.7 17 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:55.607+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.609+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 13 ==== pg_create2(e18 {1.4s0=18,2019-12-07T19:33:55.603918+0000}) v2 ==== 207+0+0 (crc 0 0 0) 0x558c5524c000 con 0x558c5476d000 2019-12-07T19:33:55.609+0000 7f8f23c56700 7 osd.7 17 handle_fast_pg_create pg_create2(e18 {1.4s0=18,2019-12-07T19:33:55.603918+0000}) v2 from mon.0 2019-12-07T19:33:55.609+0000 7f8f23c56700 20 osd.7 17 handle_fast_pg_create 1.4s0 e18@2019-12-07T19:33:55.603918+0000 history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 pi ([0,0] all_participants= intervals=) 2019-12-07T19:33:55.609+0000 7f8f23c56700 15 osd.7 17 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 NullEvt +create_info 2019-12-07T19:33:55.609+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 map 18 > 17 2019-12-07T19:33:55.609+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 18, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.614+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (crc 0 0 0) 0x558c5524c900 con 0x558c54c19800 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 osd.7 17 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 osd.7 17 do_waiters -- start 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 osd.7 17 do_waiters -- finish 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 osd.7 17 _dispatch 0x558c5524c900 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.614+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 4 ==== pg_query2(1.ds2 1.ds2 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54ce6a00 con 0x558c54c19800 2019-12-07T19:33:55.614+0000 7f8f24c58700 15 osd.7 17 enqueue_peering_evt 1.ds2 epoch_sent: 18 epoch_requested: 18 MQuery 1.ds2 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.614+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds2 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.614+0000 7f8f1199c700 3 osd.7 17 handle_osd_map epochs [18,18], i have 17, src has [1,18] 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 osd.7 17 handle_osd_map got inc map for epoch 18 2019-12-07T19:33:55.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds2 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds2 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3340e826:::osdmap.17:0# 0x0~0 2019-12-07T19:33:55.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 map 18 > 17 2019-12-07T19:33:55.614+0000 7f8f06185700 20 osd.7 op_wq _add_slot_waiter 1.ds2 peering, item epoch is 18, will wait on OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds2 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1292 size 0x1292 (4754) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:55.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55264ee0 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55264f50 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55264fc0 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,292]) SharedBlob(0x558c55265030 sbid 0x0)) need 0x1000~292 cache has 0x[1000~292] 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3340e826:::osdmap.17:0# 0x0~1292 = 4754 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 osd.7 17 add_map_bl 17 4754 bytes 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 osd.7 17 got_full_map 18, nothing requested 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 osd.7 17 handle_osd_map recording new pool 1 pg_num 16 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 osd.7 17 handle_osd_map pg_num_history pg_num_history(e18 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {18={}} 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 osd.7 17 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e18 [1,18] lci=[13,18]) 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f6c00 seq 10 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c15b972f:::inc_osdmap.18:0# key 0x7f7fffffffffffffffc15b97'/!inc_osdmap.18!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1063 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~259 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xda7 on front/back, now 0x0~1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6c00 1 blobs 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2b000~1000/1000,1000,0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2b000~1000] 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552655e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265650 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552655e0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c55265650) get_ref 0x0~259 Blob(0x558c55265650 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552655e0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c55265650) get_ref init 0x1000, 1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~259: 0x0~259 Blob(0x558c55265650 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x259) SharedBlob(0x558c552655e0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c552655f8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x259 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7008) dirty_range mark inline shard dirty 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 = 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:39c0e826:::osdmap.18:0# key 0x7f7fffffffffffffff39c0e8'&!osdmap.18!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1064 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:39c0e826:::osdmap.18:0# 0x0~13ff - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~3ff 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc01 on front/back, now 0x0~1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6c00 2 blobs 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2c000~2000/1000,2000,0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2c000~2000] 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55265490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265500 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265490 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c55265500) get_ref 0x0~1000 Blob(0x558c55265500 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265490 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c55265500) get_ref init 0x1000, 1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55265500 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265490 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c552654a8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552653b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265420 blob([!~1000,0x2d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552653b0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c55265420) get_ref 0x1000~3ff Blob(0x558c55265420 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c552653b0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c55265420) get_ref init 0x2000, 1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~3ff: 0x1000~3ff Blob(0x558c55265420 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,3ff]) SharedBlob(0x558c552653b0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c552653c8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x13ff 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7288) dirty_range mark inline shard dirty 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff = 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x38) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6c00 0 blobs 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x26 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6c00 0 blobs 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 23201 (4 ios * 4000 + 7201 bytes) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes 0x558c53a91900,0x558c54ce6f00,0x558c54ce7180,0x558c55276c80 shared_blobs 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7008) update #-1:c15b972f:::inc_osdmap.18:0# 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c15b972f:::inc_osdmap.18:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7288) update #-1:39c0e826:::osdmap.18:0# 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7288) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:39c0e826:::osdmap.18:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[2b000~3000] released 0x[] 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 freelist allocate 0x2b000~3000 2019-12-07T19:33:55.614+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:33:55.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6c00 2019-12-07T19:33:55.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 2 ==== pg_query2(1.6s2 1.6s2 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54ce7b80 con 0x558c54c5bc00 2019-12-07T19:33:55.614+0000 7f8f24457700 15 osd.7 17 enqueue_peering_evt 1.6s2 epoch_sent: 18 epoch_requested: 18 MQuery 1.6s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.614+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 aio_wait 2019-12-07T19:33:55.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:33:55.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:55.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 map 18 > 17 2019-12-07T19:33:55.615+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 18, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce6f00 had 1 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce7180 had 1 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc88d~226c to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc88d 0xc88d~226c to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xc88d 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x88d 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xaf9 and padding block with 0x507 2019-12-07T19:33:55.615+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (crc 0 0 0) 0x558c53a78fc0 con 0x558c54c5bc00 2019-12-07T19:33:55.615+0000 7f8f1199c700 20 osd.7 17 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7f8f1199c700 10 osd.7 17 do_waiters -- start 2019-12-07T19:33:55.615+0000 7f8f1199c700 10 osd.7 17 do_waiters -- finish 2019-12-07T19:33:55.615+0000 7f8f1199c700 20 osd.7 17 _dispatch 0x558c53a78fc0 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7f8f1199c700 3 osd.7 17 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.615+0000 7f8f1199c700 10 osd.7 17 no new maps here, dropping 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xeaf9 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:55.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000176168s (0.000042620s flush + 0.000133548s kv commit) 2019-12-07T19:33:55.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6c00> 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f6c00 osr 0x558c54ea4b40 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 7 0x1b000~1000 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 8 0x1b000~1000 crc 45ebe265 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 7 0x2000~1000 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 8 0x2000~1000 crc 9e06976e 2019-12-07T19:33:55.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7 17 _committed_osd_maps 18..18 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7 17 advance to epoch 18 (<= last 18 <= newest_map 18) 2019-12-07T19:33:55.615+0000 7f8f141a1700 0 osd.7 18 crush map has features 288514119978713088, adjusting msgr requires for clients 2019-12-07T19:33:55.615+0000 7f8f141a1700 0 osd.7 18 crush map has features 288514119978713088 was 288514050185503233, adjusting msgr requires for mons 2019-12-07T19:33:55.615+0000 7f8f141a1700 0 osd.7 18 crush map has features 3314933069571702784, adjusting msgr requires for osds 2019-12-07T19:33:55.615+0000 7f8f141a1700 7 osd.7 18 consume_map version 18 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:4.identify_splits_and_merges identify_splits_and_merges slot 1.4s0 has no pg and waiting_for_split 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:5.identify_splits_and_merges identify_splits_and_merges slot 1.ds2 has no pg and waiting_for_split 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:6.identify_splits_and_merges identify_splits_and_merges slot 1.6s2 has no pg and waiting_for_split 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7 18 prune_pg_created 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:0.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:1.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:2.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:3.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:4.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.4s0 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:5.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.ds2 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:6.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7f8f141a1700 10 osd.7:7.consume_map 18 (was 17) 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds2 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds2 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7f8f0217d700 10 osd.7 18 handle_pg_query_nopg 1.ds2 2019-12-07T19:33:55.615+0000 7f8f0217d700 10 osd.7 18 pg 1.ds2 dne 2019-12-07T19:33:55.615+0000 7f8f0197c700 10 osd.7 18 handle_pg_query_nopg 1.6s2 2019-12-07T19:33:55.615+0000 7f8f0197c700 10 osd.7 18 pg 1.6s2 dne 2019-12-07T19:33:55.615+0000 7f8f0217d700 10 osd.7 18 send_incremental_map 17 -> 18 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:33:55.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 no pg, should create on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f8f0197c700 10 osd.7 18 send_incremental_map 17 -> 18 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:33:55.615+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~0 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~259 size 0x259 (601) 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265650 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x259) SharedBlob(0x558c552655e0 sbid 0x0)) need 0x0~259 cache has 0x[0~259] 2019-12-07T19:33:55.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 = 601 2019-12-07T19:33:55.615+0000 7f8f0217d700 10 osd.7 18 add_map_inc_bl 18 601 bytes 2019-12-07T19:33:55.615+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(18..18 src has 1..18) v4 -- 0x558c5524cfc0 con 0x558c54c19800 2019-12-07T19:33:55.615+0000 7f8f0217d700 10 osd.7 18 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.615+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify((query:18 sent:18 1.ds2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558c53a71a40 con 0x558c54c19800 2019-12-07T19:33:55.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:55.615+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(18..18 src has 1..18) v4 -- 0x558c5524d440 con 0x558c54c5bc00 2019-12-07T19:33:55.615+0000 7f8f0197c700 10 osd.7 18 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.615+0000 7f8f0297e700 10 osd.7 18 _make_pg 1.4s0 2019-12-07T19:33:55.615+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:18 sent:18 1.6s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558c54d0e3c0 con 0x558c54c5bc00 2019-12-07T19:33:55.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:55.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4(unlocked)] enter Initial 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4(unlocked)] enter NotTrimming 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.4s0_head fresh osr 0x558c5525f540 2019-12-07T19:33:55.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 3 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54ce7b80 con 0x558c54c64000 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 0 up [7,0,1,5,2,4] acting [7,0,1,5,2,4] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:55.616+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.9s4 epoch_sent: 18 epoch_requested: 18 MQuery 1.9s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.616+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] init_hb_stamps now [0x558c54676310,0x558c5523a230,0x558c54677f10,0x558c5523a150,0x558c5523a3f0] 2019-12-07T19:33:55.616+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 2 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (crc 0 0 0) 0x558c5524d440 con 0x558c54c64000 2019-12-07T19:33:55.616+0000 7f8f1199c700 20 osd.7 18 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 19.053193879s + 0s) 2019-12-07T19:33:55.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:55.616+0000 7f8f08189700 10 osd.7 18 handle_pg_query_nopg 1.9s4 2019-12-07T19:33:55.616+0000 7f8f08189700 10 osd.7 18 pg 1.9s4 dne 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:55.616+0000 7f8f08189700 10 osd.7 18 send_incremental_map 17 -> 18 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:33:55.616+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(18..18 src has 1..18) v4 -- 0x558c5524dd40 con 0x558c54c64000 2019-12-07T19:33:55.616+0000 7f8f08189700 10 osd.7 18 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.616+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:55.616+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.9s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558c54d0f680 con 0x558c54c64000 2019-12-07T19:33:55.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:33:55.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:55.617841+0000 0 0.000000 2019-12-07T19:33:55.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 4 ==== pg_query2(1.0s3 1.0s3 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54ce7b80 con 0x558c54c64000 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:55.616+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.0s3 epoch_sent: 18 epoch_requested: 18 MQuery 1.0s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.616+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.0s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] activate_map 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] update_heartbeat_peers -> 0,1,2,4,5,7 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 18 need_heartbeat_peer_update 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:55.617879+0000 1 0.000031 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Start 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] state: transitioning to Primary 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] exit Start 2019-12-07T19:33:55.617893+0000 0 0.000000 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary 2019-12-07T19:33:55.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] enter Started/Primary/Peering 2019-12-07T19:33:55.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.0s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetInfo 2019-12-07T19:33:55.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.0s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7f8f0898a700 10 osd.7 18 handle_pg_query_nopg 1.0s3 2019-12-07T19:33:55.616+0000 7f8f0898a700 10 osd.7 18 pg 1.0s3 dne 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] build_prior all_probe 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] build_prior final: probe 0(1),1(2),2(4),4(5),5(3),7(0) down blocked_by {} 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] up_thru 0 < same_since 18, must notify monitor 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: no prior_set down osds, clearing prior_readable_until_ub 2019-12-07T19:33:55.616+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.0s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558c54d0fc20 con 0x558c54c64000 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.0(1) 2019-12-07T19:33:55.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.1(2) 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.2(4) 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.4(5) 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.5(3) 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:55.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 18 send_incremental_map 17 -> 18 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(18..18 src has 1..18) v4 -- 0x558c5524db00 con 0x558c54c64400 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 18 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_query2(1.4s1 1.4s1 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558c54d2c000 con 0x558c54c64400 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.4s2 1.4s2 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558c54d2c280 con 0x558c54c64000 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.4s4 1.4s4 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558c54d2c500 con 0x558c54c19800 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 18 send_incremental_map 17 -> 18 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(18..18 src has 1..18) v4 -- 0x558c54d2e000 con 0x558c54c5b400 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 18 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.4s5 1.4s5 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558c54d2c780 con 0x558c54c5b400 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 18 send_incremental_map 17 -> 18 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(18..18 src has 1..18) v4 -- 0x558c54d2e240 con 0x558c54c19c00 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 18 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.4s3 1.4s3 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558c54d2ca00 con 0x558c54c19c00 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d20c00 seq 1 2019-12-07T19:33:55.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.4s0_head bits 4 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.4s0_head bits 4 = 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:20000000::::head# key 0x8080000000000000012000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) r -2 v.len 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1065 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._infover' <- _infover 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20c00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20c00 onodes 0x558c54d2cc80 shared_blobs 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c54d2cd88) update 0#1:20000000::::head# 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c54d2cd88) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:20000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20c00 allocated 0x[] released 0x[] 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20c00 prepare 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20c00 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20c00 io_done 2019-12-07T19:33:55.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:55.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 18 handle_pg_create_info new pg pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7:4._attach_pg 1.4s0 0x558c548ab000 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f8f0297e700 20 osd.7 18 identify_splits_and_merges 1.4s0 e18 to e18 pg_nums {18=16} 2019-12-07T19:33:55.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7:4._prime_splits 2019-12-07T19:33:55.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 NullEvt +create_info 2019-12-07T19:33:55.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:55.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:55.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:55.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 18 queue_want_up_thru now 18 (was 0), currently 0 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 18 send_alive up_thru currently 0 want 18 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 18 send_alive want 18 2019-12-07T19:33:55.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xeaf9~56d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xeaf9 0xeaf9~56d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xeaf9 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xaf9 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x66 and padding block with 0xf9a 2019-12-07T19:33:55.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_alive(want up_thru 18 have 18) v1 -- 0x558c54d0fe00 con 0x558c5476d000 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:55.617+0000 7f8f141a1700 15 osd.7 18 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 NullEvt 2019-12-07T19:33:55.617+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 maybe_update_heartbeat_peers updating 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 adding neighbor peer osd.0 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 adding neighbor peer osd.1 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 adding neighbor peer osd.6 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 adding random peer osd.2 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 adding random peer osd.4 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 adding random peer osd.5 2019-12-07T19:33:55.617+0000 7f8f141a1700 10 osd.7 18 maybe_update_heartbeat_peers 7 peers, extras 0,1,2,3,4,5,6 2019-12-07T19:33:55.617+0000 7f8f141a1700 7 osd.7 18 activate_map version 18 2019-12-07T19:33:55.617+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 NullEvt 2019-12-07T19:33:55.617+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:55.617+0000 7f8f06986700 10 osd.7 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7f8f0918b700 20 osd.7 18 agent_entry empty queue 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf066 2019-12-07T19:33:55.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:55.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:55.617+0000 7f8f1199c700 20 osd.7 18 _dispatch 0x558c5524d440 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000151094s (0.000000176s flush + 0.000150918s kv commit) 2019-12-07T19:33:55.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:55.617+0000 7f8f1199c700 3 osd.7 18 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 no new maps here, dropping 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20c00> 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20c00 kv_submitted 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20c00 2019-12-07T19:33:55.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20c00 finishing 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20c00 onodes 0x558c54d2cc80 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20c00 done 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:55.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20c00 [] 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:55.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 2 ==== pg_notify((query:18 sent:18 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x558c54d0e3c0 con 0x558c54c19c00 2019-12-07T19:33:55.617+0000 7f8f24457700 7 osd.7 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.5 2019-12-07T19:33:55.617+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 5(3) notify: (query:18 sent:18 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 5(3) notify: (query:18 sent:18 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (unknown 0 0 0) 0x558c54d2e240 con 0x558c54c19c00 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 5(3) notify: (query:18 sent:18 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.617+0000 7f8f1199c700 20 osd.7 18 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f1199c700 20 osd.7 18 _dispatch 0x558c54d2e240 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 5(3) notify: (query:18 sent:18 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 5(3) notify: (query:18 sent:18 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.617+0000 7f8f1199c700 3 osd.7 18 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 no new maps here, dropping 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.5(3) 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.617+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 5 ==== pg_notify((query:18 sent:18 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (crc 0 0 0) 0x558c53a71a40 con 0x558c54c19800 2019-12-07T19:33:55.617+0000 7f8f24c58700 7 osd.7 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.2 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:24.209557+0000, must = 0 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,2,4,5,7 unchanged 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:55.617+0000 7f8f24c58700 15 osd.7 18 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 2(4) notify: (query:18 sent:18 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.617+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 2(4) notify: (query:18 sent:18 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 2(4) notify: (query:18 sent:18 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 2(4) notify: (query:18 sent:18 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 2(4) notify: (query:18 sent:18 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.617+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 2 ==== pg_notify((query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (crc 0 0 0) 0x558c54d0fe00 con 0x558c54c64400 2019-12-07T19:33:55.617+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.2(4) 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.617+0000 7f8f23c56700 7 osd.7 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.0 2019-12-07T19:33:55.617+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:16.322514+0000, must = 0 2019-12-07T19:33:55.617+0000 7f8f23c56700 15 osd.7 18 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 0(1) notify: (query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.617+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 0(1) notify: (query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.617+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,2,4,5,7 unchanged 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 0(1) notify: (query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.617+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:55.617+0000 7f8f06986700 10 osd.7 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 0(1) notify: (query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:33:55.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (crc 0 0 0) 0x558c5524db00 con 0x558c54c64400 2019-12-07T19:33:55.617+0000 7f8f1199c700 20 osd.7 18 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 0(1) notify: (query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:55.617+0000 7f8f1199c700 20 osd.7 18 _dispatch 0x558c5524db00 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.617+0000 7f8f1199c700 3 osd.7 18 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.617+0000 7f8f1199c700 10 osd.7 18 no new maps here, dropping 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.0(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:16.398356+0000, must = 0 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,2,4,5,7 unchanged 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 0 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:55.617+0000 7f8f0297e700 10 osd.7 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.618+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (unknown 0 0 0) 0x558c54d2e000 con 0x558c54c5b400 2019-12-07T19:33:55.618+0000 7f8f1199c700 20 osd.7 18 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.618+0000 7f8f1199c700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:55.618+0000 7f8f1199c700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:55.618+0000 7f8f1199c700 20 osd.7 18 _dispatch 0x558c54d2e000 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.618+0000 7f8f1199c700 3 osd.7 18 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.618+0000 7f8f1199c700 10 osd.7 18 no new maps here, dropping 2019-12-07T19:33:55.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 4 ==== pg_notify((query:18 sent:18 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x558c54d0e3c0 con 0x558c54c5b400 2019-12-07T19:33:55.618+0000 7f8f24457700 7 osd.7 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.4 2019-12-07T19:33:55.618+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 4(5) notify: (query:18 sent:18 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.618+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 4(5) notify: (query:18 sent:18 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 5 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54d2ca00 con 0x558c54c64000 2019-12-07T19:33:55.618+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.7s4 epoch_sent: 18 epoch_requested: 18 MQuery 1.7s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.618+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 6 ==== pg_notify((query:18 sent:18 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (crc 0 0 0) 0x558c54d0fc20 con 0x558c54c64000 2019-12-07T19:33:55.618+0000 7f8f24457700 7 osd.7 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.1 2019-12-07T19:33:55.618+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 1(2) notify: (query:18 sent:18 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.618+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 1(2) notify: (query:18 sent:18 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 4(5) notify: (query:18 sent:18 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 4(5) notify: (query:18 sent:18 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:33:55.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 4(5) notify: (query:18 sent:18 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.619+0000 7f8f05183700 10 osd.7 18 handle_pg_query_nopg 1.7s4 2019-12-07T19:33:55.619+0000 7f8f05183700 10 osd.7 18 pg 1.7s4 dne 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.4(5) 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:22.445290+0000, must = 0 2019-12-07T19:33:55.619+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.7s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558c54d383c0 con 0x558c54c64000 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,2,4,5,7 unchanged 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:55.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 1(2) notify: (query:18 sent:18 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 1(2) notify: (query:18 sent:18 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.4s0 from 1(2) notify: (query:18 sent:18 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.1(2) 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:23.718605+0000, must = 0 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,2,4,5,7 unchanged 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7f8f06986700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetInfo 2019-12-07T19:33:55.620376+0000 7 0.000262 2019-12-07T19:33:55.619+0000 7f8f06986700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetLog 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.0(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.1(2) 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.2(4) 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.4(5) 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.5(3) 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.7(0) 1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] calc_acting prefer osd.7(0) because it is current primary 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 0(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 4(5) 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_async_recovery_ec result want=[7,0,1,5,2,4] async_recovery= 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] acting_recovery_backfill is 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting want=[7,0,1,5,2,4] backfill_targets= async_recovery_targets= 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: leaving GetLog 2019-12-07T19:33:55.619+0000 7f8f06986700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetLog 2019-12-07T19:33:55.620551+0000 0 0.000000 2019-12-07T19:33:55.619+0000 7f8f06986700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetMissing 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: still need up_thru update before going active 2019-12-07T19:33:55.619+0000 7f8f06986700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetMissing 2019-12-07T19:33:55.620579+0000 0 0.000000 2019-12-07T19:33:55.619+0000 7f8f06986700 5 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:33:55.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:55.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:55.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d20000 seq 2 2019-12-07T19:33:55.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20000 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:33:55.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20000 onodes shared_blobs 2019-12-07T19:33:55.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20000 allocated 0x[] released 0x[] 2019-12-07T19:33:55.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20000 prepare 2019-12-07T19:33:55.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20000 2019-12-07T19:33:55.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20000 io_done 2019-12-07T19:33:55.619+0000 7f8f06986700 10 osd.7 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf066~2e to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf066 0xf066~2e to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xf066 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x66 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x94 and padding block with 0xf6c 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf094 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:55.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000113274s (0.000000049s flush + 0.000113225s kv commit) 2019-12-07T19:33:55.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20000> 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20000 kv_submitted 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20000 2019-12-07T19:33:55.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20000 finishing 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20000 onodes 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20000 done 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:55.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20000 [] 2019-12-07T19:33:55.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:55.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:55.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 5 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c54d2c280 con 0x558c54c5b400 2019-12-07T19:33:55.620+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.3s4 epoch_sent: 18 epoch_requested: 18 MQuery 1.3s4 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.620+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s4 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s4 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s4 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.620+0000 7f8f07187700 10 osd.7 18 handle_pg_query_nopg 1.3s4 2019-12-07T19:33:55.620+0000 7f8f07187700 10 osd.7 18 pg 1.3s4 dne 2019-12-07T19:33:55.620+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:18 sent:18 1.3s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558c54d38960 con 0x558c54c5b400 2019-12-07T19:33:55.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:55.657+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.707+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.757+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.807+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.857+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.907+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.916+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c552c3400 0x558c552b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.916+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c552c3400 0x558c552b5700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.916+0000 7f8f24457700 10 osd.7 18 ms_handle_authentication new session 0x558c54c5c9c0 con 0x558c552c3400 entity client.admin addr 2019-12-07T19:33:55.916+0000 7f8f24457700 10 osd.7 18 ms_handle_authentication session 0x558c54c5c9c0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.917+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/808367751 conn(0x558c552c3400 0x558c552b5700 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=client.4528 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:55.917+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4528 172.21.15.131:0/808367751 1 ==== command(tid 15: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c54d383c0 con 0x558c552c3400 2019-12-07T19:33:55.917+0000 7f8f164bb700 20 osd.7 18 OSD::ms_dispatch: command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:55.917+0000 7f8f164bb700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:55.917+0000 7f8f164bb700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:55.917+0000 7f8f164bb700 20 osd.7 18 _dispatch 0x558c54d383c0 command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:55.917+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/808367751 -- command_reply(tid 15: 0 ) v1 -- 0x558c53a6f180 con 0x558c552c3400 2019-12-07T19:33:55.939+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4528 172.21.15.131:0/808367751 2 ==== command(tid 16: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c54d38960 con 0x558c552c3400 2019-12-07T19:33:55.939+0000 7f8f164bb700 20 osd.7 18 OSD::ms_dispatch: command(tid 16: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:33:55.939+0000 7f8f164bb700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:55.939+0000 7f8f164bb700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:55.939+0000 7f8f164bb700 20 osd.7 18 _dispatch 0x558c54d38960 command(tid 16: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:33:55.941+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/808367751 -- command_reply(tid 16: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c53a6ee00 con 0x558c552c3400 2019-12-07T19:33:55.943+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/808367751 conn(0x558c552c3400 msgr2=0x558c552b5700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:55.943+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/808367751 conn(0x558c552c3400 msgr2=0x558c552b5700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:55.943+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/808367751 conn(0x558c552c3400 0x558c552b5700 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:55.943+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/808367751 conn(0x558c552c3400 0x558c552b5700 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:55.943+0000 7f8f164bb700 2 osd.7 18 ms_handle_reset con 0x558c552c3400 session 0x558c54c5c9c0 2019-12-07T19:33:55.953+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.6 172.21.15.109:0/36937 4 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.019322605s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c53ae4c00 2019-12-07T19:33:55.953+0000 7f8f24457700 20 osd.7 18 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018850712s,0.019322605s]) 2019-12-07T19:33:55.953+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.389897725s delta_ub 18446744073.690700904s) v5 -- 0x558c55250200 con 0x558c53ae4c00 2019-12-07T19:33:55.953+0000 7f8f24457700 10 osd.7 18 maybe_share_map con 0x558c54c19400 v2:172.21.15.109:6818/36937 map epoch 17 -> 18 (as per caller) 2019-12-07T19:33:55.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.6 172.21.15.109:0/36937 4 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.019322605s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c552d0400 2019-12-07T19:33:55.953+0000 7f8f23c56700 20 osd.7 18 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018593829s,0.019322605s]) 2019-12-07T19:33:55.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.390154608s delta_ub 18446744073.690957787s) v5 -- 0x558c55250400 con 0x558c552d0400 2019-12-07T19:33:55.957+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.008+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.057+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.104+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:33:56.108+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.158+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.208+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.258+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.308+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.341+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.808987388s delta_ub 0.031104013s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c54c5a400 2019-12-07T19:33:56.341+0000 7f8f24c58700 20 osd.7 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030640260s,0.031104013s]) 2019-12-07T19:33:56.341+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.778347128s delta_ub 18446744073.678911356s) v5 -- 0x558c55250600 con 0x558c54c5a400 2019-12-07T19:33:56.341+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.808987388s delta_ub 0.031104013s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c54c5ac00 2019-12-07T19:33:56.341+0000 7f8f23c56700 20 osd.7 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030565458s,0.031104013s]) 2019-12-07T19:33:56.341+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.778421930s delta_ub 18446744073.678986158s) v5 -- 0x558c55250400 con 0x558c54c5ac00 2019-12-07T19:33:56.358+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.408+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.421+0000 7f8f21451700 10 osd.7 18 tick 2019-12-07T19:33:56.421+0000 7f8f21451700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:56.421+0000 7f8f21451700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:56.421+0000 7f8f21451700 20 osd.7 18 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:56.452+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 5 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.019543146s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c54c18400 2019-12-07T19:33:56.452+0000 7f8f23c56700 20 osd.7 18 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019002272s,0.019543146s]) 2019-12-07T19:33:56.452+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.889020596s delta_ub 18446744073.690549344s) v5 -- 0x558c55250400 con 0x558c54c18400 2019-12-07T19:33:56.452+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 5 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.019543146s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c54c18c00 2019-12-07T19:33:56.452+0000 7f8f23c56700 20 osd.7 18 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018951199s,0.019543146s]) 2019-12-07T19:33:56.452+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.889071669s delta_ub 18446744073.690600417s) v5 -- 0x558c55250400 con 0x558c54c18c00 2019-12-07T19:33:56.458+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.508+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.559+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 10 osd.7 18 tick_without_osd_lock 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffd2000/0x40000000/0x1680000000, data 0xda65/0x2c000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 25 osd.7 18 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772649+0000 last_rx_front 2019-12-07T19:33:54.772626+0000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 25 osd.7 18 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772556+0000 last_rx_front 2019-12-07T19:33:54.772753+0000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 25 osd.7 18 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772696+0000 last_rx_front 2019-12-07T19:33:54.772667+0000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 25 osd.7 18 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772638+0000 last_rx_front 2019-12-07T19:33:54.772726+0000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 25 osd.7 18 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772464+0000 last_rx_front 2019-12-07T19:33:54.772448+0000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 25 osd.7 18 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772502+0000 last_rx_front 2019-12-07T19:33:54.772537+0000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 25 osd.7 18 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:54.772210+0000 last_rx_back 2019-12-07T19:33:54.772683+0000 last_rx_front 2019-12-07T19:33:54.772519+0000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 20 osd.7 18 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 10 osd.7 18 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 20 osd.7 18 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:56.582+0000 7f8f1fbbe700 10 osd.7 18 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:56.609+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258776 2019-12-07T19:33:56.609+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 14 ==== osd_map(18..19 src has 1..19) v4 ==== 1298+0+0 (crc 0 0 0) 0x558c5524c000 con 0x558c5476d000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 osd.7 18 OSD::ms_dispatch: osd_map(18..19 src has 1..19) v4 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 osd.7 18 do_waiters -- start 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 osd.7 18 do_waiters -- finish 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 osd.7 18 _dispatch 0x558c5524c000 osd_map(18..19 src has 1..19) v4 2019-12-07T19:33:56.613+0000 7f8f164bb700 3 osd.7 18 handle_osd_map epochs [18,19], i have 18, src has [1,19] 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 osd.7 18 handle_osd_map got inc map for epoch 19 2019-12-07T19:33:56.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:39c0e826:::osdmap.18:0# 0x0~0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~13ff size 0x13ff (5119) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265500 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265490 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265420 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,3ff]) SharedBlob(0x558c552653b0 sbid 0x0)) need 0x1000~3ff cache has 0x[1000~3ff] 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff = 5119 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 osd.7 18 add_map_bl 18 5119 bytes 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 osd.7 18 got_full_map 19, nothing requested 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 osd.7 18 handle_osd_map pg_num_history pg_num_history(e19 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {19={}} 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 osd.7 18 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e19 [1,19] lci=[13,19]) 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54d20600 seq 11 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:ca5b972f:::inc_osdmap.19:0# key 0x7f7fffffffffffffffca5b97'/!inc_osdmap.19!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1066 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~285 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd7b on front/back, now 0x0~1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20600 1 blobs 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2e000~1000/1000,1000,0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2e000~1000] 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54676ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54676fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54676ee0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54676fc0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c54676ee0) get_ref 0x0~285 Blob(0x558c54676ee0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54676fc0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c54676ee0) get_ref init 0x1000, 1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~285: 0x0~285 Blob(0x558c54676ee0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x558c54676fc0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54676fd8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x285 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d008) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 = 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:34c0e826:::osdmap.19:0# key 0x7f7fffffffffffffff34c0e8'&!osdmap.19!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:34c0e826:::osdmap.19:0# 0x0~140c 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1067 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:34c0e826:::osdmap.19:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~40c 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20600 2 blobs 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2f000~2000/1000,2000,0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2f000~2000] 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552791f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279180 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552791f0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c55279180) get_ref 0x0~1000 Blob(0x558c55279180 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552791f0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c55279180) get_ref init 0x1000, 1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55279180 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c552791f0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55279208 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c547404d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552792d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c547404d0 blob([!~1000,0x30000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552792d0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c547404d0) get_ref 0x1000~40c Blob(0x558c547404d0 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c552792d0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c547404d0) get_ref init 0x2000, 1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558c547404d0 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558c552792d0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c552792e8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x140c 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d288) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:34c0e826:::osdmap.19:0# 0x0~140c = 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20600 0 blobs 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20600 0 blobs 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 23258 (4 ios * 4000 + 7258 bytes) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes 0x558c53a91900,0x558c54d2cf00,0x558c54d2d180,0x558c55276c80 shared_blobs 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d008) update #-1:ca5b972f:::inc_osdmap.19:0# 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:ca5b972f:::inc_osdmap.19:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d288) update #-1:34c0e826:::osdmap.19:0# 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d288) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:34c0e826:::osdmap.19:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[2e000~3000] released 0x[] 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 freelist allocate 0x2e000~3000 2019-12-07T19:33:56.613+0000 7f8f164bb700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:33:56.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d20600 2019-12-07T19:33:56.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 6 ==== PGlog(1.3s4 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558c54896000 con 0x558c54c5b400 2019-12-07T19:33:56.614+0000 7f8f24457700 15 osd.7 18 enqueue_peering_evt 1.3s4 epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info 2019-12-07T19:33:56.614+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 map 19 > 18 2019-12-07T19:33:56.614+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 19, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 aio_wait 2019-12-07T19:33:56.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:33:56.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3948 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cf00 had 1 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2d180 had 1 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf094~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf094 0xf094~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.564757+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _allocate len 0x130b from 0 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _allocate len 0x130b from 1 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd130000 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd0f0000~10000/10000,10000,acd130000 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x20000 to include allocated 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 23 (was clean) 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xf094 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x94 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x30b and padding block with 0xcf5 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x20000 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1130b 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (23) on file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]), flushing log 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 23 len 0x30 crc 0x4ed7fc42) 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x16000~1000 to file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x16000 0x16000~1000 to file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x16000 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x17000 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:56.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000017 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 23 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000422384s (0.000040539s flush + 0.000381845s kv commit) 2019-12-07T19:33:56.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20600> 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54d20600 osr 0x558c54ea4b40 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 8 0x1b000~1000 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 9 0x1b000~1000 crc 5f344e98 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 8 0x2000~1000 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 9 0x2000~1000 crc 253d8353 2019-12-07T19:33:56.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7 18 _committed_osd_maps 19..19 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7 18 advance to epoch 19 (<= last 19 <= newest_map 19) 2019-12-07T19:33:56.614+0000 7f8f141a1700 7 osd.7 19 consume_map version 19 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:0.identify_splits_and_merges identify_splits_and_merges slot 1.0s3 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:1.identify_splits_and_merges identify_splits_and_merges slot 1.9s4 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.3s4 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7 19 identify_splits_and_merges 1.4s0 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:5.identify_splits_and_merges identify_splits_and_merges slot 1.ds2 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:6.identify_splits_and_merges identify_splits_and_merges slot 1.6s2 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:7.identify_splits_and_merges identify_splits_and_merges slot 1.7s4 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7 19 prune_pg_created 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:0.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 empty, pruning 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:1.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 empty, pruning 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:2.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:3.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:4.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:5.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 empty, pruning 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:6.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 empty, pruning 2019-12-07T19:33:56.614+0000 7f8f141a1700 10 osd.7:7.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:33:56.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 empty, pruning 2019-12-07T19:33:56.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 no pg, should create on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 19 _make_pg 1.3s4 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3(unlocked)] enter Initial 2019-12-07T19:33:56.615+0000 7f8f07187700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.615+0000 7f8f07187700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.615+0000 7f8f07187700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.615+0000 7f8f07187700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.615+0000 7f8f07187700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 18 pg[1.3(unlocked)] enter NotTrimming 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.3s4_head fresh osr 0x558c5525e140 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 4 up [4,1,2,5,7,6] acting [4,1,2,5,7,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.051804074s + 0s) 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.615+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.616412+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.616457+0000 1 0.000037 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.615+0000 7f8f07187700 1 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.616478+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54d21200 seq 1 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.3s4_head bits 4 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.3s4_head bits 4 = 0 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:c0000000::::head# key 0x848000000000000001c000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) r -2 v.len 0 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1068 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._infover' <- _infover 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21200 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21200 onodes 0x558c55148000 shared_blobs 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore.extentmap(0x558c55148108) update 4#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore.extentmap(0x558c55148108) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 prepare 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21200 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 io_done 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 19 handle_pg_create_info new pg pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7:3._attach_pg 1.3s4 0x558c5513e000 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 19 identify_splits_and_merges 1.3s4 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7:3._prime_splits 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 18 -> 19 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 18 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f8f07187700 10 merge_log log((0'0,0'0], crt=0'0) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f8f07187700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.064119817s int 16s) 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.013547537s -> ruub 36.050572280s 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.050572280s 2019-12-07T19:33:56.615+0000 7f8f141a1700 10 osd.7 19 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:56.615+0000 7f8f141a1700 15 osd.7 19 enqueue_peering_evt 1.3s4 epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.616751+0000 3 0.000054 2019-12-07T19:33:56.615+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7f8f141a1700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.615+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.615+0000 7f8f141a1700 7 osd.7 19 activate_map version 19 2019-12-07T19:33:56.615+0000 7f8f07187700 5 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.615+0000 7f8f0918b700 20 osd.7 19 agent_entry empty queue 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54d21500 seq 2 2019-12-07T19:33:56.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 18 -> 19 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 18 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Peering advmap 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes shared_blobs 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:33:56.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] adjust_need_up_thru now 18, need_up_thru now false 2019-12-07T19:33:56.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Started advmap 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) pg 0x558c5513e000 2019-12-07T19:33:56.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.615+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] hit_set_clear 2019-12-07T19:33:56.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] agent_stop 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f8f0297e700 7 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: handle ActMap primary 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:33:56.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:33:56.616933+0000 2 0.000306 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Leaving Peering 2019-12-07T19:33:56.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering 2019-12-07T19:33:56.616955+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] enter Started/Primary/Active 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] state: In Active, about to call activate 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.0 ruub 0s 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:33:56.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.052429711s) 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.0(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.0(1) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.0(1) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- PGlog(1.4s1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) e19/18) v6 -- 0x558c54895800 con 0x558c54c64400 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.0(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- PGlog(1.4s2 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) e19/18) v6 -- 0x558c551db800 con 0x558c54c64000 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) 1.4s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- PGlog(1.4s4 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) e19/18) v6 -- 0x558c54897000 con 0x558c54c19800 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) 1.4s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(5) 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(5) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(5) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- PGlog(1.4s5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) e19/18) v6 -- 0x558c551dc000 con 0x558c54c5b400 2019-12-07T19:33:56.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(5) 1.4s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(3) 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(3) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(3) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- PGlog(1.4s3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) e19/18) v6 -- 0x558c551dc800 con 0x558c54c19c00 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(3) 1.4s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 0(1) 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 1(2) 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 2(4) 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 4(5) 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 5(3) 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 2 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 7(0) 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1130b~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1130b 0x1130b~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x130b 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x30b 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcc6 and padding block with 0x33a 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 18:1 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: Activate Finished 2019-12-07T19:33:56.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] enter Started/Primary/Active/Activating 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d20f00 seq 3 2019-12-07T19:33:56.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20f00 cost 5345 (1 ios * 4000 + 1345 bytes) 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20f00 onodes shared_blobs 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20f00 prepare 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20f00 2019-12-07T19:33:56.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20f00 io_done 2019-12-07T19:33:56.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 6 ==== PGlog(1.ds2 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x558c54693000 con 0x558c54c19800 2019-12-07T19:33:56.616+0000 7f8f24c58700 15 osd.7 19 enqueue_peering_evt 1.ds2 epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 no pg, should create on OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f8f06185700 10 osd.7 19 _make_pg 1.ds2 2019-12-07T19:33:56.616+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.d(unlocked)] enter Initial 2019-12-07T19:33:56.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 7 ==== PGlog(1.9s4 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x558c551db800 con 0x558c54c64000 2019-12-07T19:33:56.616+0000 7f8f06185700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.616+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.9s4 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f8f06185700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.616+0000 7f8f06185700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.616+0000 7f8f06185700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.616+0000 7f8f06185700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 8 ==== PGlog(1.0s3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074625527s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x558c551dc800 con 0x558c54c64000 2019-12-07T19:33:56.616+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.0s3 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074625527s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074625527s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f8f06185700 20 osd.7 pg_epoch: 18 pg[1.d(unlocked)] enter NotTrimming 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x11cc6 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.616+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.ds2_head fresh osr 0x558c5525e280 2019-12-07T19:33:56.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.616+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 2 up [2,1,7,3,4,0] acting [2,1,7,3,4,0] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000942834s (0.000000132s flush + 0.000942702s kv commit) 2019-12-07T19:33:56.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.616+0000 7f8f06185700 20 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1188 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558c5523a150] 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1211 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1211 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.053651100s + 0s) 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x11cc6~4bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x11cc6 0x11cc6~4bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x1cc6 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcc6 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x181 and padding block with 0xe7f 2019-12-07T19:33:56.617+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.618257+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.618287+0000 1 0.000022 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.617+0000 7f8f06185700 1 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.618299+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.617+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d21b00 seq 1 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.ds2_head bits 4 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12181 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.ds2_head bits 4 = 0 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:b0000000::::head# key 0x828000000000000001b000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000145061s (0.000000081s flush + 0.000144980s kv commit) 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1069 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._infover' <- _infover 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes 0x558c55148280 shared_blobs 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore.extentmap(0x558c55148388) update 2#1:b0000000::::head# 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore.extentmap(0x558c55148388) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:b0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 19 handle_pg_create_info new pg pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7:5._attach_pg 1.ds2 0x558c55141000 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 19 identify_splits_and_merges 1.ds2 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7:5._prime_splits 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12181~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 18 -> 19 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12181 0x12181~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x2181 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x181 2019-12-07T19:33:56.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 18 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6ee and padding block with 0x912 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.617+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x126ee 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21200,0x558c54d21500,0x558c54d20f00> 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.2(0) 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 kv_submitted 2019-12-07T19:33:56.617+0000 7f8f06185700 10 merge_log log((0'0,0'0], crt=0'0) from osd.2(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000143238s (0.000000090s flush + 0.000143148s kv commit) 2019-12-07T19:33:56.617+0000 7f8f06185700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21200 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.088857038s int 16s) 2019-12-07T19:33:56.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.036074596s -> ruub 36.052782442s 2019-12-07T19:33:56.617+0000 7f8f06185700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.052782442s 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 finishing 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21200 onodes 0x558c55148000 2019-12-07T19:33:56.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.618591+0000 3 0.000033 2019-12-07T19:33:56.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21200 done 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 kv_submitted 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21200 [] 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:33:56.617+0000 7f8f06185700 5 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:33:56.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:33:56.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f8f07187700 15 osd.7 19 enqueue_peering_evt 1.3s4 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074625527s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558c5513e000 2019-12-07T19:33:56.617+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.617+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.617+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f8f07187700 10 osd.7 19 send_incremental_map 18 -> 19 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:33:56.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~0 2019-12-07T19:33:56.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~285 size 0x285 (645) 2019-12-07T19:33:56.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:56.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676ee0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x558c54676fc0 sbid 0x0)) need 0x0~285 cache has 0x[0~285] 2019-12-07T19:33:56.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 = 645 2019-12-07T19:33:56.617+0000 7f8f07187700 10 osd.7 19 add_map_inc_bl 19 645 bytes 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 no pg, should create on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d58000 seq 2 2019-12-07T19:33:56.617+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 19 _make_pg 1.9s4 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9(unlocked)] enter Initial 2019-12-07T19:33:56.617+0000 7f8f04181700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.617+0000 7f8f04181700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.617+0000 7f8f04181700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.617+0000 7f8f04181700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.617+0000 7f8f04181700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 pg_epoch: 18 pg[1.9(unlocked)] enter NotTrimming 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.9s4_head fresh osr 0x558c54cf6000 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 4 up [1,4,5,0,7,6] acting [1,4,5,0,7,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.054323367s + 0s) 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.617+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.617+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.618930+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.617+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.618973+0000 1 0.000034 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.617+0000 7f8f04181700 1 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.618995+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f8f04181700 5 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.617+0000 7f8f04181700 20 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.617+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d58c00 seq 1 2019-12-07T19:33:56.617+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.9s4_head bits 4 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.9s4_head bits 4 = 0 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:90000000::::head# key 0x8480000000000000019000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:33:56.617+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1070 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._infover' <- _infover 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.617+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.617+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes 0x558c55148500 shared_blobs 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore.extentmap(0x558c55148608) update 4#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore.extentmap(0x558c55148608) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:90000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:33:56.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074625527s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.617+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(19..19 src has 1..19) v4 -- 0x558c5524dd40 con 0x558c54c5b400 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 19 handle_pg_create_info new pg pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 no pg, should create on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074625527s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7:1._attach_pg 1.9s4 0x558c5500a000 2019-12-07T19:33:56.618+0000 7f8f07187700 10 osd.7 19 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20f00 kv_submitted 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20f00 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20f00 finishing 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20f00 onodes 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20f00 done 2019-12-07T19:33:56.618+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c548a0000 con 0x558c54c5b400 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20f00 [] 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 19 _make_pg 1.0s3 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:33:56.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 19 identify_splits_and_merges 1.9s4 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 0x558c55148280 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 prepare 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7:1._prime_splits 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 18 -> 19 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 18 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7f8f04181700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7f8f04181700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074537935s int 16s) 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.021270630s -> ruub 36.053267305s 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0(unlocked)] enter Initial 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.053267305s 2019-12-07T19:33:56.618+0000 7f8f04181700 5 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.619413+0000 3 0.000053 2019-12-07T19:33:56.618+0000 7f8f04181700 5 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.618+0000 7f8f04181700 5 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.618+0000 7f8f04982700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.618+0000 7f8f04982700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f8f04982700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.618+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.618+0000 7f8f04982700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f8f04982700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x126ee~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x126ee 0x126ee~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x26ee 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6ee 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc5b and padding block with 0x3a5 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 pg_epoch: 18 pg[1.0(unlocked)] enter NotTrimming 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.0s3_head fresh osr 0x558c54cf6c80 2019-12-07T19:33:56.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d58f00 seq 2 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 3 up [1,5,2,7,3,4] acting [1,5,2,7,3,4] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes shared_blobs 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.055039772s + 0s) 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:33:56.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:33:56.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 4 ==== PGlog(1.6s2 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x558c551dc000 con 0x558c54c5bc00 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.6s2 epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info 2019-12-07T19:33:56.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.619643+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 no pg, should create on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.619685+0000 1 0.000033 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 19 _make_pg 1.6s2 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.618+0000 7f8f04982700 1 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6(unlocked)] enter Initial 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.619706+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7f8f04982700 5 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.618+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f8f05984700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12c5b 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d21200 seq 1 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.0s3_head bits 4 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.0s3_head bits 4 = 0 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000400210s (0.000000094s flush + 0.000400116s kv commit) 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:00000000::::head# key 0x8380000000000000010000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.617+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) r -2 v.len 0 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:56.618+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1071 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:56.618+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12c5b~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._infover' <- _infover 2019-12-07T19:33:56.618+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12c5b 0x12c5b~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x2c5b 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc5b 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc0 and padding block with 0xf40 2019-12-07T19:33:56.618+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:33:56.618+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21200 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21200 onodes 0x558c55148780 shared_blobs 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore.extentmap(0x558c55148888) update 3#1:00000000::::head# 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore.extentmap(0x558c55148888) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:00000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.618+0000 7f8f05984700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f8f05984700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 prepare 2019-12-07T19:33:56.618+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21200 2019-12-07T19:33:56.618+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 io_done 2019-12-07T19:33:56.618+0000 7f8f05984700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f8f05984700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 pg_epoch: 18 pg[1.6(unlocked)] enter NotTrimming 2019-12-07T19:33:56.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.6s2_head fresh osr 0x558c54cf77c0 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 2 up [3,6,7,1,2,5] acting [3,6,7,1,2,5] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558c54ea3ce0] 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.055390787s + 0s) 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.618+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.618+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.619994+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.618+0000 7f8f06986700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 19 handle_pg_create_info new pg pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.618+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7:0._attach_pg 1.0s3 0x558c5500d000 2019-12-07T19:33:56.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 19 identify_splits_and_merges 1.0s3 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7:0._prime_splits 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x130c0 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.618+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:33:56.618+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: _activate_committed 19 peer_activated now 7(0) last_interval_started 0 last_epoch_started 0 same_interval_since 18 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:33:56.618+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 0x558c55148500 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000312231s (0.000000091s flush + 0.000312140s kv commit) 2019-12-07T19:33:56.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 kv_submitted 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.620095+0000 1 0.000036 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x130c0~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x130c0 0x130c0~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x30c0 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc0 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa7b and padding block with 0x585 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00> 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:33:56.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.618+0000 7f8f05984700 1 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 18 -> 19 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.620151+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 18 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:33:56.618+0000 7f8f05984700 5 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.618+0000 7f8f05984700 20 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.618+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:56.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:33:56.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d59200 seq 1 2019-12-07T19:33:56.618+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.6s2_head bits 4 2019-12-07T19:33:56.618+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.6s2_head bits 4 = 0 2019-12-07T19:33:56.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:60000000::::head# key 0x8280000000000000016000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.618+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.619+0000 7f8f04982700 20 osd.7 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.619+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1072 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f8f08189700 15 osd.7 19 enqueue_peering_evt 1.9s4 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074625527s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7f8f08189700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f8f04982700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._infover' <- _infover 2019-12-07T19:33:56.619+0000 7f8f04982700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes 0x558c55148a00 shared_blobs 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore.extentmap(0x558c55148b08) update 2#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore.extentmap(0x558c55148b08) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:60000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 19 handle_pg_create_info new pg pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7:6._attach_pg 1.6s2 0x558c54d5c000 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 19 identify_splits_and_merges 1.6s2 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7:6._prime_splits 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558c5500a000 2019-12-07T19:33:56.619+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f8f08189700 10 osd.7 19 send_incremental_map 18 -> 19 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:33:56.619+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(19..19 src has 1..19) v4 -- 0x558c5501afc0 con 0x558c54c64000 2019-12-07T19:33:56.619+0000 7f8f08189700 10 osd.7 19 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.619+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c548a1c00 con 0x558c54c64000 2019-12-07T19:33:56.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 18 -> 19 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074625527s int 16s) 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 18 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.619+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.021270630s -> ruub 36.053354897s 2019-12-07T19:33:56.619+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.053354897s 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.619+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:33:56.619+0000 7f8f04982700 5 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.620597+0000 3 0.000042 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.619+0000 7f8f04982700 5 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.619+0000 7f8f04982700 5 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.619+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7f8f05984700 10 merge_log log((0'0,0'0], crt=0'0) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7f8f05984700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:56.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d21500 seq 2 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.075844307s int 16s) 2019-12-07T19:33:56.619+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.021846865s -> ruub 36.053997442s 2019-12-07T19:33:56.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.053997442s 2019-12-07T19:33:56.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.619+0000 7f8f05984700 5 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.620662+0000 3 0.000082 2019-12-07T19:33:56.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes shared_blobs 2019-12-07T19:33:56.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7f8f05984700 5 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:33:56.619+0000 7f8f05984700 5 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:33:56.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x13a7b 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000628047s (0.000000087s flush + 0.000627960s kv commit) 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d21b00 seq 2 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:56.619+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x13a7b~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes shared_blobs 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x13a7b 0x13a7b~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x3a7b 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa7b 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x436 and padding block with 0xbca 2019-12-07T19:33:56.619+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:33:56.619+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58000,0x558c54d21200> 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 kv_submitted 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21200 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21200 finishing 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21200 onodes 0x558c55148780 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21200 done 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 kv_submitted 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21200 [] 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14436 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000146565s (0.000000085s flush + 0.000146480s kv commit) 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200,0x558c54d21500> 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 0x558c55148a00 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 kv_queued 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14436~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14436 0x14436~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x4436 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x436 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x89b and padding block with 0x765 2019-12-07T19:33:56.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1489b 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000109205s (0.000000076s flush + 0.000109129s kv commit) 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:33:56.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:33:56.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.619+0000 7f8f05984700 15 osd.7 19 enqueue_peering_evt 1.6s2 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558c54d5c000 2019-12-07T19:33:56.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.619+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 19 send_incremental_map 18 -> 19 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:33:56.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(19..19 src has 1..19) v4 -- 0x558c5501bd40 con 0x558c54c5bc00 2019-12-07T19:33:56.619+0000 7f8f05984700 10 osd.7 19 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.6s0 1.6s0 1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c548a2300 con 0x558c54c5bc00 2019-12-07T19:33:56.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f8f06185700 15 osd.7 19 enqueue_peering_evt 1.ds2 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f8f06185700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558c55141000 2019-12-07T19:33:56.619+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f8f06185700 10 osd.7 19 send_incremental_map 18 -> 19 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:33:56.619+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(19..19 src has 1..19) v4 -- 0x558c54d2f680 con 0x558c54c19800 2019-12-07T19:33:56.619+0000 7f8f06185700 10 osd.7 19 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.619+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.ds0 1.ds0 1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c548a2a00 con 0x558c54c19800 2019-12-07T19:33:56.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f8f0898a700 15 osd.7 19 enqueue_peering_evt 1.0s3 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558c5500d000 2019-12-07T19:33:56.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.620+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.0s0 1.0s0 1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c548a3100 con 0x558c54c64000 2019-12-07T19:33:56.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 9 ==== PGlog(1.7s4 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x558c551dd800 con 0x558c54c64000 2019-12-07T19:33:56.621+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.7s4 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info 2019-12-07T19:33:56.621+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 no pg, should create on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 19 _make_pg 1.7s4 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7(unlocked)] enter Initial 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 18 pg[1.7(unlocked)] enter NotTrimming 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.7s4_head fresh osr 0x558c54cf7b80 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 4 up [1,6,4,0,7,2] acting [1,6,4,0,7,2] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.058865375s + 0s) 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.622+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.623483+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.623517+0000 1 0.000029 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.622+0000 7f8f0117b700 1 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.623528+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d58c00 seq 1 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.7s4_head bits 4 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.7s4_head bits 4 = 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:e0000000::::head# key 0x848000000000000001e000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1073 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._infover' <- _infover 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes 0x558c55148c80 shared_blobs 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore.extentmap(0x558c55148d88) update 4#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore.extentmap(0x558c55148d88) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:e0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 19 handle_pg_create_info new pg pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7:7._attach_pg 1.7s4 0x558c54d5f000 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 19 identify_splits_and_merges 1.7s4 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7:7._prime_splits 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 18 -> 19 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 18 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.075985074s int 16s) 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.021270630s -> ruub 36.054714444s 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.054714444s 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.623700+0000 3 0.000026 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.622+0000 7f8f0117b700 5 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d21500 seq 2 2019-12-07T19:33:56.622+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes shared_blobs 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:33:56.622+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 2 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1489b~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1489b 0x1489b~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x489b 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x89b 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe08 and padding block with 0x1f8 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14e08 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000090071s (0.000000091s flush + 0.000089980s kv commit) 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 0x558c55148c80 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 kv_queued 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14e08~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14e08 0x14e08~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x4e08 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe08 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x26d and padding block with 0xd93 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1526d 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000108706s (0.000000096s flush + 0.000108610s kv commit) 2019-12-07T19:33:56.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21500> 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:33:56.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.622+0000 7f8f05183700 15 osd.7 19 enqueue_peering_evt 1.7s4 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558c54d5f000 2019-12-07T19:33:56.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f8f05183700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7f8f05183700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.622+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c548a0000 con 0x558c54c64000 2019-12-07T19:33:56.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 7 ==== pg_info2(1.4s0 1.4s0 1.4s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x558c548a3100 con 0x558c54c5b400 2019-12-07T19:33:56.623+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(5) info: 1.4s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.623+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(5) info: 1.4s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(5) info: 1.4s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(5) info: 1.4s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.623+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(5) info: 1.4s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.0 ruub 0s 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.052429711s 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.052429711s) 2019-12-07T19:33:56.623+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.4(5) activated and committed 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.623+0000 7f8f06986700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:2 2019-12-07T19:33:56.623+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 8 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x558c5501afc0 con 0x558c54c5b400 2019-12-07T19:33:56.623+0000 7f8f1199c700 20 osd.7 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.623+0000 7f8f1199c700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:56.623+0000 7f8f1199c700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:56.623+0000 7f8f1199c700 20 osd.7 19 _dispatch 0x558c5501afc0 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.623+0000 7f8f1199c700 3 osd.7 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.623+0000 7f8f1199c700 10 osd.7 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 4 ==== pg_info2(1.4s0 1.4s0 1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (crc 0 0 0) 0x558c548a2300 con 0x558c54c64400 2019-12-07T19:33:56.624+0000 7f8f23c56700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(1) info: 1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.624+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(1) info: 1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(1) info: 1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(1) info: 1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.624+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 5 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (crc 0 0 0) 0x558c55275d40 con 0x558c54c64400 2019-12-07T19:33:56.624+0000 7f8f1199c700 20 osd.7 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f8f1199c700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:56.624+0000 7f8f1199c700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:56.624+0000 7f8f1199c700 20 osd.7 19 _dispatch 0x558c55275d40 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(1) info: 1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.624+0000 7f8f1199c700 3 osd.7 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7f8f1199c700 10 osd.7 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.0 ruub 36.052429711s 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.052429711s 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.052429711s) 2019-12-07T19:33:56.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 10 ==== pg_info2(1.4s0 1.4s0 1.4s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (crc 0 0 0) 0x558c548a3100 con 0x558c54c64000 2019-12-07T19:33:56.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.0(1) activated and committed 2019-12-07T19:33:56.624+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.4s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.4s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.4s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 11 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (crc 0 0 0) 0x558c5501bd40 con 0x558c54c64000 2019-12-07T19:33:56.624+0000 7f8f1199c700 20 osd.7 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f1199c700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:56.624+0000 7f8f1199c700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:56.624+0000 7f8f1199c700 20 osd.7 19 _dispatch 0x558c5501bd40 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.624+0000 7f8f1199c700 3 osd.7 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7f8f1199c700 10 osd.7 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.624+0000 7f8f0297e700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:3 2019-12-07T19:33:56.624+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.4s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.4s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.0 ruub 36.052429711s 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.052429711s 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.052429711s) 2019-12-07T19:33:56.624+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.1(2) activated and committed 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.624+0000 7f8f06986700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:4 2019-12-07T19:33:56.624+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 7 ==== pg_info2(1.4s0 1.4s0 1.4s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (crc 0 0 0) 0x558c548a2a00 con 0x558c54c19800 2019-12-07T19:33:56.626+0000 7f8f24c58700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.4s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.626+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.4s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.4s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.4s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 8 ==== pg_info2(1.ds2 1.ds2 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c548a2a00 con 0x558c54c19800 2019-12-07T19:33:56.626+0000 7f8f24c58700 15 osd.7 19 enqueue_peering_evt 1.ds2 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.626+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.4s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.0 ruub 36.052429711s 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) pg 0x558c55141000 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.626+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 9 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (crc 0 0 0) 0x558c54d2f680 con 0x558c54c19800 2019-12-07T19:33:56.626+0000 7f8f1199c700 20 osd.7 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 36.052429711s 2019-12-07T19:33:56.626+0000 7f8f1199c700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:56.626+0000 7f8f1199c700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:56.626+0000 7f8f1199c700 20 osd.7 19 _dispatch 0x558c54d2f680 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.626+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.052429711s 2019-12-07T19:33:56.626+0000 7f8f1199c700 3 osd.7 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.626+0000 7f8f1199c700 10 osd.7 19 no new maps here, dropping 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.052429711s) 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.2(4) activated and committed 2019-12-07T19:33:56.626+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:33:56.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:56.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d58300 seq 3 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.626+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:56.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:56.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes shared_blobs 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:33:56.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:33:56.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1526d~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1526d 0x1526d~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x526d 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x26d 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x683 and padding block with 0x97d 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.626+0000 7f8f0297e700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:5 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15683 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.626+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000073572s (0.000000083s flush + 0.000073489s kv commit) 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300> 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 10 ==== pg_info2(1.ds2 1.ds2 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c548a2a00 con 0x558c54c19800 2019-12-07T19:33:56.626+0000 7f8f24c58700 15 osd.7 19 enqueue_peering_evt 1.ds2 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.626+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) pg 0x558c55141000 2019-12-07T19:33:56.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.626+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:33:56.626+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d58f00 seq 4 2019-12-07T19:33:56.626+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.626+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:56.626+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes shared_blobs 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.626+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:33:56.626+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15683~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15683 0x15683~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x5683 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x683 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xad0 and padding block with 0x530 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15ad0 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119793s (0.000000060s flush + 0.000119733s kv commit) 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00> 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:33:56.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 12 ==== pg_info2(1.7s4 1.7s4 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c64000 2019-12-07T19:33:56.627+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.7s4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.627+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) pg 0x558c54d5f000 2019-12-07T19:33:56.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.627+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:33:56.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d58000 seq 3 2019-12-07T19:33:56.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:56.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:33:56.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:33:56.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15ad0~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15ad0 0x15ad0~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x5ad0 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xad0 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xee6 and padding block with 0x11a 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15ee6 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000080472s (0.000000103s flush + 0.000080369s kv commit) 2019-12-07T19:33:56.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58000> 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 13 ==== pg_info2(1.7s4 1.7s4 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c64000 2019-12-07T19:33:56.628+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.7s4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.628+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) pg 0x558c54d5f000 2019-12-07T19:33:56.628+0000 7f8f05183700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.628+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:33:56.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:56.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59800 seq 4 2019-12-07T19:33:56.628+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:56.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:56.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59800 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59800 onodes shared_blobs 2019-12-07T19:33:56.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 prepare 2019-12-07T19:33:56.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59800 2019-12-07T19:33:56.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 io_done 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15ee6~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15ee6 0x15ee6~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x5ee6 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xee6 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x333 and padding block with 0xccd 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x16333 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000087559s (0.000000062s flush + 0.000087497s kv commit) 2019-12-07T19:33:56.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59800> 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 kv_submitted 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59800 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 finishing 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59800 onodes 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59800 done 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59800 [] 2019-12-07T19:33:56.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.628+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.659+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.702+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 4 ==== pg_info2(1.4s0 1.4s0 1.4s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x558c548a1c00 con 0x558c54c19c00 2019-12-07T19:33:56.702+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(3) info: 1.4s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.702+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(3) info: 1.4s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(3) info: 1.4s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(3) info: 1.4s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s)) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.702+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 5 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x558c5524dd40 con 0x558c54c19c00 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(3) info: 1.4s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.052429711s) 2019-12-07T19:33:56.702+0000 7f8f1199c700 20 osd.7 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.702+0000 7f8f1199c700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:56.702+0000 7f8f1199c700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:56.702+0000 7f8f1199c700 20 osd.7 19 _dispatch 0x558c5524dd40 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.0 ruub 36.052429711s 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:33:56.702+0000 7f8f1199c700 3 osd.7 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 36.052429711s 2019-12-07T19:33:56.702+0000 7f8f1199c700 10 osd.7 19 no new maps here, dropping 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 36.052429711s 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.052429711s 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 36.052429711s (sent 36.052429711s) 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.5(3) activated and committed 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.702+0000 7f8f06986700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:6 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: all_activated_and_committed 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] state: mnow 20.139343933s >= prior_readable_until_ub 0s 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 19 send_pg_created 2019-12-07T19:33:56.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_pg_created(1.4) v1 -- 0x558c54d39e00 con 0x558c5476d000 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] share_pg_info 2019-12-07T19:33:56.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c548a3800 con 0x558c54c64400 2019-12-07T19:33:56.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c54e98000 con 0x558c54c64000 2019-12-07T19:33:56.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c54e98700 con 0x558c54c19800 2019-12-07T19:33:56.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c54e98e00 con 0x558c54c5b400 2019-12-07T19:33:56.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558c54e99500 con 0x558c54c19c00 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.702+0000 7f8f06986700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:7 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] check_local 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_backfill does not need backfill 2019-12-07T19:33:56.702+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] activate all replicas clean, no recovery 2019-12-07T19:33:56.702+0000 7f8f06986700 15 osd.7 19 enqueue_peering_evt 1.4s0 epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) 2019-12-07T19:33:56.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.702+0000 7f8f06986700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:8 2019-12-07T19:33:56.702+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:56.703+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] agent_stop 2019-12-07T19:33:56.703+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:33:56.703+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:56.703+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d21b00 seq 4 2019-12-07T19:33:56.703+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:56.703+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.703+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:56.703+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5258 (1 ios * 4000 + 1258 bytes) 2019-12-07T19:33:56.703+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes shared_blobs 2019-12-07T19:33:56.703+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.703+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:33:56.703+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:33:56.703+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:33:56.703+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) pg 0x558c548ab000 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1089 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] exit Started/Primary/Active/Activating 2019-12-07T19:33:56.704299+0000 8 0.001459 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1112 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] enter Started/Primary/Active/Recovered 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1112 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x16333~458 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x16333 0x16333~458 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x6333 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x333 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 0(1),1(2),2(4),4(5),5(3),7(0) upset 0(1),1(2),2(4),4(5),5(3),7(0) acting_recovery_backfill 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x78b and padding block with 0x875 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 7(0) primary objects 0 missing 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 0(1) objects 0 missing 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(5) objects 0 missing 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 0(1) missing= 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1678b 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118079s (0.000000172s flush + 0.000117907s kv commit) 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:56.703+0000 7f8f0297e700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:9 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.703+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] exit Started/Primary/Active/Recovered 2019-12-07T19:33:56.704435+0000 0 0.000000 2019-12-07T19:33:56.703+0000 7f8f0297e700 5 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] enter Started/Primary/Active/Clean 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] share_pg_info 2019-12-07T19:33:56.703+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558c548a2a00 con 0x558c54c64400 2019-12-07T19:33:56.703+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558c548a0000 con 0x558c54c64000 2019-12-07T19:33:56.703+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558c548a3100 con 0x558c54c19800 2019-12-07T19:33:56.703+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558c54e98700 con 0x558c54c5b400 2019-12-07T19:33:56.703+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558c54e98000 con 0x558c54c19c00 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.703+0000 7f8f0297e700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:10 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_recovery 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] clear_recovery_state 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d58c00 seq 5 2019-12-07T19:33:56.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:33:56.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.703+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1678b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1678b 0x1678b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x678b 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x78b 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbd8 and padding block with 0x428 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x16bd8 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.703+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000068176s (0.000000078s flush + 0.000068098s kv commit) 2019-12-07T19:33:56.703+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:56.703+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:33:56.703+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.703+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.703+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] _finish_recovery 2019-12-07T19:33:56.703+0000 7f8f06986700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] purge_strays 2019-12-07T19:33:56.703+0000 7f8f06986700 20 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.703+0000 7f8f06986700 15 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:11 2019-12-07T19:33:56.703+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 5 ==== pg_info2(1.6s2 1.6s2 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c54e99500 con 0x558c54c5bc00 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.6s2 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) pg 0x558c54d5c000 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 6 ==== pg_info2(1.6s2 1.6s2 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c54e99500 con 0x558c54c5bc00 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.6s2 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d58000 seq 3 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 14 ==== pg_info2(1.9s4 1.9s4 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c54e99500 con 0x558c54c64000 2019-12-07T19:33:56.708+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.9s4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:33:56.708+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 15 ==== pg_info2(1.0s3 1.0s3 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c54e99500 con 0x558c54c64000 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:33:56.708+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) pg 0x558c5500a000 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.0s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) pg 0x558c54d5c000 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f08189700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 16 ==== pg_info2(1.9s4 1.9s4 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c54e99500 con 0x558c54c64000 2019-12-07T19:33:56.708+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.9s4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d59800 seq 3 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x16bd8~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x16bd8 0x16bd8~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x6bd8 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbd8 2019-12-07T19:33:56.708+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfee and padding block with 0x12 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:56.708+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d59b00 seq 4 2019-12-07T19:33:56.708+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.708+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59800 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 17 ==== pg_info2(1.0s3 1.0s3 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x558c54e99500 con 0x558c54c64000 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 prepare 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.0s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s) 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e19) pg 0x558c5500d000 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59800 2019-12-07T19:33:56.708+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 io_done 2019-12-07T19:33:56.708+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s) 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:33:56.708+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) pg 0x558c5500a000 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x16fee 2019-12-07T19:33:56.708+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.708+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 9 ==== pg_info2(1.3s4 1.3s4 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558c54e99500 con 0x558c54c5b400 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.3s4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54ea0000 seq 3 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58000> 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000098149s (0.000000148s flush + 0.000098001s kv commit) 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.708+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.708+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 kv_queued 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.708+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea0900 seq 4 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:56.708+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:33:56.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 10 ==== pg_info2(1.3s4 1.3s4 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558c54e99500 con 0x558c54c5b400 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:33:56.708+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:33:56.708+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:56.708+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f24457700 15 osd.7 19 enqueue_peering_evt 1.3s4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:33:56.708+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x16fee~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e19) pg 0x558c5500d000 2019-12-07T19:33:56.708+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:33:56.708+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x16fee 0x16fee~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x6fee 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfee 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x83a and padding block with 0x7c6 2019-12-07T19:33:56.708+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.0s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s) 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) pg 0x558c5513e000 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54ea0300 seq 4 2019-12-07T19:33:56.708+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea1200 seq 3 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:33:56.708+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:56.708+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:33:56.708+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) pg 0x558c5513e000 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea1500 seq 4 2019-12-07T19:33:56.708+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:33:56.708+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1783a 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000173648s (0.000000087s flush + 0.000173561s kv commit) 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59800,0x558c54d59b00> 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 kv_submitted 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59800 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 finishing 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59800 onodes 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59800 done 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 2 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 kv_queued 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59800 [] 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1990 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2112 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3190 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:33:56.708+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4213 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 2 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5291 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5314 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5314 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1783a~14c2 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1783a 0x1783a~14c2 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x783a 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x83a 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.708+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcfc and padding block with 0x304 2019-12-07T19:33:56.708+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x18cfc 2019-12-07T19:33:56.709+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.709+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:56.709+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:56.709+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:56.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 5 cleaned 0 in 0.000164364s (0.000000100s flush + 0.000164264s kv commit) 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:56.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0000,0x558c54ea0900,0x558c54ea0300,0x558c54ea1200,0x558c54ea1500> 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:33:56.709+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:33:56.709+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.709+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:56.709+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 kv_submitted 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:33:56.709+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:33:56.709+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:33:56.709+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:56.709+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:56.709+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.709+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.759+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.809+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.859+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.909+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.959+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.010+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.053+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.6 172.21.15.109:0/36937 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.019106793s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250a00 con 0x558c53ae4c00 2019-12-07T19:33:57.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.6 172.21.15.109:0/36937 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.019106793s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1ee00 con 0x558c552d0400 2019-12-07T19:33:57.053+0000 7f8f24457700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018702385s,0.019106793s]) 2019-12-07T19:33:57.053+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.490455463s delta_ub 18446744073.690849231s) v5 -- 0x558c55250200 con 0x558c53ae4c00 2019-12-07T19:33:57.053+0000 7f8f24457700 10 osd.7 19 maybe_share_map con 0x558c54c19400 v2:172.21.15.109:6818/36937 map epoch 18 -> 19 (as per caller) 2019-12-07T19:33:57.053+0000 7f8f23c56700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018670547s,0.019106793s]) 2019-12-07T19:33:57.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.490487301s delta_ub 18446744073.690881069s) v5 -- 0x558c55250400 con 0x558c552d0400 2019-12-07T19:33:57.060+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.071+0000 7f8f0097a700 5 osd.7 19 heartbeat osd_stat(store_statfs(0x163ffd2000/0x40000000/0x1680000000, data 0xda65/0x2c000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,6] op hist []) 2019-12-07T19:33:57.071+0000 7f8f0097a700 20 osd.7 19 check_full_status cur ratio 1.94973e-06, physical ratio 1.94973e-06, new state none 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690600417s) v5 -- 0x558c55250c00 con 0x558c5476d400 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690600417s) v5 -- 0x558c55250e00 con 0x558c551b6800 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.688280986s) v5 -- 0x558c55251000 con 0x558c54c9e000 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.688280986s) v5 -- 0x558c55251200 con 0x558c54c9e400 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.673477020s) v5 -- 0x558c55251400 con 0x558c551b7800 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.673477020s) v5 -- 0x558c54d76000 con 0x558c551b7c00 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.687704751s) v5 -- 0x558c54d76200 con 0x558c552c2000 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.687704751s) v5 -- 0x558c54d76400 con 0x558c552c2400 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.696004079s) v5 -- 0x558c54d76600 con 0x558c552c2800 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.696004079s) v5 -- 0x558c54d76800 con 0x558c552c2c00 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.678986158s) v5 -- 0x558c54d76a00 con 0x558c54c65800 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.678986158s) v5 -- 0x558c54d76c00 con 0x558c54c65c00 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690881069s) v5 -- 0x558c54d76e00 con 0x558c551b7000 2019-12-07T19:33:57.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690881069s) v5 -- 0x558c54d77000 con 0x558c551b7400 2019-12-07T19:33:57.071+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 1 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.538941185s delta_ub 0.031038785s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c54c65800 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772502+0000 -> 2019-12-07T19:33:57.072687+0000 last_rx_front 2019-12-07T19:33:54.772537+0000 2019-12-07T19:33:57.071+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6822/36937 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527128116s delta_ub 0.019225716s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c551b7000 2019-12-07T19:33:57.071+0000 7f8f23c56700 10 osd.7 19 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 18 -> 19 (as per caller) 2019-12-07T19:33:57.071+0000 7f8f23c56700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030814303s,0.031038785s]) 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772683+0000 -> 2019-12-07T19:33:57.072706+0000 last_rx_front 2019-12-07T19:33:54.772519+0000 2019-12-07T19:33:57.071+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.521823704s delta_ub 0.013921304s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552c2c00 2019-12-07T19:33:57.071+0000 7f8f24c58700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018982550s,0.019225716s]) 2019-12-07T19:33:57.071+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 1 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.538953421s delta_ub 0.031051021s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c54c65c00 2019-12-07T19:33:57.071+0000 7f8f24457700 25 osd.7 19 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772464+0000 last_rx_front 2019-12-07T19:33:54.772448+0000 -> 2019-12-07T19:33:57.072719+0000 2019-12-07T19:33:57.071+0000 7f8f24457700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013665195s,0.013921304s]) 2019-12-07T19:33:57.071+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.521879135s delta_ub 0.013976735s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c552c2800 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072687+0000 last_rx_front 2019-12-07T19:33:54.772537+0000 -> 2019-12-07T19:33:57.072733+0000 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:57.072463+0000) and older pending ping(s) 2019-12-07T19:33:57.071+0000 7f8f23c56700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030780959s,0.031051021s]) 2019-12-07T19:33:57.071+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.530229271s delta_ub 0.022326871s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c552c2000 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772464+0000 -> 2019-12-07T19:33:57.072765+0000 last_rx_front 2019-12-07T19:33:57.072719+0000 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:57.072463+0000) and older pending ping(s) 2019-12-07T19:33:57.071+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.544431274s delta_ub 0.036528874s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c551b7800 2019-12-07T19:33:57.071+0000 7f8f24c58700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013674784s,0.013976735s]) 2019-12-07T19:33:57.071+0000 7f8f24457700 25 osd.7 19 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772638+0000 -> 2019-12-07T19:33:57.072787+0000 last_rx_front 2019-12-07T19:33:54.772726+0000 2019-12-07T19:33:57.071+0000 7f8f24457700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022002871s,0.022326871s]) 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772696+0000 -> 2019-12-07T19:33:57.072797+0000 last_rx_front 2019-12-07T19:33:54.772667+0000 2019-12-07T19:33:57.071+0000 7f8f23c56700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036194416s,0.036528874s]) 2019-12-07T19:33:57.071+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.544457942s delta_ub 0.036555542s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c551b7c00 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072797+0000 last_rx_front 2019-12-07T19:33:54.772667+0000 -> 2019-12-07T19:33:57.072809+0000 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:57.072463+0000) and older pending ping(s) 2019-12-07T19:33:57.071+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6820/36937 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527279822s delta_ub 0.019377422s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55270400 con 0x558c551b7400 2019-12-07T19:33:57.071+0000 7f8f24c58700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036208935s,0.036555542s]) 2019-12-07T19:33:57.071+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 1 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.529638598s delta_ub 0.021736198s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c54c9e000 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772556+0000 -> 2019-12-07T19:33:57.072827+0000 last_rx_front 2019-12-07T19:33:54.772753+0000 2019-12-07T19:33:57.071+0000 7f8f23c56700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021372403s,0.021736198s]) 2019-12-07T19:33:57.071+0000 7f8f24457700 25 osd.7 19 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072706+0000 last_rx_front 2019-12-07T19:33:54.772519+0000 -> 2019-12-07T19:33:57.072836+0000 2019-12-07T19:33:57.071+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527448320s delta_ub 0.019545920s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c551b6800 2019-12-07T19:33:57.071+0000 7f8f24457700 25 osd.7 19 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:57.072463+0000) and older pending ping(s) 2019-12-07T19:33:57.071+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.530254226s delta_ub 0.022351826s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c552c2400 2019-12-07T19:33:57.071+0000 7f8f24457700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.019004481s,0.019377422s]) 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772649+0000 last_rx_front 2019-12-07T19:33:54.772626+0000 -> 2019-12-07T19:33:57.072855+0000 2019-12-07T19:33:57.071+0000 7f8f24c58700 10 osd.7 19 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 18 -> 19 (as per caller) 2019-12-07T19:33:57.071+0000 7f8f24c58700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019153943s,0.019545920s]) 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072787+0000 last_rx_front 2019-12-07T19:33:54.772726+0000 -> 2019-12-07T19:33:57.072868+0000 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:57.072463+0000) and older pending ping(s) 2019-12-07T19:33:57.071+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 1 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.529685229s delta_ub 0.021782829s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55250000 con 0x558c54c9e400 2019-12-07T19:33:57.071+0000 7f8f23c56700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021946415s,0.022351826s]) 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072827+0000 last_rx_front 2019-12-07T19:33:54.772753+0000 -> 2019-12-07T19:33:57.072882+0000 2019-12-07T19:33:57.071+0000 7f8f24c58700 25 osd.7 19 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:57.072463+0000) and older pending ping(s) 2019-12-07T19:33:57.071+0000 7f8f24c58700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021363773s,0.021782829s]) 2019-12-07T19:33:57.071+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527407750s delta_ub 0.019505350s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c5476d400 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:54.772649+0000 -> 2019-12-07T19:33:57.072893+0000 last_rx_front 2019-12-07T19:33:57.072855+0000 2019-12-07T19:33:57.071+0000 7f8f23c56700 25 osd.7 19 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:57.072463+0000) and older pending ping(s) 2019-12-07T19:33:57.071+0000 7f8f23c56700 20 osd.7 19 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019074741s,0.019505350s]) 2019-12-07T19:33:57.110+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.160+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.210+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.260+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.310+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.360+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.411+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.446+0000 7f8f21451700 10 osd.7 19 tick 2019-12-07T19:33:57.446+0000 7f8f21451700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:57.446+0000 7f8f21451700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:57.446+0000 7f8f21451700 20 osd.7 19 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:57.461+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.511+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.561+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 10 osd.7 19 tick_without_osd_lock 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072893+0000 last_rx_front 2019-12-07T19:33:57.072855+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072827+0000 last_rx_front 2019-12-07T19:33:57.072882+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072797+0000 last_rx_front 2019-12-07T19:33:57.072809+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072787+0000 last_rx_front 2019-12-07T19:33:57.072868+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072765+0000 last_rx_front 2019-12-07T19:33:57.072719+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072687+0000 last_rx_front 2019-12-07T19:33:57.072733+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072706+0000 last_rx_front 2019-12-07T19:33:57.072836+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 20 osd.7 19 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 20 osd.7 19 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 20 osd.7 19 scrub_load_below_threshold loadavg per cpu 0.05625 < max 5 = yes 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 20 osd.7 19 sched_scrub load_is_low=1 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 10 osd.7 19 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:33:57.599426+0000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 20 osd.7 19 sched_scrub done 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 10 osd.7 19 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 20 osd.7 19 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:57.598+0000 7f8f1fbbe700 10 osd.7 19 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:57.611+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:33:57.611+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:33:57.611+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 9 0x2000~1000 2019-12-07T19:33:57.611+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 253d8353 2019-12-07T19:33:57.611+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 9 0x1b000~1000 2019-12-07T19:33:57.611+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 5f344e98 2019-12-07T19:33:57.611+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.611+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:33:57.611+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:33:57.662+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.711+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.762+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.812+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.862+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.912+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.925+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:33:57.962+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.013+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.026+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:33:58.063+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.113+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.163+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.213+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.263+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.280+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9ec00 0x558c54d04100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.280+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9ec00 0x558c54d04100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.281+0000 7f8f24457700 10 osd.7 19 ms_handle_authentication new session 0x558c54c0b1e0 con 0x558c54c9ec00 entity client.admin addr 2019-12-07T19:33:58.281+0000 7f8f24457700 10 osd.7 19 ms_handle_authentication session 0x558c54c0b1e0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.281+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/4040951113 conn(0x558c54c9ec00 0x558c54d04100 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=client.4346 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:58.281+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4346 172.21.15.131:0/4040951113 1 ==== command(tid 15: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c54d0e3c0 con 0x558c54c9ec00 2019-12-07T19:33:58.281+0000 7f8f164bb700 20 osd.7 19 OSD::ms_dispatch: command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:58.281+0000 7f8f164bb700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:58.281+0000 7f8f164bb700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:58.281+0000 7f8f164bb700 20 osd.7 19 _dispatch 0x558c54d0e3c0 command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:58.281+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/4040951113 -- command_reply(tid 15: 0 ) v1 -- 0x558c53a6fa40 con 0x558c54c9ec00 2019-12-07T19:33:58.305+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4346 172.21.15.131:0/4040951113 2 ==== command(tid 16: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c54d0f680 con 0x558c54c9ec00 2019-12-07T19:33:58.305+0000 7f8f164bb700 20 osd.7 19 OSD::ms_dispatch: command(tid 16: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:58.305+0000 7f8f164bb700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:58.305+0000 7f8f164bb700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:58.305+0000 7f8f164bb700 20 osd.7 19 _dispatch 0x558c54d0f680 command(tid 16: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:58.306+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/4040951113 -- command_reply(tid 16: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c53a6f180 con 0x558c54c9ec00 2019-12-07T19:33:58.308+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/4040951113 conn(0x558c54c9ec00 msgr2=0x558c54d04100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:58.308+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/4040951113 conn(0x558c54c9ec00 msgr2=0x558c54d04100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:58.308+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/4040951113 conn(0x558c54c9ec00 0x558c54d04100 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.308+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/4040951113 conn(0x558c54c9ec00 0x558c54d04100 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:58.308+0000 7f8f164bb700 2 osd.7 19 ms_handle_reset con 0x558c54c9ec00 session 0x558c54c0b1e0 2019-12-07T19:33:58.313+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.363+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.414+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.420+0000 7f8f21451700 10 osd.7 19 tick 2019-12-07T19:33:58.420+0000 7f8f21451700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:58.420+0000 7f8f21451700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:58.420+0000 7f8f21451700 20 osd.7 19 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:58.464+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.514+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.564+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.614+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 60810 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 10 osd.7 19 tick_without_osd_lock 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072893+0000 last_rx_front 2019-12-07T19:33:57.072855+0000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072827+0000 last_rx_front 2019-12-07T19:33:57.072882+0000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072797+0000 last_rx_front 2019-12-07T19:33:57.072809+0000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072787+0000 last_rx_front 2019-12-07T19:33:57.072868+0000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072765+0000 last_rx_front 2019-12-07T19:33:57.072719+0000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072687+0000 last_rx_front 2019-12-07T19:33:57.072733+0000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 25 osd.7 19 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:57.072463+0000 last_rx_back 2019-12-07T19:33:57.072706+0000 last_rx_front 2019-12-07T19:33:57.072836+0000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 20 osd.7 19 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 10 osd.7 19 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 20 osd.7 19 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:58.622+0000 7f8f1fbbe700 10 osd.7 19 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:58.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 19 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 14256+0+0 (crc 0 0 0) 0x558c54d84000 con 0x558c54c64000 2019-12-07T19:33:58.628+0000 7f8f24457700 15 osd.7 19 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000067 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.628+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.628+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 18 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (crc 0 0 0) 0x558c53a79440 con 0x558c54c64000 2019-12-07T19:33:58.628+0000 7f8f1199c700 20 osd.7 19 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.628+0000 7f8f1199c700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:58.628+0000 7f8f1199c700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:58.628+0000 7f8f1199c700 20 osd.7 19 _dispatch 0x558c53a79440 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.628+0000 7f8f1199c700 3 osd.7 19 handle_osd_map epochs [20,21], i have 19, src has [1,21] 2019-12-07T19:33:58.628+0000 7f8f1199c700 10 osd.7 19 handle_osd_map got inc map for epoch 20 2019-12-07T19:33:58.628+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:34c0e826:::osdmap.19:0# 0x0~0 2019-12-07T19:33:58.628+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:33:58.628+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:58.628+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279180 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c552791f0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.628+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c547404d0 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558c552792d0 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:33:58.628+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:34c0e826:::osdmap.19:0# 0x0~140c = 5132 2019-12-07T19:33:58.628+0000 7f8f1199c700 10 osd.7 19 add_map_bl 19 5132 bytes 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 osd.7 19 got_full_map 20, nothing requested 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 osd.7 19 handle_osd_map got inc map for epoch 21 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3180e826:::osdmap.20:0# 0x0~0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3180e826:::osdmap.20:0# key 0x7f7fffffffffffffff3180e8'&!osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3180e826:::osdmap.20:0# 0x0~0 = -2 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 osd.7 19 got_full_map 21, nothing requested 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 osd.7 19 handle_osd_map pg_num_history pg_num_history(e21 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {20={},21={}} 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 osd.7 19 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e21 [1,21] lci=[13,21]) 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f6300 seq 12 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cd1b972f:::inc_osdmap.20:0# key 0x7f7fffffffffffffffcd1b97'/!inc_osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1074 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 1 blobs 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x31000~1000/1000,1000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x31000~1000] 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3ce00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3ce00 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74070 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d3ce00) get_ref 0x0~24d Blob(0x558c54d3ce00 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74070 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d3ce00) get_ref init 0x1000, 1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c54d3ce00 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c54d74070 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d74088 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3180e826:::osdmap.20:0# key 0x7f7fffffffffffffff3180e8'&!osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3180e826:::osdmap.20:0# 0x0~140c 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1075 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3180e826:::osdmap.20:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~40c 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 2 blobs 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x32000~2000/1000,2000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x32000~2000] 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d741c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74150 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d741c0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74150) get_ref 0x0~1000 Blob(0x558c54d74150 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d741c0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74150) get_ref init 0x1000, 1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c54d74150 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d741c0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d741d8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d742a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74230 blob([!~1000,0x33000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d742a0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74230) get_ref 0x1000~40c Blob(0x558c54d74230 blob([!~1000,0x33000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c54d742a0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558c54d74230 blob([!~1000,0x33000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558c54d742a0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d742b8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x140c 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3180e826:::osdmap.20:0# 0x0~140c = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c21b972f:::inc_osdmap.21:0# key 0x7f7fffffffffffffffc21b97'/!inc_osdmap.21!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1076 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 1 blobs 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x34000~1000/1000,1000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x34000~1000] 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d743f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d743f0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74460 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d743f0) get_ref 0x0~24d Blob(0x558c54d743f0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74460 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d743f0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c54d743f0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c54d74460 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d74478 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3a80e826:::osdmap.21:0# key 0x7f7fffffffffffffff3a80e8'&!osdmap.21!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3a80e826:::osdmap.21:0# 0x0~140c 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1077 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3a80e826:::osdmap.21:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~40c 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 2 blobs 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x35000~2000/1000,2000,0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x35000~2000] 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d745b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74540 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d745b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74540) get_ref 0x0~1000 Blob(0x558c54d74540 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d745b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74540) get_ref init 0x1000, 1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c54d74540 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d745b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d745c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74620 blob([!~1000,0x36000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74690 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74620) get_ref 0x1000~40c Blob(0x558c54d74620 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c54d74690 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54d74620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558c54d74620 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558c54d74690 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d746a8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:33:58.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 8 ==== MOSDECSubOpWrite(1.6s2 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 18352+0+0 (crc 0 0 0) 0x558c54d84800 con 0x558c54c5bc00 2019-12-07T19:33:58.629+0000 7f8f24457700 15 osd.7 19 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000028 epoch 20 MOSDECSubOpWrite(1.6s2 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x140c 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e20) 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3a80e826:::osdmap.21:0# 0x0~140c = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e20) queued 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e20) pg 0x558c54d5c000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 0 blobs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 osd.7 19 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.000071 MOSDECSubOpWrite(1.6s2 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 osd.7 19 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 19 -> 20 (as per caller) 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 0 blobs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 41203 (7 ios * 4000 + 13203 bytes) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes 0x558c53a91900,0x558c54ce7400,0x558c54ce7680,0x558c54ce7900,0x558c55148f00,0x558c55276c80 shared_blobs 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [20'1 (0'0) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:10 2019-12-07T19:33:58.626763+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7508) update #-1:cd1b972f:::inc_osdmap.20:0# 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cd1b972f:::inc_osdmap.20:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7788) update #-1:3180e826:::osdmap.20:0# 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 20'1 (0'0) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:10 2019-12-07T19:33:58.626763+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7788) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3180e826:::osdmap.20:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7a08) update #-1:c21b972f:::inc_osdmap.21:0# 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c54ce7a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c21b972f:::inc_osdmap.21:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149008) update #-1:3a80e826:::osdmap.21:0# 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149008) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3a80e826:::osdmap.21:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.629+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 20'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[31000~6000] released 0x[] 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 freelist allocate 0x31000~6000 2019-12-07T19:33:58.629+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:33:58.629+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6300 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d58c00 seq 5 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:657ac0fe:::smithi13140796-4:head# key 0x828000000000000001657ac0fe21736d'ithi13140796-4!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1078 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# = 0 2019-12-07T19:33:58.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 2 keys 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 2 keys = 0 2019-12-07T19:33:58.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 0x0~4000 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:657ac0fe:::smithi13140796-4:head# 0x0~4000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~4000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58c00 2 blobs 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x4000/1000,4000,0 2019-12-07T19:33:58.629+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x37000~4000/1000,4000,0 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x37000~4000] 2019-12-07T19:33:58.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d750a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552656c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d750a0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552656c0 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) get_ref 0x0~2000 Blob(0x558c54d750a0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552656c0 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d750a0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552656c0 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c552656d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75110 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75110) get_ref 0x0~2000 Blob(0x558c54d75110 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d75110 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 7 ==== osd_map(20..20 src has 1..20) v4 ==== 633+0+0 (crc 0 0 0) 0x558c55195200 con 0x558c54c5bc00 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.630+0000 7f8f1199c700 20 osd.7 19 OSD::ms_dispatch: osd_map(20..20 src has 1..20) v4 2019-12-07T19:33:58.630+0000 7f8f1199c700 10 osd.7 19 do_waiters -- start 2019-12-07T19:33:58.630+0000 7f8f1199c700 10 osd.7 19 do_waiters -- finish 2019-12-07T19:33:58.630+0000 7f8f1199c700 20 osd.7 19 _dispatch 0x558c55195200 osd_map(20..20 src has 1..20) v4 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.630+0000 7f8f1199c700 3 osd.7 19 handle_osd_map epochs [20,20], i have 21, src has [1,20] 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.630+0000 7f8f1199c700 10 osd.7 19 no new maps here, dropping 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 0x0~4000 = 0 2019-12-07T19:33:58.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000020.00000000000000000001' <- 0000000020.00000000000000000001 2019-12-07T19:33:58.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 19 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.001416 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 19 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 30793 (3 ios * 4000 + 18793 bytes) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes 0x558c54d96280 shared_blobs 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96388) update 2#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96388) update inline shard 41 bytes from 2 extents 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:657ac0fe:::smithi13140796-4:head# is 459 (412 bytes onode + 2 bytes spanning blobs + 45 bytes inline extents) 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[37000~4000] released 0x[] 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 freelist allocate 0x37000~4000 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58c00 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.630+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.630+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d59800 seq 5 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9694a0e5:::smithi13140796-3:head# key 0x8480000000000000019694a0e521736d'ithi13140796-3!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.630+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1079 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# = 0 2019-12-07T19:33:58.630+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:33:58.630+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 osd.7 pg_epoch: 19 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 20'1 2019-12-07T19:33:58.630+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.630+0000 7f8f0197c700 10 osd.7 19 dequeue_op 0x558c54e78580 finish 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~3000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d59800 2 blobs 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.630+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate 0x3000/1000,3000,0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3b000~3000/1000,3000,0 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 aio_wait 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3b000~3000] 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3de30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c7e0 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3de30 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.blob(0x558c54d3c7e0) get_ref 0x0~2000 Blob(0x558c54d3c7e0 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3de30 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.blob(0x558c54d3c7e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d3c7e0 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d3de48 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74fc0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c620 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.blob(0x558c54d74fc0) get_ref 0x0~1000 Blob(0x558c54d74fc0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c620 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.blob(0x558c54d74fc0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~1000: 0x0~1000 Blob(0x558c54d74fc0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d3c638 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d96b08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 = 0 2019-12-07T19:33:58.630+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.630+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59800 cost 26697 (3 ios * 4000 + 14697 bytes) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59800 onodes 0x558c54d96a00 shared_blobs 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d96b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d96b08) update inline shard 38 bytes from 2 extents 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 455 (411 bytes onode + 2 bytes spanning blobs + 42 bytes inline extents) 2019-12-07T19:33:58.630+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59800 allocated 0x[3b000~3000] released 0x[] 2019-12-07T19:33:58.630+0000 7f8f08189700 10 freelist allocate 0x3b000~3000 2019-12-07T19:33:58.630+0000 7f8f08189700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 prepare 2019-12-07T19:33:58.630+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d59800 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 pg_epoch: 19 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.630+0000 7f8f08189700 10 osd.7 19 dequeue_op 0x558c54e78420 finish 2019-12-07T19:33:58.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1959 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96280 had 1 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860725760 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998762) bluestore 86 GiB free (0.959997), bluefs_ratio 0.039953 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2018 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2018 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x18cfc~7e2 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x18cfc 0x18cfc~7e2 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x8cfc 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcfc 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4de and padding block with 0xb22 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 aio_wait 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x194de 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 aio_wait 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59800 2019-12-07T19:33:58.630+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 io_done 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000346798s (0.000054292s flush + 0.000292506s kv commit) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c5525f680> 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 0x558c54d96280 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a870 space 0x558c54d75198 0x0~2000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552656d8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a7e0 space 0x558c552656d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9015 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 deferred_cleanup 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 0x558c53a91900,0x558c54ce6000,0x558c54ce6280,0x558c54ce6500,0x558c54ce6780,0x558c55276c80 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55264a28 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9320 space 0x558c55264a28 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55264b78 in 0x558c53a59340) _finish_write discard buffer(0x558c551a93b0 space 0x558c55264b78 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55264c58 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9440 space 0x558c55264c58 0x1000~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce7400 had 1 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55264e18 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8ea0 space 0x558c55264e18 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce7680 had 1 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce7900 had 1 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148f00 had 1 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55264f68 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8f30 space 0x558c55264f68 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55265048 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8fc0 space 0x558c55265048 0x1000~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10970 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 deferred_cleanup 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 deferred_cleanup 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96a00 had 1 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 0x558c53a91900,0x558c54ce6f00,0x558c54ce7180,0x558c55276c80 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552653c8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9710 space 0x558c552653c8 0x1000~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10993 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10993 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552654a8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9680 space 0x558c552654a8 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552655f8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9170 space 0x558c552655f8 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 deferred_cleanup 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x194de~2af1 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x194de 0x194de~2af1 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 deferred_cleanup 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x94de 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4de 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 0x558c53a91900,0x558c54d2cf00,0x558c54d2d180,0x558c55276c80 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfcf and padding block with 0x31 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54676fd8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9950 space 0x558c54676fd8 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279208 in 0x558c53a59340) _finish_write discard buffer(0x558c53a69d40 space 0x558c55279208 0x0~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552792e8 in 0x558c53a59340) _finish_write discard buffer(0x558c53a69ef0 space 0x558c552792e8 0x1000~1000 writing nocache) 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 kv_submitted 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:33:58.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1bfcf 2019-12-07T19:33:58.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000243365s (0.000005641s flush + 0.000237724s kv commit) 2019-12-07T19:33:58.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300,0x558c54d59800> 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f6300 osr 0x558c54ea4b40 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 10 0x1b000~1000 crc 1f5a496 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 10 0x2000~1000 crc b19360ce 2019-12-07T19:33:58.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 kv_submitted 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59800 2019-12-07T19:33:58.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 finishing 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59800 onodes 0x558c54d96a00 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c638 in 0x558c53a59a40) _finish_write discard buffer(0x558c54c9abd0 space 0x558c54d3c638 0x0~1000 writing nocache) 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3de48 in 0x558c53a59a40) _finish_write discard buffer(0x558c54c9ab40 space 0x558c54d3de48 0x0~2000 writing nocache) 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59800 done 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:33:58.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59800 [] 2019-12-07T19:33:58.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7 19 _committed_osd_maps 20..21 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7 19 advance to epoch 20 (<= last 21 <= newest_map 21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7 20 advance to epoch 21 (<= last 21 <= newest_map 21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 7 osd.7 21 consume_map version 21 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 identify_splits_and_merges 1.0s3 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 identify_splits_and_merges 1.9s4 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 identify_splits_and_merges 1.3s4 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 identify_splits_and_merges 1.4s0 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 identify_splits_and_merges 1.ds2 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 identify_splits_and_merges 1.6s2 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 identify_splits_and_merges 1.7s4 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 prune_pg_created 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 21 prune_pg_created pruning 1.4 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:0.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:1.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:2.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:3.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:4.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:5.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:6.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7:7.consume_map 21 (was 19) 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:33:58.631+0000 7f8f141a1700 10 osd.7 21 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:58.631+0000 7f8f141a1700 15 osd.7 21 enqueue_peering_evt 1.0s3 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 15 osd.7 21 enqueue_peering_evt 1.9s4 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 15 osd.7 21 enqueue_peering_evt 1.3s4 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 15 osd.7 21 enqueue_peering_evt 1.4s0 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 15 osd.7 21 enqueue_peering_evt 1.ds2 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 15 osd.7 21 enqueue_peering_evt 1.6s2 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 15 osd.7 21 enqueue_peering_evt 1.7s4 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.631+0000 7f8f141a1700 7 osd.7 21 activate_map version 21 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558c5500d000 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558c5500a000 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.631+0000 7f8f0918b700 20 osd.7 21 agent_entry empty queue 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 19 -> 20 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 19 -> 20 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 19 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 19 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 20 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 20 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558c5513e000 2019-12-07T19:33:58.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 20 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 pg_epoch: 20 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 pg_epoch: 20 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 19 -> 20 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 20 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 20 -> 21 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 19 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 20 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 20 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 20 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:33:58.631+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 20 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 pg_epoch: 20 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 pg_epoch: 20 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 20 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 20 -> 21 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 20 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.631+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 20 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 20 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 20 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 pg_epoch: 20 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 pg_epoch: 20 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 20 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 20 -> 21 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 20 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.631+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 19 -> 20 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 19 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 20 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 20 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 20 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 pg_epoch: 20 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 pg_epoch: 20 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 20 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 20 -> 21 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 20 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558c55141000 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 19 -> 20 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] activate_map 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 19 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.631+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 20 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 20 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 20 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 pg_epoch: 20 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 pg_epoch: 20 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c548cc600 con 0x558c54c64000 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 20 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 20 -> 21 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 20 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:33:58.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.631+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.631+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.631+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 19 -> 20 2019-12-07T19:33:58.631+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.631+0000 7f8f05984700 10 osd.7 pg_epoch: 19 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:33:58.631+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:33:58.631+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7f8f05984700 10 osd.7 pg_epoch: 20 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 20 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.632+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 20 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 pg_epoch: 20 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558c548ab000 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 pg_epoch: 20 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 20 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 20 -> 21 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 20 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map: 20 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 19 -> 20 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 19 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.632+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active advmap 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Started advmap 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.632+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] plpg_on_pool_change 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] activate_map 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] hit_set_clear 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.632+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] agent_stop 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 lcod 0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.632+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 21 send_incremental_map 20 -> 21 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map: 21 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 20 -> 21 2019-12-07T19:33:58.632+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~0 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 20 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:33:58.632+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:33:58.632+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active advmap 2019-12-07T19:33:58.632+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d743f0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c54d74460 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Started advmap 2019-12-07T19:33:58.632+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d = 589 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 21 add_map_inc_bl 21 589 bytes 2019-12-07T19:33:58.632+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] plpg_on_pool_change 2019-12-07T19:33:58.632+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(21..21 src has 1..21) v4 -- 0x558c55275200 con 0x558c54c5bc00 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] hit_set_clear 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] agent_stop 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.632+0000 7f8f05984700 10 osd.7 21 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 20 -> 21 (shared) 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_activate_map: 21 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] activate_map 2019-12-07T19:33:58.632+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=0, last_complete=20'1, committed=1, applied=1)) v2 -- 0x558c5501d200 con 0x558c54c5bc00 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] Active: kicking snap trim 2019-12-07T19:33:58.632+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7f8f0297e700 7 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:12 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.632+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:33:58.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 20 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 42932+0+0 (crc 0 0 0) 0x558c54d85000 con 0x558c54c64000 2019-12-07T19:33:58.632+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.632+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.632+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000126 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.632+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.632+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.632+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.632+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.632+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.632+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.632+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0000 seq 5 2019-12-07T19:33:58.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ecc72bdb:::smithi13140796-10:head# key 0x848000000000000001ecc72bdb21736d'ithi13140796-10!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1080 2019-12-07T19:33:58.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# = 0 2019-12-07T19:33:58.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:33:58.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:33:58.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 2019-12-07T19:33:58.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0000 5 blobs 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate 0xa000/1000,a000,0 2019-12-07T19:33:58.633+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3e000~a000/1000,a000,0 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3e000~a000] 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74a10 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74cb0) get_ref 0x0~2000 Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74a10 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d74a28 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74af0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74a80) get_ref 0x0~2000 Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74af0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d74b08 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74c40 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74bd0) get_ref 0x0~2000 Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74c40 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d74c58 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3ce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3ce70 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74d20) get_ref 0x0~2000 Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3ce70 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d74d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d3ce88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d749a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74930 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d749a0) get_ref 0x0~2000 Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74930 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.blob(0x558c54d749a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d74948 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xa000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 = 0 2019-12-07T19:33:58.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 67390 (6 ios * 4000 + 43390 bytes) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes 0x558c54d97180 shared_blobs 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) update inline shard 98 bytes from 5 extents 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 517 (413 bytes onode + 2 bytes spanning blobs + 102 bytes inline extents) 2019-12-07T19:33:58.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[3e000~a000] released 0x[] 2019-12-07T19:33:58.633+0000 7f8f05183700 10 freelist allocate 0x3e000~a000 2019-12-07T19:33:58.633+0000 7f8f05183700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:33:58.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0000 2019-12-07T19:33:58.633+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.633+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:33:58.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 21 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 26544+0+0 (crc 0 0 0) 0x558c54d84800 con 0x558c54c64000 2019-12-07T19:33:58.633+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.633+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.633+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.633+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.633+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] add_log_entry 21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.633+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.633+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.633+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.633+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea0c00 seq 6 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:head# key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.633+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1081 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# = 0 2019-12-07T19:33:58.633+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:33:58.633+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.633+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~6000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0c00 3 blobs 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0x6000/1000,6000,0 2019-12-07T19:33:58.633+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x48000~6000/1000,6000,0 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x48000~6000] 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d960) get_ref 0x0~2000 Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c55279748 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d9d0) get_ref 0x0~2000 Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d9d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3dac8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3db20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.blob(0x558c54d3db20) get_ref 0x0~2000 Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.blob(0x558c54d3db20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3dc18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x6000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 = 0 2019-12-07T19:33:58.633+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.633+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:58.633+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 aio_wait 2019-12-07T19:33:58.633+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.633+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 42213 (4 ios * 4000 + 26213 bytes) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes 0x558c54d97900 shared_blobs 2019-12-07T19:33:58.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97a08) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97a08) update inline shard 60 bytes from 3 extents 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 478 (412 bytes onode + 2 bytes spanning blobs + 64 bytes inline extents) 2019-12-07T19:33:58.633+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[48000~6000] released 0x[] 2019-12-07T19:33:58.633+0000 7f8f04181700 10 freelist allocate 0x48000~6000 2019-12-07T19:33:58.633+0000 7f8f04181700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:33:58.633+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0c00 2019-12-07T19:33:58.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2043 < min_flush_size 524288 2019-12-07T19:33:58.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97180 had 1 2019-12-07T19:33:58.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2066 < min_flush_size 524288 2019-12-07T19:33:58.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2066 < min_flush_size 524288 2019-12-07T19:33:58.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1bfcf~812 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1bfcf 0x1bfcf~812 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xbfcf 2019-12-07T19:33:58.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfcf 2019-12-07T19:33:58.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7e1 and padding block with 0x81f 2019-12-07T19:33:58.635+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 9 ==== osd_map(21..21 src has 1..21) v4 ==== 633+0+0 (crc 0 0 0) 0x558c55275200 con 0x558c54c5bc00 2019-12-07T19:33:58.635+0000 7f8f1199c700 20 osd.7 21 OSD::ms_dispatch: osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.635+0000 7f8f1199c700 10 osd.7 21 do_waiters -- start 2019-12-07T19:33:58.635+0000 7f8f1199c700 10 osd.7 21 do_waiters -- finish 2019-12-07T19:33:58.635+0000 7f8f1199c700 20 osd.7 21 _dispatch 0x558c55275200 osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.635+0000 7f8f1199c700 3 osd.7 21 handle_osd_map epochs [21,21], i have 21, src has [1,21] 2019-12-07T19:33:58.635+0000 7f8f1199c700 10 osd.7 21 no new maps here, dropping 2019-12-07T19:33:58.635+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1c7e1 2019-12-07T19:33:58.635+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.635+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 10 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 34736+0+0 (crc 0 0 0) 0x558c54d85800 con 0x558c54c5bc00 2019-12-07T19:33:58.635+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e789a0 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.635+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e789a0 prio 127 cost 0 latency 0.000067 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 20'1 (0'0,20'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,20'1], crt=0'0) [21'2 (0'0) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:22 2019-12-07T19:33:58.629655+0000 0 ObjectCleanRegions clean_offsets: [131072~18446744073709420543], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] add_log_entry 21'2 (0'0) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:22 2019-12-07T19:33:58.629655+0000 0 ObjectCleanRegions clean_offsets: [131072~18446744073709420543], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.635+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d58000 seq 6 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:head# key 0x8280000000000000016c2051'w!smithi13140796-8!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1082 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# = 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~8000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6c205177:::smithi13140796-8:head# 0x0~8000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~8000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58000 4 blobs 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x8000/1000,8000,0 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4e000~8000/1000,8000,0 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4e000~8000] 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75340) get_ref 0x0~2000 Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75ac8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3df80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d753b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3df80 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d753b0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df80) get_ref 0x0~2000 Blob(0x558c54d3df80 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d753b0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3df80 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d753c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75420 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8000 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75420) get_ref 0x0~2000 Blob(0x558c54d75420 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8000 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d75420 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54cf8018 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf80e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8070 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf80e0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf8070) get_ref 0x0~2000 Blob(0x558c54cf8070 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf80e0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf8070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54cf8070 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 sbid 0x0)) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54cf80f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x8000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dirty_range mark inline shard dirty 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~8000 = 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 55195 (5 ios * 4000 + 35195 bytes) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes 0x558c54d96500 shared_blobs 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) update inline shard 79 bytes from 4 extents 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 497 (412 bytes onode + 2 bytes spanning blobs + 83 bytes inline extents) 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[4e000~8000] released 0x[] 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 freelist allocate 0x4e000~8000 2019-12-07T19:33:58.635+0000 7f8f0197c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:33:58.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58000 2019-12-07T19:33:58.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002654978s (0.001206250s flush + 0.001448728s kv commit) 2019-12-07T19:33:58.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0000> 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:33:58.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 0x558c54d97180 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3ce88 in 0x558c53a59880) _finish_write discard buffer(0x558c551a8ea0 space 0x558c54d3ce88 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74948 in 0x558c53a59880) _finish_write discard buffer(0x558c551a8f30 space 0x558c54d74948 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74a28 in 0x558c53a59880) _finish_write discard buffer(0x558c551a9320 space 0x558c54d74a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74b08 in 0x558c53a59880) _finish_write discard buffer(0x558c551a93b0 space 0x558c54d74b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74c58 in 0x558c53a59880) _finish_write discard buffer(0x558c551a9440 space 0x558c54d74c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:33:58.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=3, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c548cc000 con 0x558c54c64000 2019-12-07T19:33:58.636+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.636+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e78840 finish 2019-12-07T19:33:58.636+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.636+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e789a0 finish 2019-12-07T19:33:58.636+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 aio_wait 2019-12-07T19:33:58.636+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:33:58.636+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:33:58.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.639+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 22 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2228+0+0 (crc 0 0 0) 0x558c54d86000 con 0x558c54c64000 2019-12-07T19:33:58.639+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.639+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.9s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 21'1 active mbc={}] 2019-12-07T19:33:58.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1212 < min_flush_size 524288 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97900 had 1 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 lcod 21'1 active mbc={}] append_log log((0'0,21'2], crt=0'0) [21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1235 < min_flush_size 524288 2019-12-07T19:33:58.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1235 < min_flush_size 524288 2019-12-07T19:33:58.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1c7e1~4d3 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1c7e1 0x1c7e1~4d3 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'1 active mbc={}] add_log_entry 21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xc7e1 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7e1 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcb4 and padding block with 0x34c 2019-12-07T19:33:58.639+0000 7f8f08189700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.639+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.639+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d59b00 seq 7 2019-12-07T19:33:58.639+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 3 keys 2019-12-07T19:33:58.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 3 keys = 0 2019-12-07T19:33:58.639+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.639+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes 0x558c54d96a00 shared_blobs 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d96b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 526 (482 bytes onode + 2 bytes spanning blobs + 42 bytes inline extents) 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:33:58.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:33:58.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.639+0000 7f8f08189700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.639+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 23 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 55220+0+0 (crc 0 0 0) 0x558c551dc000 con 0x558c54c64000 2019-12-07T19:33:58.639+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.639+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 aio_wait 2019-12-07T19:33:58.639+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:33:58.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:33:58.639+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1ccb4 2019-12-07T19:33:58.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.640+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000285 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 lcod 21'1 active mbc={}] 2019-12-07T19:33:58.640+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 lcod 21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 lcod 21'1 active mbc={}] append_log log((0'0,21'3], crt=21'1) [21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.640+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.640+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.640+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.640+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.640+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea0f00 seq 8 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9da56077:::smithi13140796-13:head# key 0x8480000000000000019da560'w!smithi13140796-13!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1083 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# = 0 2019-12-07T19:33:58.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2 keys 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2 keys = 0 2019-12-07T19:33:58.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~d000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9da56077:::smithi13140796-13:head# 0x0~d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~d000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0f00 7 blobs 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0xd000/1000,d000,0 2019-12-07T19:33:58.640+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x56000~d000/1000,d000,0 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x56000~d000] 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55278d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75c00 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c55278d90) get_ref 0x0~2000 Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75c00 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c55278d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d75c18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d757a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d757a0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75730) get_ref 0x0~2000 Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d757a0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d757b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d758f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d758f0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75880) get_ref 0x0~2000 Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d758f0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d75908 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d759d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d759d0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75960) get_ref 0x0~2000 Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d759d0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d759e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf85b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf85b0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8540) get_ref 0x0~2000 Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf85b0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54cf85c8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf84d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8460 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf84d0) get_ref 0x0~2000 Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8460 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf84d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54cf8478 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf83f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55265ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265ab0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf83f0) get_ref 0x0~1000 Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265ab0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf83f0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~1000: 0x0~1000 Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c55265ac8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xd000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) dirty_range mark inline shard dirty 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~d000 = 0 2019-12-07T19:33:58.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 86870 (8 ios * 4000 + 54870 bytes) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes 0x558c54ce2780 shared_blobs 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) update 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) update inline shard 133 bytes from 7 extents 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9da56077:::smithi13140796-13:head# is 552 (413 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:33:58.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[56000~d000] released 0x[] 2019-12-07T19:33:58.640+0000 7f8f04181700 10 freelist allocate 0x56000~d000 2019-12-07T19:33:58.640+0000 7f8f04181700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:33:58.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0f00 2019-12-07T19:33:58.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.004079710s (0.002618684s flush + 0.001461026s kv commit) 2019-12-07T19:33:58.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:33:58.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 0x558c54d97900 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dac8 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a98c0 space 0x558c54d3dac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dc18 in 0x558c53a59a40) _finish_write discard buffer(0x558c54c9a090 space 0x558c54d3dc18 0x0~2000 writing nocache) 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279748 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a9830 space 0x558c55279748 0x0~2000 writing nocache) 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 kv_queued 2019-12-07T19:33:58.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:33:58.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1563 < min_flush_size 524288 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96a00 had 1 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:33:58.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3584 < min_flush_size 524288 2019-12-07T19:33:58.641+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96500 had 1 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3607 < min_flush_size 524288 2019-12-07T19:33:58.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3607 < min_flush_size 524288 2019-12-07T19:33:58.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1ccb4~e17 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1ccb4 0x1ccb4~e17 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xccb4 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcb4 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xacb and padding block with 0x535 2019-12-07T19:33:58.641+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.641+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:33:58.641+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.641+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=4, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c55271c00 con 0x558c54c64000 2019-12-07T19:33:58.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.642+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 24 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d84000 con 0x558c54c64000 2019-12-07T19:33:58.642+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000282 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.642+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59200 seq 6 2019-12-07T19:33:58.642+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:33:58.642+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:33:58.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.642+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 25 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54d84800 con 0x558c54c64000 2019-12-07T19:33:58.642+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.642+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.642+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.642+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.642+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.642+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.642+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.642+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.642+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.642+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d20600 seq 7 2019-12-07T19:33:58.642+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 3 keys 2019-12-07T19:33:58.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 3 keys = 0 2019-12-07T19:33:58.642+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.642+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes 0x558c54d97180 shared_blobs 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 589 (485 bytes onode + 2 bytes spanning blobs + 102 bytes inline extents) 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:33:58.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:33:58.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:33:58.642+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.643+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:33:58.643+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 26 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 2228+0+0 (crc 0 0 0) 0x558c54d87000 con 0x558c54c64000 2019-12-07T19:33:58.643+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79080 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.643+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.643+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.643+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.643+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.643+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e79080 prio 127 cost 0 latency 0.000222 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.9s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] append_log log((0'0,21'4], crt=21'1) [21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'1 lcod 21'2 active mbc={}] add_log_entry 21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.643+0000 7f8f04181700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] rollforward: entry=21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.643+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.643+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.643+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea1200 seq 9 2019-12-07T19:33:58.643+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 3 keys 2019-12-07T19:33:58.643+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 3 keys = 0 2019-12-07T19:33:58.643+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.643+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.643+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.643+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.643+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes 0x558c54d97900 shared_blobs 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97a08) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 549 (483 bytes onode + 2 bytes spanning blobs + 64 bytes inline extents) 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.643+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:33:58.643+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 blocked by 0x558c54ea0f00 aio_wait 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.643+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e79080 finish 2019-12-07T19:33:58.643+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1dacb 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.003764782s (0.000373304s flush + 0.003391478s kv commit) 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 2 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2743 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97180 had 1 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2766 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2766 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1dacb~ace to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.644+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1dacb 0x1dacb~ace to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xdacb 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xacb 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.644+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x599 and padding block with 0xa67 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00,0x558c54d58000> 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 0x558c54d96a00 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 aio_wait 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 0x558c54d96500 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8018 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a120 space 0x558c54cf8018 0x0~2000 writing nocache) 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf80f8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a000 space 0x558c54cf80f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d753c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9aab0 space 0x558c54d753c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75ac8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9aa20 space 0x558c54d75ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:33:58.644+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.644+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=5, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c55270200 con 0x558c54c64000 2019-12-07T19:33:58.644+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 aio_wait 2019-12-07T19:33:58.644+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:33:58.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:33:58.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:33:58.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c54c1e200 con 0x558c54c5bc00 2019-12-07T19:33:58.644+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.645+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 11 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 ==== 2228+0+0 (crc 0 0 0) 0x558c54d87800 con 0x558c54c5bc00 2019-12-07T19:33:58.645+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.645+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000083 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 pg pg[1.6s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'2], crt=0'0) [21'3 (20'1) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:11 2019-12-07T19:33:58.626793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] add_log_entry 21'3 (20'1) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:11 2019-12-07T19:33:58.626793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] rollforward: entry=20'1 (0'0) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:10 2019-12-07T19:33:58.626763+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.645+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d21b00 seq 7 2019-12-07T19:33:58.645+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 3 keys 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 3 keys = 0 2019-12-07T19:33:58.645+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.645+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes 0x558c54d96280 shared_blobs 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96388) update 2#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:657ac0fe:::smithi13140796-4:head# is 530 (483 bytes onode + 2 bytes spanning blobs + 45 bytes inline extents) 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.645+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78420 finish 2019-12-07T19:33:58.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 12 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 ==== 67508+0+0 (crc 0 0 0) 0x558c54e3e000 con 0x558c54c5bc00 2019-12-07T19:33:58.646+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000034 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.646+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.646+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.646+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000083 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 pg pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=20'1 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.646+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=20'1 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.646+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=20'1 lcod 21'2 active mbc={}] append_log log((0'0,21'3], crt=20'1) [21'4 (0'0) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:46 2019-12-07T19:33:58.643677+0000 0 ObjectCleanRegions clean_offsets: [262144~18446744073709289471], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.646+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] add_log_entry 21'4 (0'0) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:46 2019-12-07T19:33:58.643677+0000 0 ObjectCleanRegions clean_offsets: [262144~18446744073709289471], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.646+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.646+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.646+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.646+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.646+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0000 seq 8 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6336c6f2:::smithi13140796-16:head# key 0x8280000000000000016336c6f221736d'ithi13140796-16!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.646+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1084 2019-12-07T19:33:58.646+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# = 0 2019-12-07T19:33:58.646+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:33:58.646+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:33:58.646+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.646+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.646+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.646+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~10000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0000 8 blobs 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1e599 2019-12-07T19:33:58.646+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.646+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f8f05984700 10 fbmap_alloc 0x558c53a72e00 allocate 0x10000/1000,10000,0 2019-12-07T19:33:58.646+0000 7f8f05984700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x63000~10000/1000,10000,0 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x63000~10000] 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ea0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54d3c8c0) get_ref 0x0~2000 Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ea0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54d3c8c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d75eb8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3cee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3cc40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3cc40 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54d3cee0) get_ref 0x0~2000 Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3cc40 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54d3cee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d3cc58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3cb60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8d90 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54d3cb60) get_ref 0x0~2000 Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8d90 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54d3cb60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8af0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8e00) get_ref 0x0~2000 Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8af0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8b08 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8ee0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8e70) get_ref 0x0~2000 Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8ee0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8ef8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8fc0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8f50) get_ref 0x0~2000 Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8fc0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8fd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf90a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf90a0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.blob(0x558c54cf9030) get_ref 0x0~2000 Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf90a0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.blob(0x558c54cf9030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf90b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.002486362s (0.000000121s flush + 0.002486241s kv commit) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9180 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.blob(0x558c54cf9110) get_ref 0x0~2000 Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9180 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.blob(0x558c54cf9110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf9198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1264 < min_flush_size 524288 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x10000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.647+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 = 0 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2780 had 1 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:33:58.647+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2850 < min_flush_size 524288 2019-12-07T19:33:58.647+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.647+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97900 had 1 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4417 < min_flush_size 524288 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96280 had 1 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4440 < min_flush_size 524288 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4440 < min_flush_size 524288 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.647+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1e599~1158 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.647+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 103158 (9 ios * 4000 + 67158 bytes) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes 0x558c54ce3900 shared_blobs 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1e599 0x1e599~1158 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xe599 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x599 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) update inline shard 155 bytes from 8 extents 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6f1 and padding block with 0x90f 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 574 (413 bytes onode + 2 bytes spanning blobs + 159 bytes inline extents) 2019-12-07T19:33:58.647+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[63000~10000] released 0x[] 2019-12-07T19:33:58.647+0000 7f8f05984700 10 freelist allocate 0x63000~10000 2019-12-07T19:33:58.647+0000 7f8f05984700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.647+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:33:58.647+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0000 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200,0x558c54d20600> 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 kv_submitted 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 0x558c54d97180 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:33:58.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.647+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f6f1 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.647+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.647+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=9, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c5501d200 con 0x558c54c64000 2019-12-07T19:33:58.647+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=10, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c55250400 con 0x558c54c64000 2019-12-07T19:33:58.647+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 13 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 2228+0+0 (crc 0 0 0) 0x558c54e3e800 con 0x558c54c5bc00 2019-12-07T19:33:58.648+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.648+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.648+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.002647503s (0.000004968s flush + 0.002642535s kv commit) 2019-12-07T19:33:58.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00,0x558c54ea1200,0x558c54d21b00> 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 0x558c54ce2780 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8478 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a9950 space 0x558c54cf8478 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf85c8 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a90e0 space 0x558c54cf85c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d757b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a9050 space 0x558c54d757b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75908 in 0x558c53a59a40) _finish_write discard buffer(0x558c53a69d40 space 0x558c54d75908 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d759e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c53a69ef0 space 0x558c54d759e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75c18 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a9170 space 0x558c54d75c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55265ac8 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a97a0 space 0x558c55265ac8 0x0~1000 writing nocache) 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:33:58.649+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 kv_submitted 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:33:58.649+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e78840 finish 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 0x558c54d97900 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:33:58.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 0x558c54d96280 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 aio_wait 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:33:58.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.649+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=3, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c55271400 con 0x558c54c5bc00 2019-12-07T19:33:58.649+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=7, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c55270a00 con 0x558c54c64000 2019-12-07T19:33:58.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=12, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c548cc600 con 0x558c54c64000 2019-12-07T19:33:58.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.649+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.001238 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.6s2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=20'1 lcod 21'3 active mbc={}] 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=20'1 lcod 21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=20'1 lcod 21'3 active mbc={}] append_log log((0'0,21'4], crt=20'1) [21'5 (21'2) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:23 2019-12-07T19:33:58.629675+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=20'1 lcod 21'3 active mbc={}] add_log_entry 21'5 (21'2) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:23 2019-12-07T19:33:58.629675+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] rollforward: entry=21'2 (0'0) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:22 2019-12-07T19:33:58.629655+0000 0 ObjectCleanRegions clean_offsets: [131072~18446744073709420543], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.650+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0300 seq 9 2019-12-07T19:33:58.650+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 3 keys 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 3 keys = 0 2019-12-07T19:33:58.650+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.650+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes 0x558c54d96500 shared_blobs 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 568 (483 bytes onode + 2 bytes spanning blobs + 83 bytes inline extents) 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 blocked by 0x558c54ea0000 aio_wait 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.650+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.650+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 aio_wait 2019-12-07T19:33:58.650+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:33:58.650+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:33:58.650+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:33:58.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.650+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 27 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e3f000 con 0x558c54c64000 2019-12-07T19:33:58.650+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.650+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.650+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.650+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.650+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.650+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.650+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.650+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.650+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.650+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.650+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.650+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.650+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59500 seq 8 2019-12-07T19:33:58.650+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.650+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.651+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.651+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.651+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.651+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.651+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59500 onodes shared_blobs 2019-12-07T19:33:58.651+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.651+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 prepare 2019-12-07T19:33:58.651+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59500 2019-12-07T19:33:58.651+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 io_done 2019-12-07T19:33:58.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 7 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.109602748s delta_ub 0.022351826s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e200 con 0x558c552d1400 2019-12-07T19:33:58.651+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:33:58.651+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.651+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 7 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.109602748s delta_ub 0.022351826s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af400 con 0x558c54c18000 2019-12-07T19:33:58.651+0000 7f8f24457700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021926775s,0.022351826s]) 2019-12-07T19:33:58.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.087675973s delta_ub 18446744073.687624841s) v5 -- 0x558c55270a00 con 0x558c552d1400 2019-12-07T19:33:58.651+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021892391s,0.022351826s]) 2019-12-07T19:33:58.651+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.087710357s delta_ub 18446744073.687659225s) v5 -- 0x558c54d76c00 con 0x558c54c18000 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1286 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 2 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2319 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2902 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96500 had 1 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2925 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2925 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f6f1~b6d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f6f1 0x1f6f1~b6d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615738+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _allocate len 0x25e from 0 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _allocate len 0x25e from 1 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd100000 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd130000~10000/10000,10000,acd100000 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x30000 to include allocated 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 24 (was clean) 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xf6f1 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6f1 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x25e and padding block with 0xda2 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x30000 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2025e 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (24) on file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]), flushing log 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 24 len 0x3c crc 0x728460d0) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x17000~1000 to file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x17000 0x17000~1000 to file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x17000 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x18000 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000026 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 24 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.002105116s (0.001646298s flush + 0.000458818s kv commit) 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0000,0x558c54ea0300> 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 0x558c54ce3900 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8b08 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9af30 space 0x558c54cf8b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8da8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9aea0 space 0x558c54cf8da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8ef8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9afc0 space 0x558c54cf8ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8fd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b050 space 0x558c54cf8fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf90b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b0e0 space 0x558c54cf90b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9198 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b170 space 0x558c54cf9198 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3cc58 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9ab40 space 0x558c54d3cc58 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2025e~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75eb8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9abd0 space 0x558c54d75eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2025e 0x2025e~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x25e 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x25e 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6cf and padding block with 0x931 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 0x558c54d96500 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.652+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=4, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c55270200 con 0x558c54c5bc00 2019-12-07T19:33:58.652+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=5, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c55270800 con 0x558c54c5bc00 2019-12-07T19:33:58.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x206cf 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.652+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000166673s (0.000000121s flush + 0.000166552s kv commit) 2019-12-07T19:33:58.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59500> 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 kv_submitted 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59500 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 finishing 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59500 onodes 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59500 done 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59500 [] 2019-12-07T19:33:58.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.652+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=16, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c548cc000 con 0x558c54c64000 2019-12-07T19:33:58.652+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 28 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c54e3f800 con 0x558c54c64000 2019-12-07T19:33:58.654+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79340 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.654+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.654+0000 7f8f08189700 10 osd.7 21 dequeue_op 0x558c54e79340 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.654+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.654+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'5], crt=21'2) [] 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.654+0000 7f8f08189700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.654+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.654+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.654+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.654+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea0f00 seq 10 2019-12-07T19:33:58.654+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.654+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.654+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.654+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.654+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.654+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.654+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:33:58.654+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.654+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:33:58.654+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:33:58.654+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:33:58.654+0000 7f8f08189700 10 osd.7 21 dequeue_op 0x558c54e79340 finish 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x206cf~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x206cf 0x206cf~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x6cf 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6cf 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb40 and padding block with 0x4c0 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x20b40 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000096690s (0.000000135s flush + 0.000096555s kv commit) 2019-12-07T19:33:58.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:33:58.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:33:58.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:33:58.654+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.654+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=18, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c55271c00 con 0x558c54c64000 2019-12-07T19:33:58.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.659+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 14 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54e40000 con 0x558c54c5bc00 2019-12-07T19:33:58.659+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.660+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.660+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 15 ==== MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (crc 0 0 0) 0x558c54d2c780 con 0x558c54c5bc00 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.001019 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.660+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.660+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'5], crt=21'2) [21'6 (21'4) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:47 2019-12-07T19:33:58.643819+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'2 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'4) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:47 2019-12-07T19:33:58.643819+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] rollforward: entry=21'3 (20'1) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:11 2019-12-07T19:33:58.626793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] rollforward: entry=21'4 (0'0) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:46 2019-12-07T19:33:58.643677+0000 0 ObjectCleanRegions clean_offsets: [262144~18446744073709289471], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.660+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.660+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.660+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0c00 seq 10 2019-12-07T19:33:58.660+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 3 keys 2019-12-07T19:33:58.660+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 16 ==== MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (crc 0 0 0) 0x558c54ce7b80 con 0x558c54c5bc00 2019-12-07T19:33:58.660+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 3 keys = 0 2019-12-07T19:33:58.660+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.660+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.660+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.660+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.660+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.660+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.660+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes 0x558c54ce3900 shared_blobs 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 646 (485 bytes onode + 2 bytes spanning blobs + 159 bytes inline extents) 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.660+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:33:58.660+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:33:58.660+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1686 < min_flush_size 524288 2019-12-07T19:33:58.660+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e78580 finish 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.660+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1709 < min_flush_size 524288 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1709 < min_flush_size 524288 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x20b40~6ad to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x20b40 0x20b40~6ad to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xb40 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb40 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1ed and padding block with 0xe13 2019-12-07T19:33:58.660+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000254 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] 2019-12-07T19:33:58.660+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x211ed 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.660+0000 7f8f0197c700 25 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.660+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~1000 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x8000 (32768) 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118784s (0.000000151s flush + 0.000118633s kv commit) 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:33:58.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 0x558c54ce3900 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:33:58.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.660+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.660+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 11 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (unknown 0 0 0) 0x558c54e6e480 con 0x558c54c5b400 2019-12-07T19:33:58.660+0000 7f8f1199c700 20 osd.7 21 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.660+0000 7f8f1199c700 10 osd.7 21 do_waiters -- start 2019-12-07T19:33:58.660+0000 7f8f1199c700 10 osd.7 21 do_waiters -- finish 2019-12-07T19:33:58.660+0000 7f8f1199c700 20 osd.7 21 _dispatch 0x558c54e6e480 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.660+0000 7f8f1199c700 3 osd.7 21 handle_osd_map epochs [20,21], i have 21, src has [1,21] 2019-12-07T19:33:58.660+0000 7f8f1199c700 10 osd.7 21 no new maps here, dropping 2019-12-07T19:33:58.660+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 12 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 71604+0+0 (unknown 0 0 0) 0x558c54e40800 con 0x558c54c5b400 2019-12-07T19:33:58.660+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.660+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 osd.7 21 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.660+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea1800 seq 5 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:c87708e9:::smithi13140796-17:head# key 0x848000000000000001c87708e921736d'ithi13140796-17!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.660+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1085 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# = 0 2019-12-07T19:33:58.660+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 2 keys 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 2 keys = 0 2019-12-07T19:33:58.660+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.660+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 0x0~11000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:c87708e9:::smithi13140796-17:head# 0x0~11000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~11000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1800 9 blobs 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x11000/1000,11000,0 2019-12-07T19:33:58.660+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x73000~11000/1000,11000,0 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x73000~11000] 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9d50 blob([0x73000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9ce0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9d50) get_ref 0x0~2000 Blob(0x558c54cf9d50 blob([0x73000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9ce0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54cf9d50 blob([0x73000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9ce0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54cf9cf8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9880 blob([0x75000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dce0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9880) get_ref 0x0~2000 Blob(0x558c54cf9880 blob([0x75000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dce0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54cf9880 blob([0x75000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dce0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54d3dcf8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3dd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3dd50 blob([0x77000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9dc0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54d3dd50) get_ref 0x0~2000 Blob(0x558c54d3dd50 blob([0x77000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9dc0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54d3dd50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d3dd50 blob([0x77000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9dc0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54cf9dd8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9e30 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9ea0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9e30) get_ref 0x0~2000 Blob(0x558c54cf9e30 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9ea0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54cf9e30 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9ea0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54cf9eb8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9f10 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9f80 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9f10) get_ref 0x0~2000 Blob(0x558c54cf9f10 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9f80 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf9f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54cf9f10 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9f80 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54cf9f98 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42000 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42000) get_ref 0x0~2000 Blob(0x558c54e42000 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54e42000 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e420e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e420e0 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42150 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54e420e0) get_ref 0x0~2000 Blob(0x558c54e420e0 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42150 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.blob(0x558c54e420e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c54e420e0 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42150 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42168 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e421c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e421c0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42230 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.blob(0x558c54e421c0) get_ref 0x0~2000 Blob(0x558c54e421c0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42230 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.blob(0x558c54e421c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c54e421c0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42230 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42248 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e422a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e422a0 blob([0x83000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.blob(0x558c54e422a0) get_ref 0x0~1000 Blob(0x558c54e422a0 blob([0x83000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.blob(0x558c54e422a0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~1000: 0x0~1000 Blob(0x558c54e422a0 blob([0x83000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x11000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54d2cb08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 0x0~11000 = 0 2019-12-07T19:33:58.661+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.661+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1800 cost 112062 (10 ios * 4000 + 72062 bytes) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1800 onodes 0x558c54d2ca00 shared_blobs 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54d2cb08) update 4#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54d2cb08) update inline shard 171 bytes from 9 extents 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c87708e9:::smithi13140796-17:head# is 590 (413 bytes onode + 2 bytes spanning blobs + 175 bytes inline extents) 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1800 allocated 0x[73000~11000] released 0x[] 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 freelist allocate 0x73000~11000 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 freelist _xor first_key 0x0 last_key 0x80000 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 prepare 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea1800 2019-12-07T19:33:58.661+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 29 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54d87800 con 0x558c54c64000 2019-12-07T19:33:58.661+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.661+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.661+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:33:58.661+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.661+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.661+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000088 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.661+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.661+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75ac8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.661+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54c9a870 space 0x558c54d75ac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.661+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~1000 = 4096 2019-12-07T19:33:58.661+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.661+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.661+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] get_hash_info: Getting attr on 1:6c205177:::smithi13140796-8:head 2019-12-07T19:33:58.661+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.661+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] get_hash_info: not in cache 1:6c205177:::smithi13140796-8:head 2019-12-07T19:33:58.661+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.661+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] get_hash_info: found on disk, size 32768 2019-12-07T19:33:58.661+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key 2019-12-07T19:33:58.661+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.661+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key = 0 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea1500 seq 11 2019-12-07T19:33:58.661+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 3 keys 2019-12-07T19:33:58.661+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 -- 0x558c54d96000 con 0x558c54c5bc00 2019-12-07T19:33:58.661+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 3 keys = 0 2019-12-07T19:33:58.661+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.661+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.661+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.661+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78420 finish 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.661+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.661+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.661+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes 0x558c54ce2780 shared_blobs 2019-12-07T19:33:58.661+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) update 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.661+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9da56077:::smithi13140796-13:head# is 624 (485 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.661+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:33:58.661+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:33:58.661+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 aio_wait 2019-12-07T19:33:58.661+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.001382 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] 2019-12-07T19:33:58.661+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:33:58.661+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1800 2019-12-07T19:33:58.661+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 io_done 2019-12-07T19:33:58.661+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.661+0000 7f8f05984700 25 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.661+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.661+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 0x0~1000 2019-12-07T19:33:58.661+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.661+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x4000 (16384) 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.661+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.661+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d750a0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552656c0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.661+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d750a0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552656c0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.661+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.661+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1664 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2780 had 1 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3809 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2ca00 had 1 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3832 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3832 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x211ed~ef8 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x211ed 0x211ed~ef8 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x11ed 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1ed 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe5 and padding block with 0xf1b 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x220e5 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000177745s (0.000046530s flush + 0.000131215s kv commit) 2019-12-07T19:33:58.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500,0x558c54ea1800> 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 0x558c54ce2780 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 kv_submitted 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1800 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 finishing 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1800 onodes 0x558c54d2ca00 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9cf8 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a99e0 space 0x558c54cf9cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9dd8 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a90e0 space 0x558c54cf9dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9eb8 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a9050 space 0x558c54cf9eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9f98 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a9170 space 0x558c54cf9f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dcf8 in 0x558c53a596c0) _finish_write discard buffer(0x558c53a69ef0 space 0x558c54d3dcf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a97a0 space 0x558c54e42088 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42168 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a8f30 space 0x558c54e42168 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42248 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a9320 space 0x558c54e42248 0x0~2000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a93b0 space 0x558c54e42328 0x0~1000 writing nocache) 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1800 done 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1800 [] 2019-12-07T19:33:58.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.661+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.661+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=25, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c55250400 con 0x558c54c64000 2019-12-07T19:33:58.661+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c54cea800 con 0x558c54c5b400 2019-12-07T19:33:58.662+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d750a0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552656c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.664+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c552656d8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.664+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54c9b3b0 space 0x558c552656d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.664+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 0x0~1000 = 4096 2019-12-07T19:33:58.664+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.664+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] get_hash_info: Getting attr on 1:657ac0fe:::smithi13140796-4:head 2019-12-07T19:33:58.664+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] get_hash_info: not in cache 1:657ac0fe:::smithi13140796-4:head 2019-12-07T19:33:58.664+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.664+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 lcod 21'5 active mbc={}] get_hash_info: found on disk, size 16384 2019-12-07T19:33:58.664+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# hinfo_key 2019-12-07T19:33:58.664+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# hinfo_key = 0 2019-12-07T19:33:58.664+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=10, attrs_read=0)) v2 -- 0x558c54ce3680 con 0x558c54c5bc00 2019-12-07T19:33:58.664+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:33:58.664+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c5501d400 con 0x558c54c5bc00 2019-12-07T19:33:58.664+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 90573 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.665+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 30 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 79796+0+0 (crc 0 0 0) 0x558c54d86000 con 0x558c54c64000 2019-12-07T19:33:58.665+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000088 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.665+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59b00 seq 9 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ec1285ab:::smithi13140796-19:head# key 0x848000000000000001ec1285ab21736d'ithi13140796-19!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1086 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# = 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~13000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d59b00 10 blobs 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x13000/1000,13000,0 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x84000~13000/1000,13000,0 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x84000~13000] 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9650 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d180) get_ref 0x0~2000 Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9650 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54cf9668 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d110 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d030) get_ref 0x0~2000 Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d110 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d3d128 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3ccb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55265960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265960 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3ccb0) get_ref 0x0~2000 Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265960 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3ccb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55265978 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42fc0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c55265a40) get_ref 0x0~2000 Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42fc0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c55265a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e42fd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e430a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e430a0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43030) get_ref 0x0~2000 Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e430a0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e430b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43180 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43110) get_ref 0x0~2000 Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43180 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43198 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e431f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43260 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e431f0) get_ref 0x0~2000 Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43260 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e431f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43278 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e432d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43340 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e432d0) get_ref 0x0~2000 Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43340 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e432d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43358 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e433b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43420 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e433b0) get_ref 0x0~2000 Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43420 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e433b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43438 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43500 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43490) get_ref 0x0~1000 Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43500 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43490) get_ref init 0x1000, 1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~1000: 0x0~1000 Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43518 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x13000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 = 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 123446 (11 ios * 4000 + 79446 bytes) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes 0x558c54e44000 shared_blobs 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update inline shard 190 bytes from 10 extents 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 609 (413 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[84000~13000] released 0x[] 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 freelist allocate 0x84000~13000 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d59b00 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.665+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:33:58.665+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.665+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 31 ==== MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d87000 con 0x558c54c64000 2019-12-07T19:33:58.665+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79600 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.665+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.665+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.665+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558c5500a000 2019-12-07T19:33:58.665+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e79600 prio 127 cost 0 latency 0.000053 MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.666+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s4 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.666+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.666+0000 7f8f04181700 20 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.666+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.666+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.666+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.666+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.666+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:33:58.666+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d58c00 seq 12 2019-12-07T19:33:58.666+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:33:58.666+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.666+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.666+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.666+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:33:58.666+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.666+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 2019-12-07T19:33:58.666+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.666+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:33:58.666+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:33:58.666+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:33:58.666+0000 7f8f04181700 10 osd.7 21 dequeue_op 0x558c54e79600 finish 2019-12-07T19:33:58.666+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x220e5~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x220e5 0x220e5~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x20e5 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe5 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x556 and padding block with 0xaaa 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x22556 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000406092s (0.000000150s flush + 0.000405942s kv commit) 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:33:58.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:33:58.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:33:58.666+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.666+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=31, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c55270800 con 0x558c54c64000 2019-12-07T19:33:58.666+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 aio_wait 2019-12-07T19:33:58.666+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:33:58.666+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1321 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1344 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1344 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x22556~540 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x22556 0x22556~540 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x2556 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x556 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa96 and padding block with 0x56a 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x22a96 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000273230s (0.000117453s flush + 0.000155777s kv commit) 2019-12-07T19:33:58.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00> 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.667+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:33:58.667+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 0x558c54e44000 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9668 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b200 space 0x558c54cf9668 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d128 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9af30 space 0x558c54d3d128 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42fd8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9afc0 space 0x558c54e42fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e430b8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b050 space 0x558c54e430b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43198 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b0e0 space 0x558c54e43198 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43278 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b170 space 0x558c54e43278 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43358 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9ab40 space 0x558c54e43358 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43438 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9a090 space 0x558c54e43438 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=30, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c53a5ea00 con 0x558c54c64000 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43518 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9a120 space 0x558c54e43518 0x0~1000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55265978 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9aea0 space 0x558c55265978 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.667+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:33:58.667+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.667+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 17 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d86800 con 0x558c54c5bc00 2019-12-07T19:33:58.668+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79760 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.668+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.668+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 18 ==== MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c54ce3680 con 0x558c54c5bc00 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79a20 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.668+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79760 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'6], crt=21'4) [] 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'5 (21'2) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:23 2019-12-07T19:33:58.629675+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'4) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:47 2019-12-07T19:33:58.643819+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.668+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.668+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 13 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54d85000 con 0x558c54c5b400 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea1b00 seq 11 2019-12-07T19:33:58.668+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.668+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.668+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes shared_blobs 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79760 finish 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79a20 prio 127 cost 0 latency 0.000198 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.668+0000 7f8f0197c700 25 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.668+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~1000 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000106 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x10000 (65536) 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.668+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.668+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea0900 seq 6 2019-12-07T19:33:58.668+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x22a96~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x22a96 0x22a96~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes shared_blobs 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x2a96 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa96 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf07 and padding block with 0xf9 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:33:58.668+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x22f07 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127995s (0.000000156s flush + 0.000127839s kv commit) 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1b00> 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x22f07~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x22f07 0x22f07~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x2f07 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf07 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x378 and padding block with 0xc88 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x23378 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000125726s (0.000000116s flush + 0.000125610s kv commit) 2019-12-07T19:33:58.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0900> 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:33:58.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.668+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=2, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c548cc000 con 0x558c54c5b400 2019-12-07T19:33:58.668+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75eb8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c551a8fc0 space 0x558c54d75eb8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~1000 = 4096 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 65536 2019-12-07T19:33:58.668+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key = 0 2019-12-07T19:33:58.668+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=15, attrs_read=0)) v2 -- 0x558c54d2c780 con 0x558c54c5bc00 2019-12-07T19:33:58.668+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:33:58.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=13, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c55271c00 con 0x558c54c5bc00 2019-12-07T19:33:58.669+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 14 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558c551dc000 con 0x558c54c5b400 2019-12-07T19:33:58.669+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.669+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000135 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.669+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea1200 seq 7 2019-12-07T19:33:58.669+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 3 keys 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 3 keys = 0 2019-12-07T19:33:58.669+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.669+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes 0x558c54d2ca00 shared_blobs 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54d2cb08) update 4#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c87708e9:::smithi13140796-17:head# is 662 (485 bytes onode + 2 bytes spanning blobs + 175 bytes inline extents) 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1702 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2ca00 had 1 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1725 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1725 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x23378~6bd to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x23378 0x23378~6bd to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x3378 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x378 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa35 and padding block with 0x5cb 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x23a35 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000089889s (0.000000120s flush + 0.000089769s kv commit) 2019-12-07T19:33:58.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200> 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:33:58.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 0x558c54d2ca00 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:33:58.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.669+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c54ceaa00 con 0x558c54c5b400 2019-12-07T19:33:58.669+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.670+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9f000 0x558c54d04680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.670+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9f000 0x558c54d04680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.670+0000 7f8f24457700 10 osd.7 21 ms_handle_authentication new session 0x558c54c5cb60 con 0x558c54c9f000 entity client.0 addr 2019-12-07T19:33:58.670+0000 7f8f24457700 10 osd.7 21 ms_handle_authentication session 0x558c54c5cb60 client.0 has caps osdcap[grant(rwx)] 'allow rwx' 2019-12-07T19:33:58.670+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/224485236 conn(0x558c54c9f000 0x558c54d04680 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4540 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:58.671+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 1 ==== osd_op(client.4540.0:64 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 264+0+360448 (crc 0 0 0) 0x558c55230840 con 0x558c54c9f000 2019-12-07T19:33:58.671+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78840 prio 63 cost 360448 latency 0.000028 epoch 21 osd_op(client.4540.0:64 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.671+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:64 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 360448 e21) 2019-12-07T19:33:58.671+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.671+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 2 ==== osd_op(client.4540.0:65 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 226+0+63 (crc 0 0 0) 0x558c55230b00 con 0x558c54c9f000 2019-12-07T19:33:58.671+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:64 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 360448 e21) queued 2019-12-07T19:33:58.671+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.671+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78580 prio 63 cost 63 latency 0.000015 epoch 21 osd_op(client.4540.0:65 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.671+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:64 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 360448 e21) pg 0x558c548ab000 2019-12-07T19:33:58.671+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:65 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.671+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e78840 prio 63 cost 360448 latency 0.000069 osd_op(client.4540.0:64 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.671+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 3 ==== osd_op(client.4540.0:66 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 226+0+0 (crc 0 0 0) 0x558c55230dc0 con 0x558c54c9f000 2019-12-07T19:33:58.671+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e782c0 prio 63 cost 0 latency 0.000010 epoch 21 osd_op(client.4540.0:66 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:65 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.671+0000 7f8f06986700 10 osd.7 21 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 0 -> 21 (as per caller) 2019-12-07T19:33:58.671+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.671+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:64 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.671+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.671+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.671+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.671+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: obc NOT found in cache: 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.671+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# _ 2019-12-07T19:33:58.671+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:276d5164:::smithi13140796-22:head# key 0x808000000000000001276d51'd!smithi13140796-22!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.672+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) r -2 v.len 0 2019-12-07T19:33:58.672+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# _ = -2 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: no obc for soid 1:276d5164:::smithi13140796-22:head but can_create 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] create_object_context 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] populate_obc_watchers 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x558c55159440 snapset: 0=[]:{} 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.672+0000 7f8f06986700 25 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op oi 1:276d5164:::smithi13140796-22:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op obc obc(1:276d5164:::smithi13140796-22:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx 0x558c54e99c00 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat] ov 0'0 av 21'1 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat] 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op append 0~360448 in=360448b 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op 1:276d5164:::smithi13140796-22:head [write 0~360448 in=360448b] 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op write 0~360448 in=360448b 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] clean_regions modifiedclean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op stat 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] stat oi has 360448 0.000000 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] make_writeable 1:276d5164:::smithi13140796-22:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] setting DIRTY flag 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] make_writeable 1:276d5164:::smithi13140796-22:head done, snapset=0=[]:{} 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_ctx 1:276d5164:::smithi13140796-22:head 0x558c54e99c00 op modify 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.671144+0000 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_ctx object 1:276d5164:::smithi13140796-22:head marks clean_regions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x558c54e44280 result 0 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] op order client.4540 tid 64 (first) 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] new_repop rep_tid 0 on osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] new_repop: repgather(0x558c55159320 0'0 rep_tid=0 committed?=0 r=0) 2019-12-07T19:33:58.672+0000 7f8f06986700 7 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] issue_repop rep_tid 0 o 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:276d5164:::smithi13140796-22:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:64 client_op=osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.672+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.672+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:276d5164:::smithi13140796-22:head# key 0x808000000000000001276d51'd!smithi13140796-22!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.672+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) r -2 v.len 0 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): 1:276d5164:::smithi13140796-22:head projected size 360448 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] start_rmw: Op(1:276d5164:::smithi13140796-22:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:64 client_op=osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[0~360448]}) 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:276d5164:::smithi13140796-22:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:64 client_op=osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[0~360448]}) 2019-12-07T19:33:58.672+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:64 client_op=osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[0~360448]}) 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:276d5164:::smithi13140796-22:head)[ Extent(0~360448:1) ) 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] generate_transactions: 1:276d5164:::smithi13140796-22:head, current size is 0 buffers are {0~360448(360448)} 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): adding buffer_update 0,360448 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): to_append: {0~360448(360448)} 2019-12-07T19:33:58.672+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): appending 0~360448 2019-12-07T19:33:58.672+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 32 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c551db800 con 0x558c54c64000 2019-12-07T19:33:58.672+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.672+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.672+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 33 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54e3f000 con 0x558c54c64000 2019-12-07T19:33:58.672+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79080 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.672+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000125 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.673+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59200 seq 10 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] encode_and_write: 1:276d5164:::smithi13140796-22:head new_size 360448 2019-12-07T19:33:58.673+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): 1:276d5164:::smithi13140796-22:head resetting hinfo to logical size 360448 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): marking append 0 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e79080 prio 127 cost 0 latency 0.000214 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:276d5164:::smithi13140796-22:head)[ Extent(0~360448:1) ) 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:276d5164:::smithi13140796-22:head={0~360448(360448)}} 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x23a35~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x23a35 0x23a35~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x3a35 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa35 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xea6 and padding block with 0x15a 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:64 client_op=osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[0~360448]}) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:276d5164:::smithi13140796-22:head=[0~360448]} 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:276d5164:::smithi13140796-22:head,{0~360448(360448)} 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.673+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0300 seq 11 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x23ea6 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.673+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 3 keys 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 21 send_incremental_map 19 -> 21 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 3 keys = 0 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107172s (0.000000139s flush + 0.000107033s kv commit) 2019-12-07T19:33:58.673+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.673+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~0 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3ce00 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c54d74070 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d = 589 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 21 add_map_inc_bl 20 589 bytes 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes 0x558c54e44000 shared_blobs 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(20..21 src has 1..21) v4 -- 0x558c57376240 con 0x558c54c64400 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 681 (485 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 21 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c54e41000 con 0x558c54c64400 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.673+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e79080 finish 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c54e41800 con 0x558c54c64000 2019-12-07T19:33:58.673+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 21 send_incremental_map 19 -> 21 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(20..21 src has 1..21) v4 -- 0x558c57376480 con 0x558c54c19800 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 21 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c5748c000 con 0x558c54c19800 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c5748c800 con 0x558c54c5b400 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 21 send_incremental_map 19 -> 21 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(20..21 src has 1..21) v4 -- 0x558c57376fc0 con 0x558c54c19c00 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 21 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.673+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c5748d000 con 0x558c54c19c00 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1721 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.673+0000 7f8f06986700 20 trim 21'1 (0'0) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:64 2019-12-07T19:33:58.671144+0000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1744 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1744 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x23ea6~6d0 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x23ea6 0x23ea6~6d0 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x3ea6 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xea6 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:64 2019-12-07T19:33:58.671144+0000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x576 and padding block with 0xa8a 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'1 (0'0) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:64 2019-12-07T19:33:58.671144+0000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 1 2019-12-07T19:33:58.673+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.673+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.673+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d21500 seq 6 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:276d5164:::smithi13140796-22:head# key 0x808000000000000001276d51'd!smithi13140796-22!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1087 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24576 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# = 0 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys = 0 2019-12-07T19:33:58.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000137291s (0.000000121s flush + 0.000137170s kv commit) 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x0~16000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:276d5164:::smithi13140796-22:head# 0x0~16000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~16000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d21500 11 blobs 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.673+0000 7f8f06986700 10 fbmap_alloc 0x558c53a72e00 allocate 0x16000/1000,16000,0 2019-12-07T19:33:58.673+0000 7f8f06986700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x97000~16000/1000,16000,0 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x97000~16000] 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e568c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e568c0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56930) get_ref 0x0~2000 Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e568c0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e568d8 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56a80 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56b60 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56a80) get_ref 0x0~2000 Blob(0x558c54e56a80 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56b60 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54e56a80 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e56b78 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e569a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e569a0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56a10 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e569a0) get_ref 0x0~2000 Blob(0x558c54e569a0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56a10 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e569a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54e569a0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e56a28 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56540 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56af0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56540) get_ref 0x0~2000 Blob(0x558c54e56540 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56af0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54e56540 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e56b08 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e564d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e563f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e564d0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e563f0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e564d0) get_ref 0x0~2000 Blob(0x558c54e564d0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e563f0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e564d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54e564d0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e56408 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42c40 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42af0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e42c40) get_ref 0x0~2000 Blob(0x558c54e42c40 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42af0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e42c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54e42c40 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e42b08 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43dc0 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43d50 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e43dc0) get_ref 0x0~2000 Blob(0x558c54e43dc0 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43d50 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e43dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c54e43dc0 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e43d68 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43ce0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42d20 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e43ce0) get_ref 0x0~2000 Blob(0x558c54e43ce0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42d20 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e43ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c54e43ce0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e42d38 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56bd0 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56c40 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56bd0) get_ref 0x0~2000 Blob(0x558c54e56bd0 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56c40 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c54e56bd0 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e56c58 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56cb0 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d20 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56cb0) get_ref 0x0~2000 Blob(0x558c54e56cb0 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d20 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c54e56cb0 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e56d38 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56d90 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56e00 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56d90) get_ref 0x0~2000 Blob(0x558c54e56d90 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56e00 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.blob(0x558c54e56d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c54e56d90 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 sbid 0x0)) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e56e18 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x16000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.extentmap(0x558c54e44b08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x0~16000 = 0 2019-12-07T19:33:58.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 184542 (23 ios * 4000 + 92542 bytes) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes 0x558c54e44a00 shared_blobs 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.extentmap(0x558c54e44b08) update 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore.extentmap(0x558c54e44b08) update inline shard 212 bytes from 11 extents 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:head# is 631 (413 bytes onode + 2 bytes spanning blobs + 216 bytes inline extents) 2019-12-07T19:33:58.673+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[97000~16000] released 0x[] 2019-12-07T19:33:58.673+0000 7f8f06986700 10 freelist allocate 0x97000~16000 2019-12-07T19:33:58.673+0000 7f8f06986700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:33:58.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d21500 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'1 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x558c55159320 21'1 rep_tid=0 committed?=0 r=0) 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e78840 finish 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200,0x558c54ea0300> 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.674+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:65 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558c548ab000 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.674+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78580 prio 63 cost 63 latency 0.002680 osd_op(client.4540.0:65 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 0x558c54e44000 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.674+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:65 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.674+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=36, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c5501d400 con 0x558c54c64000 2019-12-07T19:33:58.674+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.674+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=37, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c54c1fe00 con 0x558c54c64000 2019-12-07T19:33:58.674+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.674+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.674+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:33:58.674+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(excl n=1 w=0) oi: 1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: 0=[]:{} 2019-12-07T19:33:58.674+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 aio_wait 2019-12-07T19:33:58.674+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.674+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:33:58.674+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:33:58.674+0000 7f8f0297e700 25 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.674+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.674+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78580 finish 2019-12-07T19:33:58.674+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e782c0 prio 63 cost 0 latency 0.002825 osd_op(client.4540.0:66 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:66 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2157 < min_flush_size 524288 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44a00 had 1 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2180 < min_flush_size 524288 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2180 < min_flush_size 524288 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24576~884 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24576 0x24576~884 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x4576 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x576 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdfa and padding block with 0x206 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(excl n=1 w=1) oi: 1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: 0=[]:{} 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.674+0000 7f8f06986700 25 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.674+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e782c0 finish 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24dfa 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.674+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000158512s (0.000050225s flush + 0.000108287s kv commit) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21500> 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 0x558c54e44a00 2019-12-07T19:33:58.674+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42b08 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b680 space 0x558c54e42b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42d38 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c54e42d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43d68 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b710 space 0x558c54e43d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56408 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b5f0 space 0x558c54e56408 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e568d8 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b200 space 0x558c54e568d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56a28 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9afc0 space 0x558c54e56a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56b08 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b050 space 0x558c54e56b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56b78 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9af30 space 0x558c54e56b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56c58 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b830 space 0x558c54e56c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56d38 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c54e56d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56e18 in 0x558c53a59260) _finish_write discard buffer(0x558c54c9b950 space 0x558c54e56e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:33:58.674+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.675+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 6 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c1fe00 con 0x558c54c19c00 2019-12-07T19:33:58.675+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.675+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.675+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000051 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.675+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.675+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 15 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c55250600 con 0x558c54c5b400 2019-12-07T19:33:58.675+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.675+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.675+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000045 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.675+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.675+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:33:58.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 16 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c5748c800 con 0x558c54c5b400 2019-12-07T19:33:58.675+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.675+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.675+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 34 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5501d400 con 0x558c54c64000 2019-12-07T19:33:58.675+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79080 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.675+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.675+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e79080 prio 196 cost 0 latency 0.000059 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.675+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.675+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e79080 finish 2019-12-07T19:33:58.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.675+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.675+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000254 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.676+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.676+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.676+0000 7f8f07187700 20 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.676+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.676+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.676+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.676+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.676+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.676+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea0f00 seq 8 2019-12-07T19:33:58.676+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.676+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.676+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.676+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.676+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.676+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.676+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:33:58.676+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.676+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:33:58.676+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:33:58.676+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:33:58.676+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.676+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24dfa~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24dfa 0x24dfa~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x4dfa 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdfa 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x26b and padding block with 0xd95 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2526b 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.676+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000096429s (0.000000086s flush + 0.000096343s kv commit) 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:33:58.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:33:58.676+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.676+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.676+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=6, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c55270800 con 0x558c54c5b400 2019-12-07T19:33:58.676+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 35 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748d000 con 0x558c54c64000 2019-12-07T19:33:58.677+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.677+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.677+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1800 seq 12 2019-12-07T19:33:58.677+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1800 onodes shared_blobs 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 prepare 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1800 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 io_done 2019-12-07T19:33:58.677+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2526b~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2526b 0x2526b~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x526b 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x26b 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6dc and padding block with 0x924 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x256dc 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.677+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000214933s (0.000000104s flush + 0.000214829s kv commit) 2019-12-07T19:33:58.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1800> 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.677+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 kv_submitted 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1800 2019-12-07T19:33:58.677+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 finishing 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1800 onodes 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1800 done 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.677+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1800 [] 2019-12-07T19:33:58.677+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.677+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=38, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c55271c00 con 0x558c54c64000 2019-12-07T19:33:58.677+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 11 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55250e00 con 0x558c54c19800 2019-12-07T19:33:58.678+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e786e0 prio 196 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.678+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.678+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e786e0 prio 196 cost 0 latency 0.000364 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.678+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.678+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:33:58.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 6 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c1fa00 con 0x558c54c64400 2019-12-07T19:33:58.678+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e79760 prio 196 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.678+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e79760 prio 196 cost 0 latency 0.000070 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:276d5164:::smithi13140796-22:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:64 client_op=osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[0~360448]}) 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] repop_all_committed: repop tid 0 all committed 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x558c55159320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] commit: repgather(0x558c55159320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.678+0000 7f8f0297e700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] log_op_stats osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 360448 outb 0 lat 0.006899 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] sending reply on osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558c54e44280 2019-12-07T19:33:58.678+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(64 smithi13140796-22 [append 0~360448,stat] v21'1 uv1 ondisk = 0) v8 -- 0x558c54e44280 con 0x558c54c9f000 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.678+0000 7f8f0297e700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:13 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] removing repgather(0x558c55159320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] q front is repgather(0x558c55159320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.678+0000 7f8f0297e700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] remove_repop repgather(0x558c55159320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] requeue_op 0x558c54e782c0 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] requeue_op 0x558c54e78580 2019-12-07T19:33:58.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) shuffled w/ OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:64 client_op=osd_op(client.4540.0:64 1.4s0 1:276d5164:::smithi13140796-22:head [append 0~360448 in=360448b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[0~360448]}) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: ExtentCache( Extents(1:276d5164:::smithi13140796-22:head)[ Extent(0~360448:1) ) 2019-12-07T19:33:58.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=1 reqid=client.4540.0:64 roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=1 reqid=client.4540.0:64 roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.678+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.678+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c551dd000 con 0x558c54c64400 2019-12-07T19:33:58.678+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c5748c800 con 0x558c54c64000 2019-12-07T19:33:58.678+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c5748d800 con 0x558c54c19800 2019-12-07T19:33:58.679+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c551dc000 con 0x558c54c5b400 2019-12-07T19:33:58.679+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c54d85000 con 0x558c54c19c00 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] rollforward: entry=21'1 (0'0) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:64 2019-12-07T19:33:58.671144+0000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 1 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.679+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d58300 seq 7 2019-12-07T19:33:58.679+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes shared_blobs 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e79760 finish 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558c548ab000 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78580 prio 63 cost 63 latency 0.007360 osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: 0=[]:{} 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.679+0000 7f8f0297e700 25 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:276d5164:::smithi13140796-22:head(21'1 client.4540.0:64 dirty|data_digest s 360448 uv 1 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op obc obc(1:276d5164:::smithi13140796-22:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x558c54e99c00 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] ov 21'1 av 21'2 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_osd_op setxattr _header (56) in=63b 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:276d5164:::smithi13140796-22:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:276d5164:::smithi13140796-22:head done, snapset=0=[]:{} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] finish_ctx 1:276d5164:::smithi13140796-22:head 0x558c54e99c00 op modify 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.671204+0000 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] finish_ctx object 1:276d5164:::smithi13140796-22:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x558c54e45180 result 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 21'1 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] op order client.4540 tid 65 last was 64 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] new_repop rep_tid 2 on osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] new_repop: repgather(0x558c55159320 0'0 rep_tid=2 committed?=0 r=0) 2019-12-07T19:33:58.679+0000 7f8f0297e700 7 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] issue_repop rep_tid 2 o 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:276d5164:::smithi13140796-22:head v=21'2 tt=0'0 tid=2 reqid=client.4540.0:65 client_op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: found on disk, size 90112 2019-12-07T19:33:58.679+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:276d5164:::smithi13140796-22:head projected size 360448 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] start_rmw: Op(1:276d5164:::smithi13140796-22:head v=21'2 tt=0'0 tid=2 reqid=client.4540.0:65 client_op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:276d5164:::smithi13140796-22:head v=21'2 tt=0'0 tid=2 reqid=client.4540.0:65 client_op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:head v=21'2 tt=0'0 tid=2 reqid=client.4540.0:65 client_op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] generate_transactions: 1:276d5164:::smithi13140796-22:head, current size is 360448 buffers are {} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] operator(): new_size start 360448 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:276d5164:::smithi13140796-22:head resetting hinfo to logical size 360448 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:276d5164:::smithi13140796-22:head={}} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:head v=21'2 tt=0'0 tid=2 reqid=client.4540.0:65 client_op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:276d5164:::smithi13140796-22:head=[]} 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:276d5164:::smithi13140796-22:head,{} 2019-12-07T19:33:58.679+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c5748e000 con 0x558c54c64400 2019-12-07T19:33:58.679+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c5748e800 con 0x558c54c64000 2019-12-07T19:33:58.679+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c5748f000 con 0x558c54c19800 2019-12-07T19:33:58.679+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c5748f800 con 0x558c54c5b400 2019-12-07T19:33:58.679+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x558c574a2000 con 0x558c54c19c00 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 trim 21'2 (21'1) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:65 2019-12-07T19:33:58.671204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:65 2019-12-07T19:33:58.671204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 lua=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'2 (21'1) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:65 2019-12-07T19:33:58.671204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 lua=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 2 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 lua=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.679+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d58f00 seq 8 2019-12-07T19:33:58.679+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 3 keys 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 3 keys = 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes 0x558c54e44a00 shared_blobs 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore.extentmap(0x558c54e44b08) update 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:head# is 703 (485 bytes onode + 2 bytes spanning blobs + 216 bytes inline extents) 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 lua=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'2 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x558c55159320 21'2 rep_tid=2 committed?=0 r=0) 2019-12-07T19:33:58.679+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78580 finish 2019-12-07T19:33:58.679+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.679+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e782c0 prio 63 cost 0 latency 0.007986 osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.679+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.679+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.679+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.679+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.679+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:33:58.679+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(write n=1 w=0) oi: 1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: 0=[]:{} 2019-12-07T19:33:58.679+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.679+0000 7f8f06986700 25 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.679+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.679+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.679+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e782c0 finish 2019-12-07T19:33:58.679+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 2 2019-12-07T19:33:58.679+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2857 < min_flush_size 524288 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44a00 had 1 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:33:58.679+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2880 < min_flush_size 524288 2019-12-07T19:33:58.679+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2880 < min_flush_size 524288 2019-12-07T19:33:58.679+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.679+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x256dc~b40 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.679+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x256dc 0x256dc~b40 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x56dc 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6dc 2019-12-07T19:33:58.679+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.680+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x21c and padding block with 0xde4 2019-12-07T19:33:58.680+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2621c 2019-12-07T19:33:58.680+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.680+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.680+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.680+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000119895s (0.000000105s flush + 0.000119790s kv commit) 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300,0x558c54d58f00> 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 kv_submitted 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 0x558c54e44a00 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:33:58.680+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 7 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54d76200 con 0x558c54c64400 2019-12-07T19:33:58.680+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c552b69a0 prio 196 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.680+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c552b69a0 prio 196 cost 0 latency 0.000046 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.680+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c552b69a0 finish 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 17 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d77000 con 0x558c54c5b400 2019-12-07T19:33:58.680+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79b80 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79b80 prio 196 cost 0 latency 0.000052 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 18 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c548cc000 con 0x558c54c5b400 2019-12-07T19:33:58.680+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.680+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 196 cost 0 latency 0.000052 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.680+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.680+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 8 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c1ec00 con 0x558c54c64400 2019-12-07T19:33:58.680+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e78840 prio 196 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 7 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c574a0000 con 0x558c54c19c00 2019-12-07T19:33:58.680+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79340 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.680+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.680+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 8 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d76c00 con 0x558c54c19c00 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78f20 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e78840 prio 196 cost 0 latency 0.000050 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e78840 finish 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79340 prio 196 cost 0 latency 0.000068 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79340 finish 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e78f20 prio 196 cost 0 latency 0.000071 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:33:58.680+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.682+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 12 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54d76400 con 0x558c54c19800 2019-12-07T19:33:58.682+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.682+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.682+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 13 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54d77c00 con 0x558c54c19800 2019-12-07T19:33:58.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.682+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000005 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.682+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.682+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.682+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.682+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.682+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.682+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.682+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:33:58.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.689+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 36 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 104372+0+0 (crc 0 0 0) 0x558c54e41800 con 0x558c54c64000 2019-12-07T19:33:58.689+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000038 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.689+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.689+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.689+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.689+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.689+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.689+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.689+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.689+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.689+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.689+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.690+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c548f6c00 seq 13 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ee170468:::smithi13140796-25:head# key 0x848000000000000001ee1704'h!smithi13140796-25!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.690+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1088 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# = 0 2019-12-07T19:33:58.690+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 2 keys 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 2 keys = 0 2019-12-07T19:33:58.690+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.690+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ee170468:::smithi13140796-25:head# 0x0~19000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~19000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6c00 13 blobs 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.690+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate 0x19000/1000,19000,0 2019-12-07T19:33:58.690+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xad000~19000/1000,19000,0 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xad000~19000] 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e579d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57500 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e579d0) get_ref 0x0~2000 Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57500 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e579d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57518 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e578f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e578f0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57960) get_ref 0x0~2000 Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e578f0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57908 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e577a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56070 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e577a0) get_ref 0x0~2000 Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56070 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e577a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e56088 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e573b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d570 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e573b0) get_ref 0x0~2000 Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d570 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e573b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d3d588 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d490 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d500) get_ref 0x0~2000 Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d490 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d3d4a8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d3b0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d420) get_ref 0x0~2000 Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d3b0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d3d3c8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56620 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57340) get_ref 0x0~2000 Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56620 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e56638 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57ab0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57810) get_ref 0x0~2000 Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57ab0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57ac8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57b90 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57b20) get_ref 0x0~2000 Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57b90 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57ba8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57c70 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57c00) get_ref 0x0~2000 Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57c70 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57c88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57d50 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57ce0) get_ref 0x0~2000 Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57d50 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57d68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57e30 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57dc0) get_ref 0x0~2000 Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57e30 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57e48 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57f10 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57ea0) get_ref 0x0~1000 Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57f10 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.blob(0x558c54e57ea0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~1000: 0x0~1000 Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e57f10 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e57f28 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x19000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 0 2019-12-07T19:33:58.690+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.690+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 160022 (14 ios * 4000 + 104022 bytes) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes 0x558c574a0000 shared_blobs 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0108) update 4#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0108) update inline shard 247 bytes from 13 extents 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ee170468:::smithi13140796-25:head# is 666 (413 bytes onode + 2 bytes spanning blobs + 251 bytes inline extents) 2019-12-07T19:33:58.690+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[ad000~19000] released 0x[] 2019-12-07T19:33:58.690+0000 7f8f05183700 10 freelist allocate 0xad000~19000 2019-12-07T19:33:58.690+0000 7f8f05183700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:33:58.690+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 37 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54ceaa00 con 0x558c54c64000 2019-12-07T19:33:58.690+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6c00 2019-12-07T19:33:58.690+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78840 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.690+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.690+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.690+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:33:58.690+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.690+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.690+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.690+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.690+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.690+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78840 prio 196 cost 0 latency 0.000315 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.690+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.690+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=1 reqid=client.4540.0:64 roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.690+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.690+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78840 finish 2019-12-07T19:33:58.690+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.690+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 aio_wait 2019-12-07T19:33:58.690+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:33:58.690+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1378 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0000 had 1 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1401 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1401 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2621c~579 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2621c 0x2621c~579 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x621c 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x21c 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.690+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x795 and padding block with 0x86b 2019-12-07T19:33:58.691+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x26795 2019-12-07T19:33:58.691+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.691+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.691+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.691+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.691+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000167017s (0.000038643s flush + 0.000128374s kv commit) 2019-12-07T19:33:58.691+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6c00> 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.691+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:33:58.691+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 finishing 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 0x558c574a0000 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d3c8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b680 space 0x558c54d3d3c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d4a8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9aea0 space 0x558c54d3d4a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d588 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9a120 space 0x558c54d3d588 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56088 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9a090 space 0x558c54e56088 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56638 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c54e56638 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57518 in 0x558c53a59880) _finish_write discard buffer(0x558c551a9710 space 0x558c54e57518 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57908 in 0x558c53a59880) _finish_write discard buffer(0x558c551a9680 space 0x558c54e57908 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57ac8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b710 space 0x558c54e57ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57ba8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b5f0 space 0x558c54e57ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57c88 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b200 space 0x558c54e57c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57d68 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9afc0 space 0x558c54e57d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57e48 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b050 space 0x558c54e57e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57f28 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9af30 space 0x558c54e57f28 0x0~1000 writing nocache) 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.691+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:33:58.691+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.691+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.691+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=40, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c54c1fc00 con 0x558c54c64000 2019-12-07T19:33:58.691+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.692+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 38 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c1fc00 con 0x558c54c64000 2019-12-07T19:33:58.692+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79340 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.692+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79340 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:276d5164:::smithi13140796-22:head v=21'2 tt=0'0 tid=2 reqid=client.4540.0:65 client_op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] repop_all_committed: repop tid 2 all committed 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] eval_repop repgather(0x558c55159320 21'2 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] commit: repgather(0x558c55159320 21'2 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.692+0000 7f8f06986700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] log_op_stats osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.020715 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] sending reply on osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558c54e45180 2019-12-07T19:33:58.692+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(65 smithi13140796-22 [setxattr (56)] v21'2 uv2 ondisk = 0) v8 -- 0x558c54e45180 con 0x558c54c9f000 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.692+0000 7f8f06986700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] publish_stats_to_osd 21:14 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] removing repgather(0x558c55159320 21'2 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] q front is repgather(0x558c55159320 21'2 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.692+0000 7f8f06986700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] remove_repop repgather(0x558c55159320 21'2 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] requeue_op 0x558c54e782c0 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:head v=21'2 tt=0'0 tid=2 reqid=client.4540.0:65 client_op=osd_op(client.4540.0:65 1.4s0 1:276d5164:::smithi13140796-22:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=3 reqid=client.4540.0:65 roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=3 reqid=client.4540.0:65 roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.692+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c551dd800 con 0x558c54c64400 2019-12-07T19:33:58.692+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c551dc000 con 0x558c54c64000 2019-12-07T19:33:58.692+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c551dd000 con 0x558c54c19800 2019-12-07T19:33:58.692+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c54d85800 con 0x558c54c5b400 2019-12-07T19:33:58.692+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c5748d000 con 0x558c54c19c00 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] rollforward: entry=21'2 (21'1) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:65 2019-12-07T19:33:58.671204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log approx pg log length = 2 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.692+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.692+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.692+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:33:58.692+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d59500 seq 9 2019-12-07T19:33:58.692+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:33:58.692+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.692+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.692+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.692+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:33:58.692+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.692+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59500 onodes shared_blobs 2019-12-07T19:33:58.692+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.692+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 prepare 2019-12-07T19:33:58.692+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59500 2019-12-07T19:33:58.692+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 io_done 2019-12-07T19:33:58.692+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79340 finish 2019-12-07T19:33:58.692+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e782c0 prio 63 cost 0 latency 0.020973 osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: 0=[]:{} 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.692+0000 7f8f0297e700 25 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:33:58.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op obc obc(1:276d5164:::smithi13140796-22:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 0x558c548a1c00 2019-12-07T19:33:58.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.692+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 1:276d5164:::smithi13140796-22:head [read 0~1] ov 21'2 2019-12-07T19:33:58.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x26795~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x26795 0x26795~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_osd_op 1:276d5164:::smithi13140796-22:head [read 0~1] 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x6795 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x795 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.692+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc06 and padding block with 0x3fa 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.692+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_read 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] async_read noted for 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.692+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_all_avail_shards: checking acting 0(1) 2019-12-07T19:33:58.693+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x26c06 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:33:58.693+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.693+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.693+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.693+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_all_avail_shards: checking acting 4(5) 2019-12-07T19:33:58.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000099042s (0.000000140s flush + 0.000098902s kv commit) 2019-12-07T19:33:58.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_all_avail_shards: checking acting 5(3) 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] start_read_op: starting ReadOp(tid=4, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,16384,0], need={0(1)=[0,8],1(2)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_read_op: starting read ReadOp(tid=4, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,16384,0], need={0(1)=[0,8],1(2)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.693+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c54e45400 con 0x558c54c64400 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59500> 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 kv_submitted 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59500 2019-12-07T19:33:58.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 finishing 2019-12-07T19:33:58.693+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.4s2 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c54e45680 con 0x558c54c64000 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59500 onodes 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59500 done 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:33:58.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59500 [] 2019-12-07T19:33:58.693+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.693+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.4s3 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c54e45900 con 0x558c54c19c00 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 21 send_incremental_map 0 -> 21 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:33:58.693+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3a80e826:::osdmap.21:0# 0x0~0 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74540 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d745b0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74620 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558c54d74690 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3a80e826:::osdmap.21:0# 0x0~140c = 5132 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 21 add_map_bl 21 5132 bytes 2019-12-07T19:33:58.693+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(21..21 src has 1..21) v4 -- 0x558c54d2eb40 con 0x558c53ae5400 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 21 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 0 -> 21 (shared) 2019-12-07T19:33:58.693+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c574a0a00 con 0x558c53ae5400 2019-12-07T19:33:58.693+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(21..21 src has 1..21) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c54d2eb40 con 0x558c53ae5400 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_read_op: started ReadOp(tid=4, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,16384,0], need={0(1)=[0,8],1(2)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:276d5164:::smithi13140796-22:head=0(1),1(2),5(3),7(0)}, source_to_obj={0(1)=1:276d5164:::smithi13140796-22:head,1(2)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=0(1),1(2),5(3),7(0)) 2019-12-07T19:33:58.693+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c574a0a00 con 0x558c53ae5400 2019-12-07T19:33:58.693+0000 7f8f1199c700 20 osd.7 21 OSD::ms_dispatch: osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.693+0000 7f8f1199c700 10 osd.7 21 do_waiters -- start 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e782c0 finish 2019-12-07T19:33:58.693+0000 7f8f1199c700 10 osd.7 21 do_waiters -- finish 2019-12-07T19:33:58.693+0000 7f8f1199c700 20 osd.7 21 _dispatch 0x558c54d2eb40 osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.693+0000 7f8f1119b700 15 osd.7 21 enqueue_op 0x558c54e79ce0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.693+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f8f1199c700 3 osd.7 21 handle_osd_map epochs [21,21], i have 21, src has [1,21] 2019-12-07T19:33:58.693+0000 7f8f1199c700 10 osd.7 21 no new maps here, dropping 2019-12-07T19:33:58.693+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.693+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.693+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 9 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d77a00 con 0x558c54c19c00 2019-12-07T19:33:58.693+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79ce0 prio 127 cost 0 latency 0.000105 MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.693+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78f20 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.693+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpRead(1.4s0 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.693+0000 7f8f06986700 25 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.693+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x0~1000 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x16000 (90112) 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.693+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 19 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d76800 con 0x558c54c5b400 2019-12-07T19:33:58.693+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.693+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.693+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e568d8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:33:58.693+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c551a97a0 space 0x558c54e568d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.693+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x0~1000 = 4096 2019-12-07T19:33:58.693+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 20 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 112564+0+0 (unknown 0 0 0) 0x558c551dc000 con 0x558c54c5b400 2019-12-07T19:33:58.693+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.693+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.693+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.693+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.693+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.693+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: found on disk, size 90112 2019-12-07T19:33:58.693+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:33:58.693+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:33:58.693+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 -- 0x558c54e44780 con 0x558c53ae5400 2019-12-07T19:33:58.693+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79ce0 finish 2019-12-07T19:33:58.693+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 10 ==== MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558c54e45900 con 0x558c54c19c00 2019-12-07T19:33:58.693+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.693+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.693+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c54e44780 con 0x558c53ae5400 2019-12-07T19:33:58.693+0000 7f8f1119b700 15 osd.7 21 enqueue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.693+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78f20 prio 196 cost 0 latency 0.000702 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.693+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:33:58.694+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000497 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000198 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=4, attrs_read=0) 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=4, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,16384,0], need={0(1)=[0,8],1(2)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [7(0),4096]))}, priority=127, obj_to_source={1:276d5164:::smithi13140796-22:head=0(1),1(2),5(3),7(0)}, source_to_obj={0(1)=1:276d5164:::smithi13140796-22:head,1(2)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=0(1),1(2),5(3)) 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.694+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000270 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.694+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.694+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=4, attrs_read=0) 2019-12-07T19:33:58.694+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=4, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,16384,0], need={0(1)=[0,8],1(2)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [5(3),4096, 7(0),4096]))}, priority=127, obj_to_source={1:276d5164:::smithi13140796-22:head=0(1),1(2),5(3),7(0)}, source_to_obj={0(1)=1:276d5164:::smithi13140796-22:head,1(2)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=0(1),1(2)) 2019-12-07T19:33:58.694+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:33:58.694+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 39 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c64000 2019-12-07T19:33:58.694+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79080 prio 196 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.694+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.694+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 40 ==== MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 ==== 4261+0+0 (crc 0 0 0) 0x558c54e45400 con 0x558c54c64000 2019-12-07T19:33:58.694+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e789a0 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.694+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79080 prio 196 cost 0 latency 0.000047 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79080 finish 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e789a0 prio 127 cost 0 latency 0.000045 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=4, attrs_read=0) 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=4, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,16384,0], need={0(1)=[0,8],1(2)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(2),4096, 5(3),4096, 7(0),4096]))}, priority=127, obj_to_source={1:276d5164:::smithi13140796-22:head=0(1),1(2),5(3),7(0)}, source_to_obj={0(1)=1:276d5164:::smithi13140796-22:head,1(2)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=0(1)) 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e789a0 finish 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000814 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.694+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea0000 seq 9 2019-12-07T19:33:58.694+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 9 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54d77800 con 0x558c54c64400 2019-12-07T19:33:58.694+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e79760 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.694+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:cd22b62d:::smithi13140796-27:head# key 0x848000000000000001cd22b6'-!smithi13140796-27!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.694+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1089 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# = 0 2019-12-07T19:33:58.694+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2 keys 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2 keys = 0 2019-12-07T19:33:58.694+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.694+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x0~1b000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:cd22b62d:::smithi13140796-27:head# 0x0~1b000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1b000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.694+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 10 ==== MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 ==== 4261+0+0 (crc 0 0 0) 0x558c574a1900 con 0x558c54c64400 2019-12-07T19:33:58.694+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c552b69a0 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.694+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0000 14 blobs 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79760 prio 196 cost 0 latency 0.000098 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c54e79760 finish 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c552b69a0 prio 127 cost 0 latency 0.000093 MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=4, attrs_read=0) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1b000/1000,1b000,0 2019-12-07T19:33:58.694+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc6000~1b000/1000,1b000,0 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc6000~1b000] 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0cb0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42a80) get_ref 0x0~2000 Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0cb0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e0cc8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e427e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e427e0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54d75f80) get_ref 0x0~2000 Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e427e0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54d75f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e427f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_read_reply Complete: ReadOp(tid=4, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,16384,0], need={0(1)=[0,8],1(2)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(1),4096, 1(2),4096, 5(3),4096, 7(0),4096]))}, priority=127, obj_to_source={1:276d5164:::smithi13140796-22:head=0(1),1(2),5(3),7(0)}, source_to_obj={0(1)=1:276d5164:::smithi13140796-22:head,1(2)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e428c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e428c0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42770) get_ref 0x0~2000 Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e428c0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e428d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e429a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e429a0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42930) get_ref 0x0~2000 Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e429a0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] objects_read_async_cb: got: {1:276d5164:::smithi13140796-22:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e429b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1500 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42a10) get_ref 0x0~2000 Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1500 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c54e42a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 0x558c548a1c00 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1518 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e1490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1420 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 1:276d5164:::smithi13140796-22:head [read 0~1 out=1b] ov 21'2 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1490) get_ref 0x0~2000 Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1420 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1438 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_osd_op 1:276d5164:::smithi13140796-22:head [read 0~1 out=1b] 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e13b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1340 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f0317f700 20 bluestore.blob(0x558c574e13b0) get_ref 0x0~2000 Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1340 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e13b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx alloc reply 0x558c54e45400 result 0 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1358 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e12d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.695+0000 7f8f06986700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1260 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e12d0) get_ref 0x0~2000 Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1260 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e12d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f06986700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] log_op_stats osd_op(client.4540.0:66 1.4s0 1:276d5164:::smithi13140796-22:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.023175 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1278 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e11f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.695+0000 7f8f06986700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1180 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e11f0) get_ref 0x0~2000 Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1180 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f06986700 15 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] publish_stats_to_osd 21:15 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e11f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1198 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.695+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(66 smithi13140796-22 [read 0~1 out=1b] v0'0 uv2 ondisk = 0) v8 -- 0x558c54e45400 con 0x558c54c9f000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e1110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e10a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e10a0 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1110) get_ref 0x0~2000 Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e10a0 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e10b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e1030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0ee0 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f06986700 10 osd.7 21 dequeue_op 0x558c552b69a0 finish 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1030) get_ref 0x0~2000 Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0ee0 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.695+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e0ef8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e0fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0f50 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e0fc0) get_ref 0x0~2000 Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0f50 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e0fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e0f68 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e08c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0a80 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e08c0) get_ref 0x0~2000 Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0a80 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e08c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e0a98 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e0850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0a10 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e0850) get_ref 0x0~1000 Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0a10 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.blob(0x558c574e0850) get_ref init 0x1000, 1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~1000: 0x0~1000 Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e0a28 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1b000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x0~1b000 = 0 2019-12-07T19:33:58.695+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.695+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 172214 (15 ios * 4000 + 112214 bytes) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes 0x558c574a1680 shared_blobs 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) update 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) update inline shard 266 bytes from 14 extents 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:cd22b62d:::smithi13140796-27:head# is 685 (413 bytes onode + 2 bytes spanning blobs + 270 bytes inline extents) 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[c6000~1b000] released 0x[] 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 freelist allocate 0xc6000~1b000 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0000 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.695+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:33:58.695+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.696+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 aio_wait 2019-12-07T19:33:58.696+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:33:58.696+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:33:58.696+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.696+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.696+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.698+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 21 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 120756+0+0 (unknown 0 0 0) 0x558c5748d000 con 0x558c54c5b400 2019-12-07T19:33:58.698+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.698+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.698+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.698+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.698+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.698+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.698+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000078 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.3s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.699+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.699+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 lcod 21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.699+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.699+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.699+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.699+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.699+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea0c00 seq 10 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:c8ce3553:::smithi13140796-29:head# key 0x848000000000000001c8ce35'S!smithi13140796-29!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.699+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1090 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# = 0 2019-12-07T19:33:58.699+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys = 0 2019-12-07T19:33:58.699+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.699+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1d000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0c00 15 blobs 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1d000/1000,1d000,0 2019-12-07T19:33:58.699+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xe1000~1d000/1000,1d000,0 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xe1000~1d000] 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c54d3c070) get_ref 0x0~2000 Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c54d3c070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0c58 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c310 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c54d3c3f0) get_ref 0x0~2000 Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c310 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c54d3c3f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54d3c328 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c4d0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c54d3c150) get_ref 0x0~2000 Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c4d0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c54d3c150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54d3c4e8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2000 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c55265e30) get_ref 0x0~2000 Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2000 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c55265e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2018 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e20e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e20e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2070) get_ref 0x0~2000 Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e20e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e20f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e21c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e21c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2150) get_ref 0x0~2000 Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e21c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e21d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e22a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e22a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2230) get_ref 0x0~2000 Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e22a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e22b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2380 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2310) get_ref 0x0~2000 Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2380 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2398 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e23f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2460 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e23f0) get_ref 0x0~2000 Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2460 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e23f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2478 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e24d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2540 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e24d0) get_ref 0x0~2000 Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2540 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e24d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2558 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e25b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2620 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e25b0) get_ref 0x0~2000 Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2620 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e25b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2638 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2700 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2690) get_ref 0x0~2000 Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2700 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2718 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2770) get_ref 0x0~2000 Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e27f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2850) get_ref 0x0~2000 Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e28d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2930) get_ref 0x0~1000 Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.blob(0x558c574e2930) get_ref init 0x1000, 1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~1000: 0x0~1000 Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e29b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1d000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 = 0 2019-12-07T19:33:58.699+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.699+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 184406 (16 ios * 4000 + 120406 bytes) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes 0x558c57454000 shared_blobs 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) update 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) update inline shard 285 bytes from 15 extents 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:head# is 704 (413 bytes onode + 2 bytes spanning blobs + 289 bytes inline extents) 2019-12-07T19:33:58.699+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[e1000~1d000] released 0x[] 2019-12-07T19:33:58.699+0000 7f8f07187700 10 freelist allocate 0xe1000~1d000 2019-12-07T19:33:58.699+0000 7f8f07187700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:33:58.699+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0c00 2019-12-07T19:33:58.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1397 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1680 had 1 2019-12-07T19:33:58.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 2 2019-12-07T19:33:58.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x26c06~58c to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.699+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x26c06 0x26c06~58c to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.699+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.699+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x6c06 2019-12-07T19:33:58.699+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc06 2019-12-07T19:33:58.699+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.699+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x192 and padding block with 0xe6e 2019-12-07T19:33:58.700+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.700+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c54e78420 finish 2019-12-07T19:33:58.700+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.700+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x27192 2019-12-07T19:33:58.700+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.700+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.700+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.700+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.701+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 14 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 124852+0+0 (crc 0 0 0) 0x558c551dd000 con 0x558c54c19800 2019-12-07T19:33:58.701+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.701+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 15 ==== MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54d76e00 con 0x558c54c19800 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.701+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e79340 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558c548ab000 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.701+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e79340 prio 196 cost 0 latency 0.000047 MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.701+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:88 2019-12-07T19:33:58.696853+0000 0 ObjectCleanRegions clean_offsets: [491520~18446744073709060095], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.701+0000 7f8f0297e700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=3 reqid=client.4540.0:65 roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.701+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:88 2019-12-07T19:33:58.696853+0000 0 ObjectCleanRegions clean_offsets: [491520~18446744073709060095], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.701+0000 7f8f0297e700 20 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:33:58.701+0000 7f8f0297e700 10 osd.7 21 dequeue_op 0x558c54e79340 finish 2019-12-07T19:33:58.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.701+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d58c00 seq 5 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bca18876:::smithi13140796-30:head# key 0x828000000000000001bca188'v!smithi13140796-30!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.701+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1091 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# = 0 2019-12-07T19:33:58.701+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:33:58.701+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.701+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1e000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58c00 15 blobs 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1e000/1000,1e000,0 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xfe000~1e000/1000,1e000,0 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xfe000~1e000] 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d1f0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3340) get_ref 0x0~2000 Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d1f0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c54d3d208 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3650 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e36c0) get_ref 0x0~2000 Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3650 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e36c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3668 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e35e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3570 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e35e0) get_ref 0x0~2000 Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3570 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e35e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3588 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3420 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3500) get_ref 0x0~2000 Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3420 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3438 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e33b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e32d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e32d0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e33b0) get_ref 0x0~2000 Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e32d0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e33b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e32e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e37a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e37a0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3730) get_ref 0x0~2000 Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e37a0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e37b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3880 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3810) get_ref 0x0~2000 Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3880 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3898 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e38f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3960 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e38f0) get_ref 0x0~2000 Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3960 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e38f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3978 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e39d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3a40 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e39d0) get_ref 0x0~2000 Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3a40 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e39d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3a58 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3b20 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3ab0) get_ref 0x0~2000 Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3b20 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3b38 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3c00 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3b90) get_ref 0x0~2000 Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3c00 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3c18 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ce0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3c70) get_ref 0x0~2000 Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ce0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3cf8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3dc0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3d50) get_ref 0x0~2000 Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3dc0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3dd8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ea0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3e30) get_ref 0x0~2000 Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ea0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3eb8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e3f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3f80 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3f10) get_ref 0x0~2000 Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3f80 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574e3f98 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1e000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 = 0 2019-12-07T19:33:58.701+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.701+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 189310 (16 ios * 4000 + 125310 bytes) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes 0x558c57454a00 shared_blobs 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) update inline shard 288 bytes from 15 extents 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 707 (413 bytes onode + 2 bytes spanning blobs + 292 bytes inline extents) 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[fe000~1e000] released 0x[] 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 freelist allocate 0xfe000~1e000 2019-12-07T19:33:58.701+0000 7f8f0217d700 20 freelist _xor first_key 0x80000 last_key 0x100000 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:33:58.701+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58c00 2019-12-07T19:33:58.702+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.005864369s (0.003361652s flush + 0.002502717s kv commit) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.702+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0000> 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.702+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:33:58.702+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 0x558c574a1680 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e427f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a9830 space 0x558c54e427f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e428d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a9950 space 0x558c54e428d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e429b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a8090 space 0x558c54e429b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0a28 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8480 space 0x558c574e0a28 0x0~1000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0a98 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d83f0 space 0x558c574e0a98 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0cc8 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a98c0 space 0x558c574e0cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0ef8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d82d0 space 0x558c574e0ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0f68 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8360 space 0x558c574e0f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e10b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8240 space 0x558c574e10b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1198 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d81b0 space 0x558c574e1198 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1278 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8120 space 0x558c574e1278 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1358 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8090 space 0x558c574e1358 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1438 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8000 space 0x558c574e1438 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1518 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a9b00 space 0x558c574e1518 0x0~2000 writing nocache) 2019-12-07T19:33:58.702+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=10, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c55251000 con 0x558c54c5b400 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 aio_wait 2019-12-07T19:33:58.702+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:33:58.702+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.702+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.702+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.702+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 aio_wait 2019-12-07T19:33:58.702+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:33:58.702+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:33:58.702+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.702+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.702+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.702+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 41 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c574a2000 con 0x558c54c64000 2019-12-07T19:33:58.702+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.702+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000053 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.702+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1b00 seq 14 2019-12-07T19:33:58.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes shared_blobs 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:33:58.702+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:33:58.702+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 42 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c574a5000 con 0x558c54c64000 2019-12-07T19:33:58.703+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.703+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.703+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.703+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.703+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.703+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.703+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.703+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.704+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.704+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 19 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 128948+0+0 (crc 0 0 0) 0x558c574a5800 con 0x558c54c5bc00 2019-12-07T19:33:58.704+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.704+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.704+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.704+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.704+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.704+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.704+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.704+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59b00 seq 15 2019-12-07T19:33:58.704+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 3 keys 2019-12-07T19:33:58.704+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 3 keys = 0 2019-12-07T19:33:58.704+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.704+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.704+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.704+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.704+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes 0x558c574a0000 shared_blobs 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0108) update 4#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ee170468:::smithi13140796-25:head# is 738 (485 bytes onode + 2 bytes spanning blobs + 251 bytes inline extents) 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.704+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:33:58.704+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:33:58.704+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:33:58.704+0000 7f8f05183700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.704+0000 7f8f05183700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.704+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000254 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'6], crt=21'6) [21'7 (0'0) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:91 2019-12-07T19:33:58.699920+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] add_log_entry 21'7 (0'0) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:91 2019-12-07T19:33:58.699920+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.704+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d58000 seq 12 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ad15c4c:::smithi13140796-31:head# key 0x8280000000000000016ad15c'L!smithi13140796-31!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.704+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1092 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# = 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1f000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58000 16 blobs 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1f000/1000,1f000,0 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x11c000~1f000/1000,1f000,0 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x11c000~1f000] 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ea0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538b60) get_ref 0x0~2000 Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ea0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54e43eb8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538c40 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538bd0) get_ref 0x0~2000 Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538c40 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57538c58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538d90 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538d20) get_ref 0x0~2000 Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538d90 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57538da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538e70 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538e00) get_ref 0x0~2000 Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538e70 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57538e88 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538f50 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538ee0) get_ref 0x0~2000 Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538f50 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57538f68 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57539030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539030 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538fc0) get_ref 0x0~2000 Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539030 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57538fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539048 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575390a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57539110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539110 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575390a0) get_ref 0x0~2000 Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539110 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575390a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539128 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57539180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575391f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575391f0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539180) get_ref 0x0~2000 Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575391f0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539208 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57539260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575392d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575392d0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539260) get_ref 0x0~2000 Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575392d0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575392e8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57539340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575393b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575393b0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539340) get_ref 0x0~2000 Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575393b0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575393c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57539420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57539490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539490 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539420) get_ref 0x0~2000 Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539490 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575394a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57539500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57539570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539570 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539500) get_ref 0x0~2000 Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539570 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539588 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575395e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57539650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539650 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575395e0) get_ref 0x0~2000 Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539650 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575395e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539668 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575396c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57539730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539730 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575396c0) get_ref 0x0~2000 Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539730 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575396c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539748 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575397a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57539810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539810 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575397a0) get_ref 0x0~2000 Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539810 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c575397a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57539880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575398f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575398f0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539880) get_ref 0x0~1000 Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575398f0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.blob(0x558c57539880) get_ref init 0x1000, 1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~1000: 0x0~1000 Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 sbid 0x0)) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57539908 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1f000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 = 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 196598 (17 ios * 4000 + 128598 bytes) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes 0x558c54ce2280 shared_blobs 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update inline shard 304 bytes from 16 extents 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 723 (413 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[11c000~1f000] released 0x[] 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 freelist allocate 0x11c000~1f000 2019-12-07T19:33:58.704+0000 7f8f0197c700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:33:58.704+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58000 2019-12-07T19:33:58.704+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1416 < min_flush_size 524288 2019-12-07T19:33:58.704+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454000 had 1 2019-12-07T19:33:58.704+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.704+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1439 < min_flush_size 524288 2019-12-07T19:33:58.704+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1439 < min_flush_size 524288 2019-12-07T19:33:58.704+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.704+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x27192~59f to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.704+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x27192 0x27192~59f to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.704+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.704+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x7192 2019-12-07T19:33:58.704+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x192 2019-12-07T19:33:58.704+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.704+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x731 and padding block with 0x8cf 2019-12-07T19:33:58.705+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x27731 2019-12-07T19:33:58.705+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.705+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.705+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.705+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.005280976s (0.002387429s flush + 0.002893547s kv commit) 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.707+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:33:58.707+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 0x558c57454000 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c328 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8900 space 0x558c54d3c328 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 2 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2255 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c4e8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8990 space 0x558c54d3c4e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0c58 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8870 space 0x558c574e0c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2018 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8a20 space 0x558c574e2018 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2903 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e20f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8ab0 space 0x558c574e20f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e21d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8b40 space 0x558c574e21d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e22b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8bd0 space 0x558c574e22b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2398 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8c60 space 0x558c574e2398 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0000 had 1 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2478 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8cf0 space 0x558c574e2478 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2926 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2558 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8d80 space 0x558c574e2558 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2926 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2638 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8e10 space 0x558c574e2638 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2718 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8ea0 space 0x558c574e2718 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e27f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8f30 space 0x558c574e27f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e28d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8fc0 space 0x558c574e28d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e29b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d9050 space 0x558c574e29b8 0x0~1000 writing nocache) 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x27731~b6e to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.707+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:33:58.707+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x27731 0x27731~b6e to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.707+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.707+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x7731 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x731 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.707+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x29f and padding block with 0xd61 2019-12-07T19:33:58.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=12, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c54cea800 con 0x558c54c5b400 2019-12-07T19:33:58.708+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 22 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558c5748c800 con 0x558c54c5b400 2019-12-07T19:33:58.708+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.708+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.708+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78840 finish 2019-12-07T19:33:58.708+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.708+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78580 finish 2019-12-07T19:33:58.708+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2829f 2019-12-07T19:33:58.708+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.708+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.708+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000501 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.3s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.708+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'4], crt=21'2) [21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.708+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'4 active mbc={}] add_log_entry 21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.708+0000 7f8f0317f700 20 osd.7 pg_epoch: 21 pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.709+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740a000 seq 11 2019-12-07T19:33:58.709+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 3 keys 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 3 keys = 0 2019-12-07T19:33:58.709+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.709+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes 0x558c574a1680 shared_blobs 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) update 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:cd22b62d:::smithi13140796-27:head# is 757 (485 bytes onode + 2 bytes spanning blobs + 270 bytes inline extents) 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.709+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:33:58.709+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.002867602s (0.000000086s flush + 0.002867516s kv commit) 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1797 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1680 had 1 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1820 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1820 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2829f~71c to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2829f 0x2829f~71c to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x829f 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x29f 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9bb and padding block with 0x645 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1b00,0x558c54d59b00> 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 kv_submitted 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 0x558c574a0000 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:33:58.710+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:33:58.710+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.710+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x289bb 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.710+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.710+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.710+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 aio_wait 2019-12-07T19:33:58.710+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:33:58.710+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:33:58.710+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 aio_wait 2019-12-07T19:33:58.710+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:33:58.710+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:33:58.711+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=45, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c54d76000 con 0x558c54c64000 2019-12-07T19:33:58.711+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=46, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c53aa8c00 con 0x558c54c64000 2019-12-07T19:33:58.711+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002314846s (0.000000104s flush + 0.002314742s kv commit) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000> 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 0x558c574a1680 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2262 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.713+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=14, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c55250800 con 0x558c54c5b400 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454a00 had 1 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3697 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3720 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3720 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x289bb~e88 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x289bb 0x289bb~e88 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x89bb 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9bb 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x843 and padding block with 0x7bd 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29843 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.713+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000102859s (0.000003995s flush + 0.000098864s kv commit) 2019-12-07T19:33:58.713+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00,0x558c54d58000> 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 0x558c57454a00 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d208 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9440 space 0x558c54d3d208 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e32e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9680 space 0x558c574e32e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3438 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d95f0 space 0x558c574e3438 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3588 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9560 space 0x558c574e3588 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3668 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d94d0 space 0x558c574e3668 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e37b8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9710 space 0x558c574e37b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3898 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d97a0 space 0x558c574e3898 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3978 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9830 space 0x558c574e3978 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3a58 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d98c0 space 0x558c574e3a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3b38 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9950 space 0x558c574e3b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=8, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c54ceb600 con 0x558c54c19800 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3c18 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d99e0 space 0x558c574e3c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3cf8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9a70 space 0x558c574e3cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3dd8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9b00 space 0x558c574e3dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3eb8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9b90 space 0x558c574e3eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3f98 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9c20 space 0x558c574e3f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 0x558c54ce2280 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43eb8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a90e0 space 0x558c54e43eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538c58 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a9050 space 0x558c57538c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538da8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c53a69ef0 space 0x558c57538da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538e88 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c57538e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538f68 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b950 space 0x558c57538f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539048 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b680 space 0x558c57539048 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539128 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9aea0 space 0x558c57539128 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539208 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a120 space 0x558c57539208 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575392e8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a090 space 0x558c575392e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575393c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c575393c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575394a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b710 space 0x558c575394a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=23, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558c5501d600 con 0x558c54c5bc00 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539588 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b5f0 space 0x558c57539588 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539668 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b200 space 0x558c57539668 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539748 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9afc0 space 0x558c57539748 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539828 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b050 space 0x558c57539828 0x0~2000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57539908 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9af30 space 0x558c57539908 0x0~1000 writing nocache) 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:33:58.713+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.713+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.713+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 43 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 141236+0+0 (crc 0 0 0) 0x558c5748e800 con 0x558c54c64000 2019-12-07T19:33:58.713+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.713+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] add_log_entry 21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.713+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0900 seq 16 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:e7f24557:::smithi13140796-34:head# key 0x848000000000000001e7f245'W!smithi13140796-34!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.713+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1093 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# = 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# 2 keys 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# 2 keys = 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# 0x0~22000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:e7f24557:::smithi13140796-34:head# 0x0~22000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~22000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0900 17 blobs 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x22000/1000,22000,0 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x13b000~22000/1000,22000,0 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x13b000~22000] 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43a40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43a40) get_ref 0x0~2000 Blob(0x558c54e43a40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54e43a40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf99d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43ab0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf99d0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43ab0) get_ref 0x0~2000 Blob(0x558c54e43ab0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf99d0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54e43ab0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf99d0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54cf99e8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e439d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e439d0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43b20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e439d0) get_ref 0x0~2000 Blob(0x558c54e439d0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43b20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e439d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54e439d0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43b38 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43b90 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43c00 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43b90) get_ref 0x0~2000 Blob(0x558c54e43b90 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43c00 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54e43b90 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43c00 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43c18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43c70 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e7e0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43c70) get_ref 0x0~2000 Blob(0x558c54e43c70 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e7e0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54e43c70 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e7e0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e7f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e850 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e850) get_ref 0x0~2000 Blob(0x558c5757e850 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5757e850 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e930 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e930) get_ref 0x0~2000 Blob(0x558c5757e930 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5757e930 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ea10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ea80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ea10 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ea10) get_ref 0x0~2000 Blob(0x558c5757ea10 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ea10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5757ea10 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757eaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757eb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757eaf0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eaf0) get_ref 0x0~2000 Blob(0x558c5757eaf0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eaf0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5757eaf0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c0e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0d20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3c0e0) get_ref 0x0~2000 Blob(0x558c54d3c0e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0d20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3c0e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c54d3c0e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0d20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c574e0d38 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e30a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ebd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e30a0 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c574e30a0) get_ref 0x0~2000 Blob(0x558c574e30a0 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c574e30a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c574e30a0 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ec40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ecb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ec40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ecb0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ec40) get_ref 0x0~2000 Blob(0x558c5757ec40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ecb0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ec40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c5757ec40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ecb0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ecc8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ed20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ed90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ed20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ed20) get_ref 0x0~2000 Blob(0x558c5757ed20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ed20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c5757ed20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ee00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ee00 blob([0x155000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ee00) get_ref 0x0~2000 Blob(0x558c5757ee00 blob([0x155000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ee00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c5757ee00 blob([0x155000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ee88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757eee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ef50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757eee0 blob([0x157000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ef50 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eee0) get_ref 0x0~2000 Blob(0x558c5757eee0 blob([0x157000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ef50 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c5757eee0 blob([0x157000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ef50 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ef68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757efc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757efc0 blob([0x159000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f030 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757efc0) get_ref 0x0~2000 Blob(0x558c5757efc0 blob([0x159000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f030 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757efc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c5757efc0 blob([0x159000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f030 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757f048 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f0a0 blob([0x15b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f110 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757f0a0) get_ref 0x0~2000 Blob(0x558c5757f0a0 blob([0x15b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f110 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.blob(0x558c5757f0a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c5757f0a0 blob([0x15b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f110 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757f128 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x22000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44608) dirty_range mark inline shard dirty 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# 0x0~22000 = 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 212886 (18 ios * 4000 + 140886 bytes) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes 0x558c54e44500 shared_blobs 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44608) update 4#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44608) update inline shard 326 bytes from 17 extents 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:e7f24557:::smithi13140796-34:head# is 745 (413 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[13b000~22000] released 0x[] 2019-12-07T19:33:58.713+0000 7f8f0117b700 10 freelist allocate 0x13b000~22000 2019-12-07T19:33:58.713+0000 7f8f0117b700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.714+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:33:58.714+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0900 2019-12-07T19:33:58.714+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.714+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:33:58.714+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.714+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 145748 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.714+0000 7f8f24457700 0 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 msgr2=0x558c54c39180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.714+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 msgr2=0x558c54c39180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 91 2019-12-07T19:33:58.714+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 msgr2=0x558c54c39180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.714+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 crc :-1 s=READY pgs=11 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.714+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 crc :-1 s=READY pgs=11 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.715+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 aio_wait 2019-12-07T19:33:58.715+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:33:58.715+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1457 < min_flush_size 524288 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44500 had 1 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1480 < min_flush_size 524288 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1480 < min_flush_size 524288 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29843~5c8 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29843 0x29843~5c8 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x9843 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x843 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe0b and padding block with 0x1f5 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29e0b 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.715+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000360424s (0.000176738s flush + 0.000183686s kv commit) 2019-12-07T19:33:58.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0900> 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.716+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:33:58.716+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 0x558c54e44500 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf99e8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b560 space 0x558c54cf99e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43b38 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c54e43b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43c18 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b950 space 0x558c54e43c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9aa20 space 0x558c54e43f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0d38 in 0x558c53a59880) _finish_write discard buffer(0x558c551a9050 space 0x558c574e0d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e7f8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b680 space 0x558c5757e7f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9aea0 space 0x558c5757e8d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9a120 space 0x558c5757e9b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a59880) _finish_write discard buffer(0x558c53a69ef0 space 0x558c5757ea98 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a59880) _finish_write discard buffer(0x558c551a90e0 space 0x558c5757eb78 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a59880) _finish_write discard buffer(0x558c551a9950 space 0x558c5757ebe8 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ecc8 in 0x558c53a59880) _finish_write discard buffer(0x558c551a8090 space 0x558c5757ecc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9a090 space 0x558c5757eda8 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ee88 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c5757ee88 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ef68 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9b710 space 0x558c5757ef68 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f048 in 0x558c53a59880) _finish_write discard buffer(0x558c573d9830 space 0x558c5757f048 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f128 in 0x558c53a59880) _finish_write discard buffer(0x558c573d98c0 space 0x558c5757f128 0x0~2000 writing nocache) 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.716+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:33:58.716+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.716+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.716+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.716+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=49, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558c55250400 con 0x558c54c64000 2019-12-07T19:33:58.716+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.716+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 23 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558c5748e000 con 0x558c54c5b400 2019-12-07T19:33:58.716+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79340 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.716+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e79340 prio 127 cost 0 latency 0.000114 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'5], crt=21'3) [21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'3 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] rollforward: entry=21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.716+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740a300 seq 12 2019-12-07T19:33:58.716+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 3 keys 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 3 keys = 0 2019-12-07T19:33:58.716+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.716+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes 0x558c57454000 shared_blobs 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57454108) update 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:head# is 776 (485 bytes onode + 2 bytes spanning blobs + 289 bytes inline extents) 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.716+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c54e79340 finish 2019-12-07T19:33:58.716+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1816 < min_flush_size 524288 2019-12-07T19:33:58.716+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454000 had 1 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.716+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1839 < min_flush_size 524288 2019-12-07T19:33:58.716+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1839 < min_flush_size 524288 2019-12-07T19:33:58.716+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.716+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29e0b~72f to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.716+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29e0b 0x29e0b~72f to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x9e0b 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe0b 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.716+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x53a and padding block with 0xac6 2019-12-07T19:33:58.718+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2a53a 2019-12-07T19:33:58.718+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.718+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.718+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.718+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.718+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001521801s (0.000000095s flush + 0.001521706s kv commit) 2019-12-07T19:33:58.718+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a300> 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.718+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:33:58.718+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 0x558c57454000 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.718+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:33:58.718+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.718+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.718+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c9f400 0x558c54d04c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.718+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=16, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c55251c00 con 0x558c54c5b400 2019-12-07T19:33:58.718+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54c9f400 0x558c54d04c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.719+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 16 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54896000 con 0x558c54c19800 2019-12-07T19:33:58.719+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e79ce0 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.719+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.719+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 17 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c5748d000 con 0x558c54c19800 2019-12-07T19:33:58.719+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.719+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e79ce0 prio 127 cost 0 latency 0.000107 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:88 2019-12-07T19:33:58.696853+0000 0 ObjectCleanRegions clean_offsets: [491520~18446744073709060095], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.719+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.719+0000 7f8f23c56700 10 osd.7 21 ms_handle_authentication new session 0x558c54c5dd40 con 0x558c54c9f400 entity osd.3 addr 2019-12-07T19:33:58.719+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d58600 seq 6 2019-12-07T19:33:58.719+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58600 onodes shared_blobs 2019-12-07T19:33:58.719+0000 7f8f23c56700 10 osd.7 21 ms_handle_authentication session 0x558c54c5dd40 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 prepare 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58600 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 io_done 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e79ce0 finish 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.719+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.719+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.719+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000202 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.719+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.719+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2a53a~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.719+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2a53a 0x2a53a~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xa53a 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x53a 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.719+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9ab and padding block with 0x655 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:89 2019-12-07T19:33:58.696891+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:89 2019-12-07T19:33:58.696891+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.719+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.719+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54ea0f00 seq 7 2019-12-07T19:33:58.719+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 3 keys 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 3 keys = 0 2019-12-07T19:33:58.719+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.719+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.719+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c9f400 0x558c54d04c00 crc :-1 s=SESSION_ACCEPTING pgs=21 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558c54c5bc00 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes 0x558c57454a00 shared_blobs 2019-12-07T19:33:58.719+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c9f400 0x558c54d04c00 crc :-1 s=SESSION_ACCEPTING pgs=21 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore.extentmap(0x558c57454b08) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 779 (485 bytes onode + 2 bytes spanning blobs + 292 bytes inline extents) 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:33:58.719+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:33:58.719+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:33:58.719+0000 7f8f1199c700 2 osd.7 21 ms_handle_reset con 0x558c54c9f400 session 0x558c54c5dd40 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.719+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.719+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.719+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 unknown :-1 s=READY pgs=21 cs=1 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=64577724c341cd38 server_cookie=ea40093cfb40a50d in_seq=19 out_seq=16 2019-12-07T19:33:58.719+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 20 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c551dd800 con 0x558c54c5bc00 2019-12-07T19:33:58.719+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c000 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.719+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.719+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 21 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558c54e41000 con 0x558c54c5bc00 2019-12-07T19:33:58.719+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c160 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.719+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c000 prio 127 cost 0 latency 0.000105 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'7], crt=21'6) [] 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (0'0) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:91 2019-12-07T19:33:58.699920+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.719+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.719+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d58300 seq 13 2019-12-07T19:33:58.719+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes shared_blobs 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c000 finish 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c160 prio 127 cost 0 latency 0.000204 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:92 2019-12-07T19:33:58.699945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'7) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:92 2019-12-07T19:33:58.699945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.719+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.719+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.719+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.719+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0000 seq 14 2019-12-07T19:33:58.719+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 3 keys 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 3 keys = 0 2019-12-07T19:33:58.719+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.719+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.719+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.720+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.720+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes 0x558c54ce2280 shared_blobs 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 795 (485 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.720+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:33:58.720+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:33:58.720+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:33:58.720+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.720+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c160 finish 2019-12-07T19:33:58.720+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2a9ab 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001479760s (0.000000100s flush + 0.001479660s kv commit) 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58600> 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 kv_submitted 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58600 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 finishing 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58600 onodes 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 done 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 kv_queued 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58600 [] 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1819 < min_flush_size 524288 2019-12-07T19:33:58.720+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454a00 had 1 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2933 < min_flush_size 524288 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 2 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4768 < min_flush_size 524288 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4791 < min_flush_size 524288 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4791 < min_flush_size 524288 2019-12-07T19:33:58.720+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=11, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c55250000 con 0x558c54c19800 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2a9ab~12b7 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2a9ab 0x2a9ab~12b7 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xa9ab 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9ab 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc62 and padding block with 0x39e 2019-12-07T19:33:58.720+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2bc62 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.720+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000098472s (0.000000102s flush + 0.000098370s kv commit) 2019-12-07T19:33:58.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00,0x558c54d58300,0x558c54ea0000> 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 0x558c57454a00 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 2019-12-07T19:33:58.720+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 kv_submitted 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.720+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 0x558c54ce2280 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:33:58.720+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.720+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=12, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c54ceb800 con 0x558c54c19800 2019-12-07T19:33:58.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.720+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=27, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558c55251000 con 0x558c54c5bc00 2019-12-07T19:33:58.720+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.720+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=28, last_complete=21'8, committed=1, applied=1)) v2 -- 0x558c54c1fa00 con 0x558c54c5bc00 2019-12-07T19:33:58.720+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.721+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 44 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=53, reqid=client.4540.0:106, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 149428+0+0 (crc 0 0 0) 0x558c54d85000 con 0x558c54c64000 2019-12-07T19:33:58.721+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=53, reqid=client.4540.0:106, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.721+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=53, reqid=client.4540.0:106, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.721+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.721+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=53, reqid=client.4540.0:106, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.721+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.721+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=53, reqid=client.4540.0:106, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 osd.7 21 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000079 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=53, reqid=client.4540.0:106, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=53, reqid=client.4540.0:106, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:e6be1300:::smithi13140796-36:head by client.4540.0:106 2019-12-07T19:33:58.718753+0000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:e6be1300:::smithi13140796-36:head by client.4540.0:106 2019-12-07T19:33:58.718753+0000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.721+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.721+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.721+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d58f00 seq 5 2019-12-07T19:33:58.721+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:0e6be130:::smithi13140796-36:head# key 0x8380000000000000010e6be1'0!smithi13140796-36!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.722+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1094 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# = 0 2019-12-07T19:33:58.722+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 2 keys 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 2 keys = 0 2019-12-07T19:33:58.722+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.722+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 0x0~24000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:0e6be130:::smithi13140796-36:head# 0x0~24000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~24000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58f00 18 blobs 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0x24000/1000,24000,0 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x15d000~24000/1000,24000,0 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x15d000~24000] 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c5b0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54d3c770) get_ref 0x0~2000 Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c5b0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54d3c770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c5b0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c54d3c5c8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c700 blob([0x15f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c690 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54d3c700) get_ref 0x0~2000 Blob(0x558c54d3c700 blob([0x15f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c690 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54d3c700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3c700 blob([0x15f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c690 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c54d3c6a8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55265c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265d50 blob([0x161000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265c70 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c55265d50) get_ref 0x0~2000 Blob(0x558c55265d50 blob([0x161000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265c70 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c55265d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c55265d50 blob([0x161000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265c70 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c55265c88 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e567e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e567e0 blob([0x163000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56380 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54e567e0) get_ref 0x0~2000 Blob(0x558c54e567e0 blob([0x163000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56380 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54e567e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54e567e0 blob([0x163000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56380 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c54e56398 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54677d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c547400e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54677d50 blob([0x165000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c547400e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54677d50) get_ref 0x0~2000 Blob(0x558c54677d50 blob([0x165000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c547400e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54677d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54677d50 blob([0x165000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c547400e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c547400f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55278e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552793b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55278e70 blob([0x167000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552793b0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c55278e70) get_ref 0x0~2000 Blob(0x558c55278e70 blob([0x167000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552793b0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c55278e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c55278e70 blob([0x167000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552793b0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c552793c8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57582000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43f10 blob([0x169000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582000 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54e43f10) get_ref 0x0~2000 Blob(0x558c54e43f10 blob([0x169000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582000 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c54e43f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c54e43f10 blob([0x169000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582000 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57582018 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575820e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582070 blob([0x16b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575820e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582070) get_ref 0x0~2000 Blob(0x558c57582070 blob([0x16b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575820e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57582070 blob([0x16b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575820e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c575820f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575821c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582150 blob([0x16d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575821c0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582150) get_ref 0x0~2000 Blob(0x558c57582150 blob([0x16d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575821c0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57582150 blob([0x16d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575821c0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c575821d8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575822a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582230 blob([0x16f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575822a0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582230) get_ref 0x0~2000 Blob(0x558c57582230 blob([0x16f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575822a0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57582230 blob([0x16f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575822a0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c575822b8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57582380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582310 blob([0x171000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582380 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582310) get_ref 0x0~2000 Blob(0x558c57582310 blob([0x171000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582380 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57582310 blob([0x171000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582380 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57582398 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575823f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57582460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575823f0 blob([0x173000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582460 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c575823f0) get_ref 0x0~2000 Blob(0x558c575823f0 blob([0x173000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582460 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c575823f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c575823f0 blob([0x173000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582460 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57582478 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575824d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57582540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575824d0 blob([0x175000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582540 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c575824d0) get_ref 0x0~2000 Blob(0x558c575824d0 blob([0x175000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582540 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c575824d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c575824d0 blob([0x175000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582540 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57582558 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575825b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57582620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575825b0 blob([0x177000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582620 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c575825b0) get_ref 0x0~2000 Blob(0x558c575825b0 blob([0x177000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582620 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c575825b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c575825b0 blob([0x177000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582620 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57582638 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57582700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582690 blob([0x179000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582700 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582690) get_ref 0x0~2000 Blob(0x558c57582690 blob([0x179000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57582700 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57582690 blob([0x179000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582700 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57582718 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575827e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582770 blob([0x17b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575827e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582770) get_ref 0x0~2000 Blob(0x558c57582770 blob([0x17b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575827e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57582770 blob([0x17b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575827e0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c575827f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575828c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582850 blob([0x17d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575828c0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582850) get_ref 0x0~2000 Blob(0x558c57582850 blob([0x17d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575828c0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57582850 blob([0x17d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575828c0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c575828d8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57582930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575829a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57582930 blob([0x17f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575829a0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582930) get_ref 0x0~2000 Blob(0x558c57582930 blob([0x17f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575829a0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.blob(0x558c57582930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57582930 blob([0x17f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575829a0 sbid 0x0)) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c575829b8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57454d88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 0x0~24000 = 0 2019-12-07T19:33:58.722+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.722+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 225886 (19 ios * 4000 + 149886 bytes) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes 0x558c57454c80 shared_blobs 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57454d88) update 3#1:0e6be130:::smithi13140796-36:head# 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57454d88) update inline shard 345 bytes from 18 extents 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0e6be130:::smithi13140796-36:head# is 764 (413 bytes onode + 2 bytes spanning blobs + 349 bytes inline extents) 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[15d000~24000] released 0x[] 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 freelist allocate 0x15d000~24000 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 freelist _xor first_key 0x100000 last_key 0x180000 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58f00 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.722+0000 7f8f0898a700 10 osd.7 21 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:33:58.722+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 aio_wait 2019-12-07T19:33:58.723+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:33:58.723+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2319 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454c80 had 1 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2342 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2342 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2bc62~926 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2bc62 0x2bc62~926 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xbc62 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc62 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x588 and padding block with 0xa78 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2c588 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.723+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000344827s (0.000034423s flush + 0.000310404s kv commit) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.723+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00> 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.723+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:33:58.723+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 0x558c57454c80 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c547400f8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b830 space 0x558c547400f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c5c8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a90e0 space 0x558c54d3c5c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c6a8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9950 space 0x558c54d3c6a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56398 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9bcb0 space 0x558c54e56398 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55265c88 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8090 space 0x558c55265c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552793c8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b5f0 space 0x558c552793c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57582018 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b200 space 0x558c57582018 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575820f8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9afc0 space 0x558c575820f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575821d8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b050 space 0x558c575821d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575822b8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9af30 space 0x558c575822b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57582398 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b560 space 0x558c57582398 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57582478 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c57582478 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57582558 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b950 space 0x558c57582558 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57582638 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9aa20 space 0x558c57582638 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57582718 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b680 space 0x558c57582718 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575827f8 in 0x558c53a59340) _finish_write discard buffer(0x558c573d9b90 space 0x558c575827f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575828d8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9aea0 space 0x558c575828d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575829b8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a120 space 0x558c575829b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.723+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:33:58.723+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.723+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.723+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=53, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c54c1fc00 con 0x558c54c64000 2019-12-07T19:33:58.723+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 45 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d84000 con 0x558c54c64000 2019-12-07T19:33:58.723+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.723+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.723+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.723+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.723+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.723+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.723+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'7], crt=21'5) [] 2019-12-07T19:33:58.723+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.723+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.723+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.724+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1200 seq 17 2019-12-07T19:33:58.724+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:33:58.724+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.724+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.724+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2c588~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.724+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2c588 0x2c588~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xc588 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x588 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.724+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9f9 and padding block with 0x607 2019-12-07T19:33:58.725+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2c9f9 2019-12-07T19:33:58.725+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.725+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.725+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.725+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.725+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001877819s (0.000000117s flush + 0.001877702s kv commit) 2019-12-07T19:33:58.725+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200> 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.725+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:33:58.725+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.725+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:33:58.725+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.725+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.725+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=56, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558c53aa8c00 con 0x558c54c64000 2019-12-07T19:33:58.725+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 46 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54d86000 con 0x558c54c64000 2019-12-07T19:33:58.726+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e79080 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.726+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e79080 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.726+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0300 seq 18 2019-12-07T19:33:58.726+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# 3 keys 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# 3 keys = 0 2019-12-07T19:33:58.726+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.726+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes 0x558c54e44500 shared_blobs 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44608) update 4#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:e7f24557:::smithi13140796-34:head# is 817 (485 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.726+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c54e79080 finish 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1857 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44500 had 1 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1880 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1880 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2c9f9~758 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2c9f9 0x2c9f9~758 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xc9f9 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9f9 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x151 and padding block with 0xeaf 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2d151 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000082301s (0.000000106s flush + 0.000082195s kv commit) 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0300> 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.726+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:33:58.726+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 0x558c54e44500 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.726+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:33:58.726+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.726+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=57, last_complete=21'8, committed=1, applied=1)) v2 -- 0x558c54d76c00 con 0x558c54c64000 2019-12-07T19:33:58.726+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 22 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54693000 con 0x558c54c5bc00 2019-12-07T19:33:58.726+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.726+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'8], crt=21'7) [] 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] rollforward: entry=21'8 (21'7) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:92 2019-12-07T19:33:58.699945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.726+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740a000 seq 15 2019-12-07T19:33:58.726+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes shared_blobs 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:33:58.726+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c2c0 finish 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.726+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2d151~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2d151 0x2d151~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xd151 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x151 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5c2 and padding block with 0xa3e 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2d5c2 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.726+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000075511s (0.000000114s flush + 0.000075397s kv commit) 2019-12-07T19:33:58.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000> 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.726+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:33:58.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:33:58.727+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:33:58.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 2019-12-07T19:33:58.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:33:58.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.727+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:33:58.727+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.727+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=31, last_complete=21'8, committed=1, applied=1)) v2 -- 0x558c54c1f000 con 0x558c54c5bc00 2019-12-07T19:33:58.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.727+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 18 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 153524+0+0 (crc 0 0 0) 0x558c5748c800 con 0x558c54c19800 2019-12-07T19:33:58.727+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.727+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000069 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.ds2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [21'3 (0'0) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:109 2019-12-07T19:33:58.722797+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:109 2019-12-07T19:33:58.722797+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.727+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5740b500 seq 8 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bf81b261:::smithi13140796-37:head# key 0x828000000000000001bf81b2'a!smithi13140796-37!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.727+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1095 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# = 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys = 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x0~25000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:bf81b261:::smithi13140796-37:head# 0x0~25000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~25000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740b500 19 blobs 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate 0x25000/1000,25000,0 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x181000~25000/1000,25000,0 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x181000~25000] 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c546772d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55278a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278a80 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c546772d0) get_ref 0x0~2000 Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278a80 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c546772d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c55278a98 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538700 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57538690) get_ref 0x0~2000 Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538700 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57538690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538718 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575387e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538850 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575387e0) get_ref 0x0~2000 Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538850 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575387e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538868 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575388c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538930 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575388c0) get_ref 0x0~2000 Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538930 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575388c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538948 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575389a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538a10 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575389a0) get_ref 0x0~2000 Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538a10 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575389a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538a28 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575832d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575832d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57538a80) get_ref 0x0~2000 Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575832d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57538a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c575832e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575831f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575831f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583260) get_ref 0x0~2000 Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575831f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583208 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575835e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583650 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575835e0) get_ref 0x0~2000 Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583650 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575835e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583668 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575836c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583730 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575836c0) get_ref 0x0~2000 Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583730 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575836c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583748 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575837a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583810 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575837a0) get_ref 0x0~2000 Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583810 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c575837a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583828 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575838f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575838f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583880) get_ref 0x0~2000 Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575838f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583908 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c540 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583960) get_ref 0x0~2000 Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c540 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c54d3c558 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 23 ==== MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54676f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575839d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575839d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c54676f50) get_ref 0x0~2000 Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575839d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c580 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c54676f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c575839e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583ab0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583a40) get_ref 0x0~2000 Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583ab0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583ac8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583b90 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583b20) get_ref 0x0~2000 Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583b90 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583ba8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c580 prio 127 cost 0 latency 0.000056 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583c70 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583c00) get_ref 0x0~2000 Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583c70 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583c88 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f05984700 25 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583d50 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x1f000 (126976) 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ce0) get_ref 0x0~2000 Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583d50 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583d68 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583e30 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583dc0) get_ref 0x0~2000 Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583e30 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583e48 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583f10 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ea0) get_ref 0x0~1000 Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583f10 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ea0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~1000: 0x0~1000 Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583f28 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x25000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x0~25000 = 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 233174 (20 ios * 4000 + 153174 bytes) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes 0x558c54e44f00 shared_blobs 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) update 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) update inline shard 361 bytes from 19 extents 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head# is 780 (413 bytes onode + 2 bytes spanning blobs + 365 bytes inline extents) 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[181000~25000] released 0x[] 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 freelist allocate 0x181000~25000 2019-12-07T19:33:58.727+0000 7f8f0217d700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:33:58.727+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740b500 2019-12-07T19:33:58.727+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.727+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54e43eb8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.727+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a9710 space 0x558c54e43eb8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.727+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1000 = 4096 2019-12-07T19:33:58.727+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.727+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:33:58.727+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:33:58.727+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.727+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 126976 2019-12-07T19:33:58.727+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key 2019-12-07T19:33:58.727+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key = 0 2019-12-07T19:33:58.727+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=32, attrs_read=0)) v2 -- 0x558c54ce7b80 con 0x558c54c5bc00 2019-12-07T19:33:58.727+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c580 finish 2019-12-07T19:33:58.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.728+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.728+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:33:58.728+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.728+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 aio_wait 2019-12-07T19:33:58.728+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:33:58.728+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1492 < min_flush_size 524288 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44f00 had 1 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1515 < min_flush_size 524288 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1515 < min_flush_size 524288 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2d5c2~5eb to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2d5c2 0x2d5c2~5eb to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xd5c2 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5c2 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbad and padding block with 0x453 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2dbad 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.728+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000115017s (0.000037487s flush + 0.000077530s kv commit) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.728+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:33:58.728+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 0x558c54e44f00 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c558 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8e10 space 0x558c54d3c558 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55278a98 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9dd0 space 0x558c55278a98 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538718 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9e60 space 0x558c57538718 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538868 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9ef0 space 0x558c57538868 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538948 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9c20 space 0x558c57538948 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538a28 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8ab0 space 0x558c57538a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583208 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8bd0 space 0x558c57583208 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575832e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8b40 space 0x558c575832e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583668 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8c60 space 0x558c57583668 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583748 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8cf0 space 0x558c57583748 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583828 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c57583828 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583908 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8d80 space 0x558c57583908 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=14, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c54ceb600 con 0x558c54c19800 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575839e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8ea0 space 0x558c575839e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583ac8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8f30 space 0x558c57583ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583ba8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8fc0 space 0x558c57583ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583c88 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9050 space 0x558c57583c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583d68 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9440 space 0x558c57583d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583e48 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9680 space 0x558c57583e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583f28 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d95f0 space 0x558c57583f28 0x0~1000 writing nocache) 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.728+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:33:58.728+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.728+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 19 ==== MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=15, to_read={1:bca18876:::smithi13140796-30:head=0,4096,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c54ce6a00 con 0x558c54c19800 2019-12-07T19:33:58.729+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=15, to_read={1:bca18876:::smithi13140796-30:head=0,4096,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.729+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=15, to_read={1:bca18876:::smithi13140796-30:head=0,4096,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=15, to_read={1:bca18876:::smithi13140796-30:head=0,4096,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=15, to_read={1:bca18876:::smithi13140796-30:head=0,4096,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000077 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=15, to_read={1:bca18876:::smithi13140796-30:head=0,4096,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=15, to_read={1:bca18876:::smithi13140796-30:head=0,4096,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.729+0000 7f8f0217d700 25 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1000 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x1e000 (122880) 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.729+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 24 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54d84800 con 0x558c54c5b400 2019-12-07T19:33:58.729+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740c000 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.729+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.729+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c5740c000 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.729+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.729+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'6], crt=21'4) [] 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.729+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.729+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.729+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c54d3d208 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d94d0 space 0x558c54d3d208 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.729+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1000 = 4096 2019-12-07T19:33:58.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54d58c00 seq 13 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] get_hash_info: Getting attr on 1:bca18876:::smithi13140796-30:head 2019-12-07T19:33:58.729+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] get_hash_info: not in cache 1:bca18876:::smithi13140796-30:head 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:33:58.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] get_hash_info: found on disk, size 122880 2019-12-07T19:33:58.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 2019-12-07T19:33:58.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key 2019-12-07T19:33:58.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key = 0 2019-12-07T19:33:58.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:33:58.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:33:58.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:33:58.729+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 21/18 ECSubReadReply(tid=15, attrs_read=0)) v2 -- 0x558c57454f00 con 0x558c54c19800 2019-12-07T19:33:58.729+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c5740c000 finish 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:33:58.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2dbad~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2dbad 0x2dbad~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xdbad 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbad 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1e and padding block with 0xfe2 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2e01e 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000101208s (0.000000101s flush + 0.000101107s kv commit) 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:33:58.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:33:58.729+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.729+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=20, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c5501d600 con 0x558c54c5b400 2019-12-07T19:33:58.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 47 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=59, reqid=client.4540.0:106, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d85800 con 0x558c54c64000 2019-12-07T19:33:58.730+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740c160 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=59, reqid=client.4540.0:106, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.730+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=59, reqid=client.4540.0:106, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.730+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.730+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=59, reqid=client.4540.0:106, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.730+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.730+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=59, reqid=client.4540.0:106, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.730+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740c160 prio 127 cost 0 latency 0.000090 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=59, reqid=client.4540.0:106, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.730+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=59, reqid=client.4540.0:106, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.730+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.730+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:e6be1300:::smithi13140796-36:head by client.4540.0:106 2019-12-07T19:33:58.718753+0000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.730+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.730+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.730+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.730+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.730+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.730+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54ea0600 seq 6 2019-12-07T19:33:58.730+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.730+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.730+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.730+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.730+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.730+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.730+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0600 onodes shared_blobs 2019-12-07T19:33:58.730+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.730+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 prepare 2019-12-07T19:33:58.730+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0600 2019-12-07T19:33:58.730+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 io_done 2019-12-07T19:33:58.730+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740c160 finish 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.730+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2e01e~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2e01e 0x2e01e~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xe01e 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1e 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x48f and padding block with 0xb71 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2e48f 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000121429s (0.000000079s flush + 0.000121350s kv commit) 2019-12-07T19:33:58.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0600> 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 kv_submitted 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0600 2019-12-07T19:33:58.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 finishing 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0600 onodes 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0600 done 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0600 [] 2019-12-07T19:33:58.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.730+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=59, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558c55251c00 con 0x558c54c64000 2019-12-07T19:33:58.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 48 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e3e800 con 0x558c54c64000 2019-12-07T19:33:58.731+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.731+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 49 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=60, reqid=client.4540.0:107, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54e3e000 con 0x558c54c64000 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.731+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740c420 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=60, reqid=client.4540.0:107, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.731+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=60, reqid=client.4540.0:107, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000045 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=60, reqid=client.4540.0:107, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'8], crt=21'7) [] 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=60, reqid=client.4540.0:107, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] rollforward: entry=21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 21 dequeue_op 0x558c5740c420 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=60, reqid=client.4540.0:107, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.731+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=60, reqid=client.4540.0:107, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1800 seq 19 2019-12-07T19:33:58.731+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:e6be1300:::smithi13140796-36:head by client.4540.0:107 2019-12-07T19:33:58.718796+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1800 onodes shared_blobs 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:e6be1300:::smithi13140796-36:head by client.4540.0:107 2019-12-07T19:33:58.718796+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 prepare 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1800 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 io_done 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.731+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c5740c2c0 finish 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.731+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c5740a300 seq 7 2019-12-07T19:33:58.731+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 3 keys 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 3 keys = 0 2019-12-07T19:33:58.731+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.731+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes 0x558c57454c80 shared_blobs 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57454d88) update 3#1:0e6be130:::smithi13140796-36:head# 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0e6be130:::smithi13140796-36:head# is 836 (485 bytes onode + 2 bytes spanning blobs + 349 bytes inline extents) 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.731+0000 7f8f0898a700 10 osd.7 21 dequeue_op 0x558c5740c420 finish 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2990 < min_flush_size 524288 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454c80 had 1 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3013 < min_flush_size 524288 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3013 < min_flush_size 524288 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2e48f~bc5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2e48f 0x2e48f~bc5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xe48f 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x48f 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x54 and padding block with 0xfac 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f054 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000151854s (0.000000101s flush + 0.000151753s kv commit) 2019-12-07T19:33:58.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1800,0x558c5740a300> 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 kv_submitted 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1800 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 finishing 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1800 onodes 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1800 done 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1800 [] 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 0x558c57454c80 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:33:58.731+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.731+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=61, last_complete=21'8, committed=1, applied=1)) v2 -- 0x558c55250400 con 0x558c54c64000 2019-12-07T19:33:58.731+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=60, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c54d77000 con 0x558c54c64000 2019-12-07T19:33:58.731+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.735+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 20 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748e000 con 0x558c54c19800 2019-12-07T19:33:58.735+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.735+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.735+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.735+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 50 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=63, reqid=client.4540.0:107, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e40800 con 0x558c54c64000 2019-12-07T19:33:58.735+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.735+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740c580 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=63, reqid=client.4540.0:107, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.735+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=63, reqid=client.4540.0:107, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.735+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'3], crt=21'1) [] 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:89 2019-12-07T19:33:58.696891+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=63, reqid=client.4540.0:107, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:109 2019-12-07T19:33:58.722797+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=63, reqid=client.4540.0:107, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.735+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.735+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.735+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740c580 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=63, reqid=client.4540.0:107, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.735+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=63, reqid=client.4540.0:107, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c548f6c00 seq 9 2019-12-07T19:33:58.735+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:e6be1300:::smithi13140796-36:head by client.4540.0:107 2019-12-07T19:33:58.718796+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.735+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes shared_blobs 2019-12-07T19:33:58.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.735+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:33:58.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:33:58.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:33:58.735+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c5740a000 seq 8 2019-12-07T19:33:58.735+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.735+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.735+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.735+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes shared_blobs 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:33:58.735+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f054~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f054 0x2f054~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740c580 finish 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xf054 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x54 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4c5 and padding block with 0xb3b 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f4c5 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000106174s (0.000000123s flush + 0.000106051s kv commit) 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6c00> 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 finishing 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:33:58.735+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f4c5~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f4c5 0x2f4c5~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xf4c5 2019-12-07T19:33:58.735+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=18, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c54ceb800 con 0x558c54c19800 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4c5 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x936 and padding block with 0x6ca 2019-12-07T19:33:58.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f936 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.735+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116763s (0.000000095s flush + 0.000116668s kv commit) 2019-12-07T19:33:58.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000> 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:33:58.735+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.735+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 51 ==== MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=64, to_read={1:0e6be130:::smithi13140796-36:head=0,4096,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c54e45400 con 0x558c54c64000 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.735+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740cdc0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=64, to_read={1:0e6be130:::smithi13140796-36:head=0,4096,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.735+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=63, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558c55251c00 con 0x558c54c64000 2019-12-07T19:33:58.735+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=64, to_read={1:0e6be130:::smithi13140796-36:head=0,4096,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=64, to_read={1:0e6be130:::smithi13140796-36:head=0,4096,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=64, to_read={1:0e6be130:::smithi13140796-36:head=0,4096,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.735+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740cdc0 prio 127 cost 0 latency 0.000056 MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=64, to_read={1:0e6be130:::smithi13140796-36:head=0,4096,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.735+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=64, to_read={1:0e6be130:::smithi13140796-36:head=0,4096,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.735+0000 7f8f04982700 25 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.735+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 0x0~1000 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x24000 (147456) 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c5b0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c5b0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.735+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.736+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c5b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.736+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54d3c5c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.736+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c53a69dd0 space 0x558c54d3c5c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.736+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 0x0~1000 = 4096 2019-12-07T19:33:58.736+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.736+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:0e6be130:::smithi13140796-36:head 2019-12-07T19:33:58.736+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:0e6be130:::smithi13140796-36:head 2019-12-07T19:33:58.736+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 2019-12-07T19:33:58.736+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 147456 2019-12-07T19:33:58.736+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key 2019-12-07T19:33:58.736+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key = 0 2019-12-07T19:33:58.736+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.0s0 21/18 ECSubReadReply(tid=64, attrs_read=0)) v2 -- 0x558c57455180 con 0x558c54c64000 2019-12-07T19:33:58.736+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740cdc0 finish 2019-12-07T19:33:58.736+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 21 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54d84000 con 0x558c54c19800 2019-12-07T19:33:58.737+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.737+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:110 2019-12-07T19:33:58.722827+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:110 2019-12-07T19:33:58.722827+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.737+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5740b800 seq 10 2019-12-07T19:33:58.737+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 3 keys 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 3 keys = 0 2019-12-07T19:33:58.737+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.737+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b800 onodes 0x558c54e44f00 shared_blobs 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) update 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head# is 852 (485 bytes onode + 2 bytes spanning blobs + 365 bytes inline extents) 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 prepare 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b800 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 io_done 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.737+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1738 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1903 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44f00 had 1 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1926 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1926 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f936~786 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f936 0x2f936~786 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.653526+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _allocate len 0xbc from 0 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _allocate len 0xbc from 1 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd140000 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd160000~10000/10000,10000,acd140000 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x40000 to include allocated 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 25 (was clean) 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xf936 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x936 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbc and padding block with 0xf44 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x40000 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x300bc 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (25) on file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]), flushing log 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 25 len 0x48 crc 0x6ac8071a) 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x18000~1000 to file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x18000 0x18000~1000 to file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x18000 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x19000 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:33:58.737+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000024 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 25 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000448899s (0.000000095s flush + 0.000448804s kv commit) 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.737+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b800> 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.737+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 kv_submitted 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b800 2019-12-07T19:33:58.737+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 finishing 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b800 onodes 0x558c54e44f00 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b800 done 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.737+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b800 [] 2019-12-07T19:33:58.737+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.737+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.737+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=21, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c54d77e00 con 0x558c54c19800 2019-12-07T19:33:58.737+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.738+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 24 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 169908+0+0 (unknown 0 0 0) 0x558c54897000 con 0x558c54c5bc00 2019-12-07T19:33:58.738+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c6e0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.738+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c6e0 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'8 (0'0,21'8] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'8], crt=21'8) [21'9 (0'0) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:118 2019-12-07T19:33:58.736313+0000 0 ObjectCleanRegions clean_offsets: [671744~18446744073708879871], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] add_log_entry 21'9 (0'0) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:118 2019-12-07T19:33:58.736313+0000 0 ObjectCleanRegions clean_offsets: [671744~18446744073708879871], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.739+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740ac00 seq 16 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:639abb38:::smithi13140796-40:head# key 0x828000000000000001639abb'8!smithi13140796-40!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.739+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1096 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# = 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2 keys 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2 keys = 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~29000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740ac00 21 blobs 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x29000/1000,29000,0 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1a6000~29000/1000,29000,0 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1a6000~29000] 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9570 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c57538cb0) get_ref 0x0~2000 Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9570 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c57538cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54cf9588 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552658f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e070 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c552658f0) get_ref 0x0~2000 Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e070 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c552658f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e070 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e088 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e230 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e1c0) get_ref 0x0~2000 Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e230 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e1c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e230 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e248 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e310 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e2a0) get_ref 0x0~2000 Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e310 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e2a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e310 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e328 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e3f0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e380) get_ref 0x0~2000 Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e3f0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e3f0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e408 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e4d0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e460) get_ref 0x0~2000 Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e4d0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e4d0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e4e8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e5b0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e540) get_ref 0x0~2000 Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e5b0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e5b0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e5c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e690 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e620) get_ref 0x0~2000 Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e690 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e690 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e6a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e770 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e700) get_ref 0x0~2000 Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e770 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e770 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e788 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0d90 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0e00) get_ref 0x0~2000 Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0d90 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0d90 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0cb0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0d20) get_ref 0x0~2000 Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0cb0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0cb0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0cc8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0bd0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0c40) get_ref 0x0~2000 Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0bd0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0bd0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0be8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0af0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0b60) get_ref 0x0~2000 Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0af0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0af0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0b08 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0a10 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0a80) get_ref 0x0~2000 Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0a10 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0a10 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0a28 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f09a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0930 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f09a0) get_ref 0x0~2000 Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0930 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f09a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0930 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0948 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f08c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f07e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f07e0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f08c0) get_ref 0x0~2000 Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f07e0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f08c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f07e0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f07f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3810 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0850) get_ref 0x0~2000 Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3810 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c574f0850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3810 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54ea3828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552798f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279490 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c552798f0) get_ref 0x0~2000 Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279490 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c552798f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279490 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c552794a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742a000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742a000 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c55279500) get_ref 0x0~2000 Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742a000 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c55279500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a000 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742a018 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742a070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742a0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742a0e0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5742a070) get_ref 0x0~2000 Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742a0e0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5742a070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a0e0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742a0f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742a150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742a1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742a1c0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5742a150) get_ref 0x0~1000 Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742a1c0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.blob(0x558c5742a150) get_ref init 0x1000, 1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~1000: 0x0~1000 Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5742a1c0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742a1d8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x29000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2608) dirty_range mark inline shard dirty 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 = 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740ac00 cost 257558 (22 ios * 4000 + 169558 bytes) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740ac00 onodes 0x558c54ce2500 shared_blobs 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2608) update 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2608) update inline shard 399 bytes from 21 extents 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:639abb38:::smithi13140796-40:head# is 818 (413 bytes onode + 2 bytes spanning blobs + 403 bytes inline extents) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740ac00 allocated 0x[1a6000~29000] released 0x[] 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 freelist allocate 0x1a6000~29000 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 prepare 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740ac00 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.739+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c6e0 finish 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.739+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 aio_wait 2019-12-07T19:33:58.739+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740ac00 2019-12-07T19:33:58.739+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 io_done 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1530 < min_flush_size 524288 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2500 had 1 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1553 < min_flush_size 524288 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1553 < min_flush_size 524288 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x300bc~611 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x300bc 0x300bc~611 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xbc 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbc 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6cd and padding block with 0x933 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x306cd 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.739+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107150s (0.000036924s flush + 0.000070226s kv commit) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740ac00> 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.739+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 kv_submitted 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740ac00 2019-12-07T19:33:58.739+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 finishing 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740ac00 onodes 0x558c54ce2500 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9588 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a9830 space 0x558c54cf9588 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54ea3828 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9aa20 space 0x558c54ea3828 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552794a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b680 space 0x558c552794a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742a018 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9aea0 space 0x558c5742a018 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742a0f8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a120 space 0x558c5742a0f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742a1d8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d98c0 space 0x558c5742a1d8 0x0~1000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f07f8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b950 space 0x558c574f07f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0948 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c574f0948 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0a28 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b560 space 0x558c574f0a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0b08 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9af30 space 0x558c574f0b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=35, last_complete=21'9, committed=1, applied=1)) v2 -- 0x558c55251400 con 0x558c54c5bc00 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0be8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d9830 space 0x558c574f0be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0cc8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d9cb0 space 0x558c574f0cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0da8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a8090 space 0x558c574f0da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e088 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b830 space 0x558c5757e088 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e248 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9bcb0 space 0x558c5757e248 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e328 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b5f0 space 0x558c5757e328 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e408 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b200 space 0x558c5757e408 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e4e8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9afc0 space 0x558c5757e4e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e5c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b050 space 0x558c5757e5c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e6a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a90e0 space 0x558c5757e6a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e788 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a9950 space 0x558c5757e788 0x0~2000 writing nocache) 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740ac00 done 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.739+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740ac00 [] 2019-12-07T19:33:58.739+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.740+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 22 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d86000 con 0x558c54c19800 2019-12-07T19:33:58.740+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.740+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.740+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.740+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.740+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.740+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.740+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.740+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.740+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.740+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:110 2019-12-07T19:33:58.722827+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.740+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.740+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.740+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.740+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.740+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.740+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d21500 seq 11 2019-12-07T19:33:58.740+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.740+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.740+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.740+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.740+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.740+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.740+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes shared_blobs 2019-12-07T19:33:58.740+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.740+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:33:58.740+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:33:58.740+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:33:58.740+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e78580 finish 2019-12-07T19:33:58.740+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x306cd~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x306cd 0x306cd~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x6cd 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6cd 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb3e and padding block with 0x4c2 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x30b3e 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.740+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000086023s (0.000000098s flush + 0.000085925s kv commit) 2019-12-07T19:33:58.740+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21500> 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.740+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:33:58.740+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.740+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:33:58.740+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.740+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.740+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=22, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c54ceb800 con 0x558c54c19800 2019-12-07T19:33:58.740+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.740+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 23 ==== MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=23, to_read={1:bf81b261:::smithi13140796-37:head=0,4096,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57454f00 con 0x558c54c19800 2019-12-07T19:33:58.740+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=23, to_read={1:bf81b261:::smithi13140796-37:head=0,4096,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.740+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=23, to_read={1:bf81b261:::smithi13140796-37:head=0,4096,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=23, to_read={1:bf81b261:::smithi13140796-37:head=0,4096,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=23, to_read={1:bf81b261:::smithi13140796-37:head=0,4096,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000059 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=23, to_read={1:bf81b261:::smithi13140796-37:head=0,4096,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=23, to_read={1:bf81b261:::smithi13140796-37:head=0,4096,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.740+0000 7f8f0217d700 25 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.740+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x0~1000 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x25000 (151552) 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c55278a98 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9560 space 0x558c55278a98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x0~1000 = 4096 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:bf81b261:::smithi13140796-37:head 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:bf81b261:::smithi13140796-37:head 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 151552 2019-12-07T19:33:58.740+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key = 0 2019-12-07T19:33:58.740+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 21/18 ECSubReadReply(tid=23, attrs_read=0)) v2 -- 0x558c54ce6a00 con 0x558c54c19800 2019-12-07T19:33:58.740+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:33:58.740+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.214487565s delta_ub 0.036555542s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76600 con 0x558c552d1800 2019-12-07T19:33:58.742+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.035800979s,0.036555542s]) 2019-12-07T19:33:58.742+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.178686586s delta_ub 18446744073.673750637s) v5 -- 0x558c54d77e00 con 0x558c552d1800 2019-12-07T19:33:58.742+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:33:58.742+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.214487565s delta_ub 0.036555542s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c53aa8c00 con 0x558c552d1c00 2019-12-07T19:33:58.742+0000 7f8f24457700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.035416333s,0.036555542s]) 2019-12-07T19:33:58.742+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.179071232s delta_ub 18446744073.674135283s) v5 -- 0x558c54d76c00 con 0x558c552d1c00 2019-12-07T19:33:58.742+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 25 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54895800 con 0x558c54c5bc00 2019-12-07T19:33:58.742+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c840 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.742+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.742+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 26 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558c54e3e800 con 0x558c54c5bc00 2019-12-07T19:33:58.742+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.742+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c840 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.742+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.742+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'9], crt=21'8) [] 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] rollforward: entry=21'9 (0'0) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:118 2019-12-07T19:33:58.736313+0000 0 ObjectCleanRegions clean_offsets: [671744~18446744073708879871], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.742+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740af00 seq 17 2019-12-07T19:33:58.742+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:33:58.742+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c840 finish 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.742+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.742+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000155 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x30b3e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x30b3e 0x30b3e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log log((0'0,21'9], crt=21'9) [21'10 (21'9) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:119 2019-12-07T19:33:58.736345+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xb3e 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb3e 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfaf and padding block with 0x51 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] add_log_entry 21'10 (21'9) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:119 2019-12-07T19:33:58.736345+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.742+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.742+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x30faf 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.742+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000100737s (0.000000090s flush + 0.000100647s kv commit) 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0c00 seq 18 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00> 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:33:58.742+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 3 keys 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 prepare 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [] 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.742+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 3 keys = 0 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.742+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.742+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.742+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.742+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.742+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes 0x558c54ce2500 shared_blobs 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2608) update 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:639abb38:::smithi13140796-40:head# is 890 (485 bytes onode + 2 bytes spanning blobs + 403 bytes inline extents) 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.742+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:33:58.742+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:33:58.742+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] op_applied version 21'10 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1930 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c5740c9a0 finish 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2500 had 1 2019-12-07T19:33:58.742+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=36, last_complete=21'9, committed=1, applied=1)) v2 -- 0x558c54c1f000 con 0x558c54c5bc00 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1953 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1953 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x30faf~7a1 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x30faf 0x30faf~7a1 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xfaf 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfaf 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x750 and padding block with 0x8b0 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31750 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.742+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000100928s (0.000000077s flush + 0.000100851s kv commit) 2019-12-07T19:33:58.742+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 0x558c54ce2500 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:33:58.742+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.742+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.742+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=37, last_complete=21'10, committed=1, applied=1)) v2 -- 0x558c55250600 con 0x558c54c5bc00 2019-12-07T19:33:58.743+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 27 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54e40000 con 0x558c54c5bc00 2019-12-07T19:33:58.745+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740cb00 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.745+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740cb00 prio 127 cost 0 latency 0.000217 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 pg pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log log((0'0,21'10], crt=21'9) [] 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] rollforward: entry=21'10 (21'9) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:119 2019-12-07T19:33:58.736345+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.745+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740b200 seq 19 2019-12-07T19:33:58.745+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b200 onodes shared_blobs 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 prepare 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b200 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 io_done 2019-12-07T19:33:58.745+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740cb00 finish 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31750~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31750 0x31750~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x1750 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x750 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbc1 and padding block with 0x43f 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31bc1 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.745+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000077009s (0.000000085s flush + 0.000076924s kv commit) 2019-12-07T19:33:58.745+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b200> 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.745+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 kv_submitted 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b200 2019-12-07T19:33:58.745+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 finishing 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b200 onodes 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b200 done 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.745+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b200 [] 2019-12-07T19:33:58.745+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.745+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.745+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=38, last_complete=21'10, committed=1, applied=1)) v2 -- 0x558c55251400 con 0x558c54c5bc00 2019-12-07T19:33:58.745+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 28 ==== MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c54ce7b80 con 0x558c54c5bc00 2019-12-07T19:33:58.745+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.745+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.745+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.745+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.745+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000046 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] 2019-12-07T19:33:58.745+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.745+0000 7f8f05984700 25 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.745+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~1000 2019-12-07T19:33:58.745+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x29000 (167936) 2019-12-07T19:33:58.745+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.745+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.745+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.746+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.746+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.746+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.746+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf9588 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.746+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c573d9b90 space 0x558c54cf9588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.746+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~1000 = 4096 2019-12-07T19:33:58.746+0000 7f8f05984700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.746+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] get_hash_info: Getting attr on 1:639abb38:::smithi13140796-40:head 2019-12-07T19:33:58.746+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] get_hash_info: not in cache 1:639abb38:::smithi13140796-40:head 2019-12-07T19:33:58.746+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.746+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] get_hash_info: found on disk, size 167936 2019-12-07T19:33:58.746+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key 2019-12-07T19:33:58.746+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key = 0 2019-12-07T19:33:58.746+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=39, attrs_read=0)) v2 -- 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:33:58.746+0000 7f8f05984700 10 osd.7 21 dequeue_op 0x558c54e78420 finish 2019-12-07T19:33:58.746+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.764+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 169469 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.767+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 52 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=76, reqid=client.4540.0:130, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 186292+0+0 (crc 0 0 0) 0x558c551dd800 con 0x558c54c64000 2019-12-07T19:33:58.767+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740cf20 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=76, reqid=client.4540.0:130, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.767+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=76, reqid=client.4540.0:130, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=76, reqid=client.4540.0:130, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=76, reqid=client.4540.0:130, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 osd.7 21 dequeue_op 0x558c5740cf20 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=76, reqid=client.4540.0:130, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=76, reqid=client.4540.0:130, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:130 2019-12-07T19:33:58.756349+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:130 2019-12-07T19:33:58.756349+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.767+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54ea0300 seq 9 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:0064fe1d:::smithi13140796-44:head# key 0x8380000000000000010064fe1d21736d'ithi13140796-44!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.767+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1097 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# = 0 2019-12-07T19:33:58.767+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:33:58.767+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.767+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x0~2d000 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:0064fe1d:::smithi13140796-44:head# 0x0~2d000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.767+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2d000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.767+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0300 23 blobs 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2d000/1000,2d000,0 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1cf000~2d000/1000,2d000,0 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1cf000~2d000] 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ff10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5757ff10) get_ref 0x0~2000 Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5757ff10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c54d3d748 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575833b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575833b0 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c575833b0) get_ref 0x0~2000 Blob(0x558c575833b0 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c575833b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c575833b0 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57583358 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583490 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c57583490) get_ref 0x0~2000 Blob(0x558c57583490 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c57583490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57583490 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57583588 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742ba40 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742ba40) get_ref 0x0~2000 Blob(0x558c5742ba40 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742ba40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5742ba40 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5742bac8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742bb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742bb20 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bb20) get_ref 0x0~2000 Blob(0x558c5742bb20 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bb20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5742bb20 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5742bba8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742bc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742bc00 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bc00) get_ref 0x0~2000 Blob(0x558c5742bc00 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bc00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5742bc00 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5742bc88 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742bce0 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bce0) get_ref 0x0~2000 Blob(0x558c5742bce0 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5742bce0 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5742bd68 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742bdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742bdc0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bdc0) get_ref 0x0~2000 Blob(0x558c5742bdc0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bdc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5742bdc0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5742be48 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742bea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742bea0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bea0) get_ref 0x0~2000 Blob(0x558c5742bea0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5742bea0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5742bf28 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742bf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742bf80 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bf80) get_ref 0x0~2000 Blob(0x558c5742bf80 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c5742bf80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c5742bf80 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa018 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa070 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa070) get_ref 0x0~2000 Blob(0x558c574fa070 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c574fa070 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa0f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa150 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa150) get_ref 0x0~2000 Blob(0x558c574fa150 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c574fa150 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa1d8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa230 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa230) get_ref 0x0~2000 Blob(0x558c574fa230 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c574fa230 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa2b8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa310 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa310) get_ref 0x0~2000 Blob(0x558c574fa310 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c574fa310 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa398 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa3f0 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa3f0) get_ref 0x0~2000 Blob(0x558c574fa3f0 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa3f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c574fa3f0 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa478 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa4d0 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa4d0) get_ref 0x0~2000 Blob(0x558c574fa4d0 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa4d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c574fa4d0 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa558 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa5b0 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa5b0) get_ref 0x0~2000 Blob(0x558c574fa5b0 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa5b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c574fa5b0 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa638 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa690 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa690) get_ref 0x0~2000 Blob(0x558c574fa690 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c574fa690 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa718 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa770 blob([0x1f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa770) get_ref 0x0~2000 Blob(0x558c574fa770 blob([0x1f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c574fa770 blob([0x1f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa7f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa850 blob([0x1f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa850) get_ref 0x0~2000 Blob(0x558c574fa850 blob([0x1f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c574fa850 blob([0x1f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa8d8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa930 blob([0x1f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa930) get_ref 0x0~2000 Blob(0x558c574fa930 blob([0x1f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574fa930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c574fa930 blob([0x1f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fa9b8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574faa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574faa10 blob([0x1f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574faa10) get_ref 0x0~2000 Blob(0x558c574faa10 blob([0x1f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574faa10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c574faa10 blob([0x1f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574faa98 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574faaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574faaf0 blob([0x1fb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574faaf0) get_ref 0x0~1000 Blob(0x558c574faaf0 blob([0x1fb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.blob(0x558c574faaf0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~1000: 0x0~1000 Blob(0x558c574faaf0 blob([0x1fb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 sbid 0x0)) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574fab78 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2d000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57455a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x0~2d000 = 0 2019-12-07T19:33:58.768+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.768+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 281942 (24 ios * 4000 + 185942 bytes) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes 0x558c57455900 shared_blobs 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57455a08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57455a08) update inline shard 437 bytes from 23 extents 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 856 (413 bytes onode + 2 bytes spanning blobs + 441 bytes inline extents) 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[1cf000~2d000] released 0x[] 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 freelist allocate 0x1cf000~2d000 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0300 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.768+0000 7f8f0898a700 10 osd.7 21 dequeue_op 0x558c5740cf20 finish 2019-12-07T19:33:58.768+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.768+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 aio_wait 2019-12-07T19:33:58.768+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:33:58.768+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1568 < min_flush_size 524288 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455900 had 1 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1591 < min_flush_size 524288 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1591 < min_flush_size 524288 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31bc1~637 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31bc1 0x31bc1~637 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x1bc1 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbc1 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1f8 and padding block with 0xe08 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x321f8 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.768+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000135532s (0.000037542s flush + 0.000097990s kv commit) 2019-12-07T19:33:58.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0300> 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.768+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:33:58.768+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 0x558c57455900 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d748 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b830 space 0x558c54d3d748 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bac8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b200 space 0x558c5742bac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bba8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8090 space 0x558c5742bba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bc88 in 0x558c53a59340) _finish_write discard buffer(0x558c551a90e0 space 0x558c5742bc88 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bd68 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9830 space 0x558c5742bd68 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742be48 in 0x558c53a59340) _finish_write discard buffer(0x558c573d9dd0 space 0x558c5742be48 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bf28 in 0x558c53a59340) _finish_write discard buffer(0x558c573d9e60 space 0x558c5742bf28 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa018 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9afc0 space 0x558c574fa018 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa0f8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b050 space 0x558c574fa0f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa1d8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c574fa1d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa2b8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9aa20 space 0x558c574fa2b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa398 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b680 space 0x558c574fa398 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa478 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9aea0 space 0x558c574fa478 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa558 in 0x558c53a59340) _finish_write discard buffer(0x558c573d9ef0 space 0x558c574fa558 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa638 in 0x558c53a59340) _finish_write discard buffer(0x558c573d9c20 space 0x558c574fa638 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa718 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8ab0 space 0x558c574fa718 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa7f8 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8bd0 space 0x558c574fa7f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa8d8 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8b40 space 0x558c574fa8d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa9b8 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8c60 space 0x558c574fa9b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.769+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574faa98 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8cf0 space 0x558c574faa98 0x0~2000 writing nocache) 2019-12-07T19:33:58.768+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=76, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c54d76200 con 0x558c54c64000 2019-12-07T19:33:58.769+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fab78 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8d80 space 0x558c574fab78 0x0~1000 writing nocache) 2019-12-07T19:33:58.769+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583358 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9bcb0 space 0x558c57583358 0x0~2000 writing nocache) 2019-12-07T19:33:58.769+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583588 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9b5f0 space 0x558c57583588 0x0~2000 writing nocache) 2019-12-07T19:33:58.769+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:33:58.769+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.769+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:33:58.769+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.769+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.769+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.772+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 53 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=77, reqid=client.4540.0:130, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c551dd000 con 0x558c54c64000 2019-12-07T19:33:58.772+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740d080 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=77, reqid=client.4540.0:130, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.772+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=77, reqid=client.4540.0:130, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=77, reqid=client.4540.0:130, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=77, reqid=client.4540.0:130, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.772+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740d080 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=77, reqid=client.4540.0:130, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.772+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=77, reqid=client.4540.0:130, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.772+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.772+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:130 2019-12-07T19:33:58.756349+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.772+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.772+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.772+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.772+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.772+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.772+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c5740b500 seq 10 2019-12-07T19:33:58.772+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.772+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.772+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.772+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.772+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.772+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.772+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes shared_blobs 2019-12-07T19:33:58.772+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.772+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:33:58.772+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:33:58.772+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:33:58.772+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740d080 finish 2019-12-07T19:33:58.772+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x321f8~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x321f8 0x321f8~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x21f8 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1f8 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x669 and padding block with 0x997 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x32669 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.772+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000076468s (0.000000118s flush + 0.000076350s kv commit) 2019-12-07T19:33:58.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.772+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:33:58.772+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.772+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:33:58.772+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.772+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.772+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=77, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558c54d76000 con 0x558c54c64000 2019-12-07T19:33:58.772+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.772+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 24 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 194484+0+0 (crc 0 0 0) 0x558c54d85000 con 0x558c54c19800 2019-12-07T19:33:58.772+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.772+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.772+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.772+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.772+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:136 2019-12-07T19:33:58.768176+0000 0 ObjectCleanRegions clean_offsets: [770048~18446744073708781567], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.772+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:136 2019-12-07T19:33:58.768176+0000 0 ObjectCleanRegions clean_offsets: [770048~18446744073708781567], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.772+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.772+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.772+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.772+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.772+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5740a000 seq 12 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:be42858c:::smithi13140796-46:head# key 0x828000000000000001be42858c21736d'ithi13140796-46!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.772+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1098 2019-12-07T19:33:58.772+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# = 0 2019-12-07T19:33:58.772+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2 keys 2019-12-07T19:33:58.772+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2 keys = 0 2019-12-07T19:33:58.772+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.772+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.772+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.772+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2f000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740a000 24 blobs 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.772+0000 7f8f06185700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2f000/1000,2f000,0 2019-12-07T19:33:58.772+0000 7f8f06185700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x1fc000~2f000/1000,2f000,0 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x1fc000~2f000] 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b1f0 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.blob(0x558c574f1570) get_ref 0x0~2000 Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b1f0 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.blob(0x558c574f1570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c5742b208 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1490 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.blob(0x558c574f1420) get_ref 0x0~2000 Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1490 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.blob(0x558c574f1420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574f14a8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f16c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f16c0 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.blob(0x558c574f1650) get_ref 0x0~2000 Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f16c0 sbid 0x0)) 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore.blob(0x558c574f1650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.772+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574f16d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fb7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fb7a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574f1730) get_ref 0x0~2000 Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fb7a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574f1730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fb7b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fb730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fb6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fb6c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fb730) get_ref 0x0~2000 Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fb6c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fb730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fb6d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fb650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fb5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fb5e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fb650) get_ref 0x0~2000 Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fb5e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fb650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fb5f8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fbb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fbb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbb90 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbb20) get_ref 0x0~2000 Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbb90 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbb20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fbba8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f17a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f17a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c57583420) get_ref 0x0~2000 Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f17a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c57583420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574f17b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fbc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fbc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbc70 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbc00) get_ref 0x0~2000 Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbc70 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbc00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fbc88 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fbce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fbd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbd50 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbce0) get_ref 0x0~2000 Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbd50 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fbd68 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fbdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fbe30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbe30 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbdc0) get_ref 0x0~2000 Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbe30 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbdc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fbe48 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fbea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fbf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbf10 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbea0) get_ref 0x0~2000 Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fbf10 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fbf28 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fbf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc000 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbf80) get_ref 0x0~2000 Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc000 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fbf80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc018 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc0e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc070) get_ref 0x0~2000 Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc0e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc0f8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc1c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc150) get_ref 0x0~2000 Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc1c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc1d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc2a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc230) get_ref 0x0~2000 Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc2a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc2b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc380 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc310) get_ref 0x0~2000 Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc380 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc398 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc460 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc3f0) get_ref 0x0~2000 Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc460 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc3f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc478 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc540 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc4d0) get_ref 0x0~2000 Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc540 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc4d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc558 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc620 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc5b0) get_ref 0x0~2000 Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc620 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc5b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc638 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc700 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc690) get_ref 0x0~2000 Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc700 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc718 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc7e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc770) get_ref 0x0~2000 Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc7e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc7f8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc8c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc850) get_ref 0x0~2000 Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc8c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc8d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fc930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fc9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc9a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc930) get_ref 0x0~1000 Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fc9a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.blob(0x558c574fc930) get_ref init 0x1000, 1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~1000: 0x0~1000 Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574fc9b8 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2f000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.extentmap(0x558c57455c88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.773+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 = 0 2019-12-07T19:33:58.773+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.773+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.773+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.773+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.773+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 294134 (25 ios * 4000 + 194134 bytes) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes 0x558c57455b80 shared_blobs 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.extentmap(0x558c57455c88) update 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore.extentmap(0x558c57455c88) update inline shard 456 bytes from 24 extents 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:be42858c:::smithi13140796-46:head# is 875 (413 bytes onode + 2 bytes spanning blobs + 460 bytes inline extents) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[1fc000~2f000] released 0x[] 2019-12-07T19:33:58.773+0000 7f8f06185700 10 freelist allocate 0x1fc000~2f000 2019-12-07T19:33:58.773+0000 7f8f06185700 20 freelist _xor first_key 0x180000 last_key 0x200000 2019-12-07T19:33:58.773+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:33:58.773+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740a000 2019-12-07T19:33:58.773+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.773+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:33:58.773+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.773+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 aio_wait 2019-12-07T19:33:58.773+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:33:58.773+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1616 < min_flush_size 524288 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455b80 had 1 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1639 < min_flush_size 524288 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1639 < min_flush_size 524288 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x32669~667 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x32669 0x32669~667 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x2669 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x669 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcd0 and padding block with 0x330 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x32cd0 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.773+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112061s (0.000035403s flush + 0.000076658s kv commit) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.773+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000> 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.773+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:33:58.773+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 0x558c57455b80 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b208 in 0x558c53a59dc0) _finish_write discard buffer(0x558c551a9950 space 0x558c5742b208 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f14a8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c551a8090 space 0x558c574f14a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f16d8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c551a90e0 space 0x558c574f16d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f17b8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b560 space 0x558c574f17b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fb5f8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c574fb5f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fb6d8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b950 space 0x558c574fb6d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fb7b8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9a120 space 0x558c574fb7b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fbba8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c551a9830 space 0x558c574fbba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fbc88 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9af30 space 0x558c574fbc88 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=24, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c54d76a00 con 0x558c54c19800 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fbd68 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b830 space 0x558c574fbd68 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fbe48 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b200 space 0x558c574fbe48 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fbf28 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9afc0 space 0x558c574fbf28 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc018 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9440 space 0x558c574fc018 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc0f8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b050 space 0x558c574fc0f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc1d8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c574fc1d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc2b8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9aa20 space 0x558c574fc2b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc398 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9b680 space 0x558c574fc398 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc478 in 0x558c53a59dc0) _finish_write discard buffer(0x558c54c9aea0 space 0x558c574fc478 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc558 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9050 space 0x558c574fc558 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc638 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8fc0 space 0x558c574fc638 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc718 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d8f30 space 0x558c574fc718 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc7f8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d98c0 space 0x558c574fc7f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc8d8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9830 space 0x558c574fc8d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fc9b8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9cb0 space 0x558c574fc9b8 0x0~1000 writing nocache) 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.773+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:33:58.773+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.773+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.774+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 54 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=78, reqid=client.4540.0:131, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c551db800 con 0x558c54c64000 2019-12-07T19:33:58.774+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=78, reqid=client.4540.0:131, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.774+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=78, reqid=client.4540.0:131, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.774+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.774+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=78, reqid=client.4540.0:131, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.774+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.774+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=78, reqid=client.4540.0:131, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000053 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=78, reqid=client.4540.0:131, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=78, reqid=client.4540.0:131, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:131 2019-12-07T19:33:58.756457+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:131 2019-12-07T19:33:58.756457+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.774+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.774+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.774+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c574ff200 seq 11 2019-12-07T19:33:58.774+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 3 keys 2019-12-07T19:33:58.774+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 3 keys = 0 2019-12-07T19:33:58.774+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.774+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.774+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.774+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.774+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff200 onodes 0x558c57455900 shared_blobs 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 928 (485 bytes onode + 2 bytes spanning blobs + 441 bytes inline extents) 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.774+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 prepare 2019-12-07T19:33:58.774+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff200 2019-12-07T19:33:58.774+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 io_done 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.774+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740d1e0 finish 2019-12-07T19:33:58.774+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1968 < min_flush_size 524288 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455900 had 1 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1991 < min_flush_size 524288 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1991 < min_flush_size 524288 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x32cd0~7c7 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x32cd0 0x32cd0~7c7 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x2cd0 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcd0 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x497 and padding block with 0xb69 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x33497 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.774+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000152961s (0.000000122s flush + 0.000152839s kv commit) 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.774+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff200> 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.774+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 kv_submitted 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff200 2019-12-07T19:33:58.774+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 finishing 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff200 onodes 0x558c57455900 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 done 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.774+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff200 [] 2019-12-07T19:33:58.774+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.774+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.774+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=78, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c575b8000 con 0x558c54c64000 2019-12-07T19:33:58.774+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.774+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.778+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 29 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 ==== 198580+0+0 (unknown 0 0 0) 0x558c5748e800 con 0x558c54c5bc00 2019-12-07T19:33:58.778+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000037 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.778+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 pg pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log log((0'0,21'10], crt=21'10) [21'11 (0'0) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:139 2019-12-07T19:33:58.773780+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] add_log_entry 21'11 (0'0) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:139 2019-12-07T19:33:58.773780+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.778+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d59500 seq 20 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:head# key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.778+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1099 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# = 0 2019-12-07T19:33:58.778+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:33:58.778+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.778+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.778+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~30000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d59500 24 blobs 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x30000/1000,30000,0 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x22b000~30000/1000,30000,0 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x22b000~30000] 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742abd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742abd0) get_ref 0x0~2000 Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742abd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d3cb08 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757e150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757e150 blob([0x22d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e150) get_ref 0x0~2000 Blob(0x558c5757e150 blob([0x22d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c5757e150 blob([0x22d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54e56718 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742acb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742acb0 blob([0x22f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742acb0) get_ref 0x0~2000 Blob(0x558c5742acb0 blob([0x22f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742acb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c5742acb0 blob([0x22f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742ad38 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742ad90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742ad90 blob([0x231000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) get_ref 0x0~2000 Blob(0x558c5742ad90 blob([0x231000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([0x231000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742ae18 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742ae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742ae70 blob([0x233000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) get_ref 0x0~2000 Blob(0x558c5742ae70 blob([0x233000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([0x233000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742aef8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742af50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742af50 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742af50) get_ref 0x0~2000 Blob(0x558c5742af50 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742af50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5742af50 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742afd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b030 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742b030) get_ref 0x0~2000 Blob(0x558c5742b030 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742b030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5742b030 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742b0b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b110 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742b110) get_ref 0x0~2000 Blob(0x558c5742b110 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5742b110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5742b110 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742b198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fde30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fde30 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c574fde30) get_ref 0x0~2000 Blob(0x558c574fde30 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c574fde30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c574fde30 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574fdeb8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fdf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fdf10 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c574fdf10) get_ref 0x0~2000 Blob(0x558c574fdf10 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c574fdf10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c574fdf10 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574fdf98 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d8f0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d8f0) get_ref 0x0~2000 Blob(0x558c54d3d8f0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d8f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c54d3d8f0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d3d898 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d810 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d810) get_ref 0x0~2000 Blob(0x558c54d3d810 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c54d3d810 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d3d6d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d5e0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d5e0) get_ref 0x0~2000 Blob(0x558c54d3d5e0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d5e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c54d3d5e0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d3d668 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d7a0 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d7a0) get_ref 0x0~2000 Blob(0x558c54d3d7a0 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d7a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c54d3d7a0 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f1828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ff80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ff80 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5757ff80) get_ref 0x0~2000 Blob(0x558c5757ff80 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c5757ff80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c5757ff80 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574e0638 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc000 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc000) get_ref 0x0~2000 Blob(0x558c575bc000 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c575bc000 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc088 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc0e0 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc0e0) get_ref 0x0~2000 Blob(0x558c575bc0e0 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc0e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c575bc0e0 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc168 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc1c0 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc1c0) get_ref 0x0~2000 Blob(0x558c575bc1c0 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc1c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c575bc1c0 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc248 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc2a0 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc2a0) get_ref 0x0~2000 Blob(0x558c575bc2a0 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc2a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c575bc2a0 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc328 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc380 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc380) get_ref 0x0~2000 Blob(0x558c575bc380 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c575bc380 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc408 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc460 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc460) get_ref 0x0~2000 Blob(0x558c575bc460 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c575bc460 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc4e8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc540 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc540) get_ref 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc5c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc620 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) get_ref 0x0~2000 Blob(0x558c575bc620 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c575bc620 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc6a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bc700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bc700 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) get_ref 0x0~2000 Blob(0x558c575bc700 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c575bc700 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bc788 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x30000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 = 0 2019-12-07T19:33:58.779+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.779+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59500 cost 298230 (25 ios * 4000 + 198230 bytes) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59500 onodes 0x558c575ba000 shared_blobs 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) update inline shard 459 bytes from 24 extents 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 878 (413 bytes onode + 2 bytes spanning blobs + 463 bytes inline extents) 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59500 allocated 0x[22b000~30000] released 0x[] 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 freelist allocate 0x22b000~30000 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 prepare 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d59500 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] op_applied version 21'11 2019-12-07T19:33:58.779+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:33:58.779+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.779+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 aio_wait 2019-12-07T19:33:58.779+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59500 2019-12-07T19:33:58.779+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 io_done 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1590 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba000 had 1 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1613 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1613 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x33497~64d to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x33497 0x33497~64d to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x3497 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x497 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xae4 and padding block with 0x51c 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x33ae4 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.779+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131625s (0.000035997s flush + 0.000095628s kv commit) 2019-12-07T19:33:58.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59500> 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 kv_submitted 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59500 2019-12-07T19:33:58.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59500 finishing 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59500 onodes 0x558c575ba000 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3cb08 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d82d0 space 0x558c54d3cb08 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d668 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8bd0 space 0x558c54d3d668 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d6d8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8ab0 space 0x558c54d3d6d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d898 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d9c20 space 0x558c54d3d898 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56718 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8360 space 0x558c54e56718 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742ad38 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8240 space 0x558c5742ad38 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742ae18 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d81b0 space 0x558c5742ae18 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742aef8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8120 space 0x558c5742aef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742afd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8090 space 0x558c5742afd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b0b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8000 space 0x558c5742b0b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b198 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8900 space 0x558c5742b198 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0638 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8c60 space 0x558c574e0638 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1828 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8b40 space 0x558c574f1828 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fdeb8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9bcb0 space 0x558c574fdeb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fdf98 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b5f0 space 0x558c574fdf98 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc088 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8cf0 space 0x558c575bc088 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc168 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8d80 space 0x558c575bc168 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc248 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d9440 space 0x558c575bc248 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc328 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a90e0 space 0x558c575bc328 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc408 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a9830 space 0x558c575bc408 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc4e8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b560 space 0x558c575bc4e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc5c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c575bc5c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc6a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b950 space 0x558c575bc6a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc788 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a120 space 0x558c575bc788 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59500 done 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.780+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59500 [] 2019-12-07T19:33:58.780+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.780+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=40, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558c55250e00 con 0x558c54c5bc00 2019-12-07T19:33:58.780+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.780+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.780+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 25 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d85800 con 0x558c54c19800 2019-12-07T19:33:58.780+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.780+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000043 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:136 2019-12-07T19:33:58.768176+0000 0 ObjectCleanRegions clean_offsets: [770048~18446744073708781567], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.780+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5740bb00 seq 13 2019-12-07T19:33:58.780+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:33:58.780+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:33:58.780+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x33ae4~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x33ae4 0x33ae4~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x3ae4 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xae4 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf55 and padding block with 0xab 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x33f55 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.780+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.780+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.780+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000082737s (0.000000113s flush + 0.000082624s kv commit) 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740bb00> 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.781+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=25, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558c54ceb800 con 0x558c54c19800 2019-12-07T19:33:58.781+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.781+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 26 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x558c54896000 con 0x558c54c19800 2019-12-07T19:33:58.781+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.781+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:137 2019-12-07T19:33:58.768215+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:137 2019-12-07T19:33:58.768215+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.781+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54ea1b00 seq 14 2019-12-07T19:33:58.781+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 3 keys 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 3 keys = 0 2019-12-07T19:33:58.781+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.781+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes 0x558c57455b80 shared_blobs 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57455c88) update 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:be42858c:::smithi13140796-46:head# is 947 (485 bytes onode + 2 bytes spanning blobs + 460 bytes inline extents) 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.781+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1987 < min_flush_size 524288 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455b80 had 1 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2010 < min_flush_size 524288 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2010 < min_flush_size 524288 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x33f55~7da to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x33f55 0x33f55~7da to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x3f55 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf55 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x72f and padding block with 0x8d1 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3472f 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.781+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122980s (0.000000090s flush + 0.000122890s kv commit) 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.781+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1b00> 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 0x558c57455b80 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:33:58.781+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.781+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.781+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.781+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=26, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c575b8600 con 0x558c54c19800 2019-12-07T19:33:58.781+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.782+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 55 ==== MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=79, reqid=client.4540.0:131, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c551dc000 con 0x558c54c64000 2019-12-07T19:33:58.782+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=79, reqid=client.4540.0:131, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.782+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=79, reqid=client.4540.0:131, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.782+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.782+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=79, reqid=client.4540.0:131, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.782+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.782+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=79, reqid=client.4540.0:131, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.782+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000046 MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=79, reqid=client.4540.0:131, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.782+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.0s3 21/18 ECSubWrite(tid=79, reqid=client.4540.0:131, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.782+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.782+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:131 2019-12-07T19:33:58.756457+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.782+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.782+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.782+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.782+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.782+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:33:58.782+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c574ff500 seq 12 2019-12-07T19:33:58.782+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:33:58.782+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.782+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.782+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.782+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:33:58.782+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.782+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff500 onodes shared_blobs 2019-12-07T19:33:58.782+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.782+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 prepare 2019-12-07T19:33:58.782+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff500 2019-12-07T19:33:58.782+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 io_done 2019-12-07T19:33:58.782+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.782+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3472f~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3472f 0x3472f~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x472f 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x72f 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xba0 and padding block with 0x460 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x34ba0 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.782+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000128647s (0.000000113s flush + 0.000128534s kv commit) 2019-12-07T19:33:58.782+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff500> 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.782+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 kv_submitted 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff500 2019-12-07T19:33:58.782+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 finishing 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff500 onodes 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff500 done 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:33:58.782+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff500 [] 2019-12-07T19:33:58.782+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.782+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.782+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.782+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 21/18 ECSubWriteReply(tid=79, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558c54d76200 con 0x558c54c64000 2019-12-07T19:33:58.782+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 56 ==== MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=80, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57455180 con 0x558c54c64000 2019-12-07T19:33:58.783+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740cf20 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=80, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.783+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=80, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.783+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=80, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.783+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=80, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c5500d000 2019-12-07T19:33:58.783+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740cf20 prio 127 cost 0 latency 0.000044 MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=80, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.783+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.0s3 21/18 ECSubRead(tid=80, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.783+0000 7f8f04982700 25 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.783+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x0~1000 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x2d000 (184320) 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.783+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54d3d748 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:33:58.783+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c573d8ea0 space 0x558c54d3d748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.783+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x0~1000 = 4096 2019-12-07T19:33:58.783+0000 7f8f04982700 20 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.783+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:0064fe1d:::smithi13140796-44:head 2019-12-07T19:33:58.783+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:0064fe1d:::smithi13140796-44:head 2019-12-07T19:33:58.783+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:33:58.783+0000 7f8f04982700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 184320 2019-12-07T19:33:58.783+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key 2019-12-07T19:33:58.783+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key = 0 2019-12-07T19:33:58.783+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.0s0 21/18 ECSubReadReply(tid=80, attrs_read=0)) v2 -- 0x558c54e45400 con 0x558c54c64000 2019-12-07T19:33:58.783+0000 7f8f04982700 10 osd.7 21 dequeue_op 0x558c5740cf20 finish 2019-12-07T19:33:58.783+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 27 ==== MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e40000 con 0x558c54c19800 2019-12-07T19:33:58.783+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.783+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000044 MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds2 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:137 2019-12-07T19:33:58.768215+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.783+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54ea0600 seq 15 2019-12-07T19:33:58.783+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0600 onodes shared_blobs 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 prepare 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0600 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 io_done 2019-12-07T19:33:58.783+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c5740cc60 finish 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.783+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.783+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:33:58.783+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x34ba0~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x34ba0 0x34ba0~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x4ba0 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xba0 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x11 and padding block with 0xfef 2019-12-07T19:33:58.783+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x35011 2019-12-07T19:33:58.784+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.784+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.784+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.784+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.784+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129029s (0.000000064s flush + 0.000128965s kv commit) 2019-12-07T19:33:58.784+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0600> 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.784+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 kv_submitted 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0600 2019-12-07T19:33:58.784+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 finishing 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0600 onodes 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0600 done 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:33:58.784+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0600 [] 2019-12-07T19:33:58.784+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.784+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.784+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.784+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 28 ==== MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=28, to_read={1:be42858c:::smithi13140796-46:head=0,4096,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c54ce6a00 con 0x558c54c19800 2019-12-07T19:33:58.784+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=27, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558c54d76a00 con 0x558c54c19800 2019-12-07T19:33:58.784+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c5740c580 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=28, to_read={1:be42858c:::smithi13140796-46:head=0,4096,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.784+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=28, to_read={1:be42858c:::smithi13140796-46:head=0,4096,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.784+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.784+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=28, to_read={1:be42858c:::smithi13140796-46:head=0,4096,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.784+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.784+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=28, to_read={1:be42858c:::smithi13140796-46:head=0,4096,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.784+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.784+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c5740c580 prio 127 cost 0 latency 0.000039 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=28, to_read={1:be42858c:::smithi13140796-46:head=0,4096,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.784+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=28, to_read={1:be42858c:::smithi13140796-46:head=0,4096,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.784+0000 7f8f06185700 25 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.784+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~1000 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x2f000 (192512) 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.784+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c5742b208 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:33:58.784+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c573d9e60 space 0x558c5742b208 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.784+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~1000 = 4096 2019-12-07T19:33:58.784+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.784+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.784+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.784+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.784+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 192512 2019-12-07T19:33:58.784+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key 2019-12-07T19:33:58.784+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key = 0 2019-12-07T19:33:58.784+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 21/18 ECSubReadReply(tid=28, attrs_read=0)) v2 -- 0x558c57455680 con 0x558c54c19800 2019-12-07T19:33:58.784+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c5740c580 finish 2019-12-07T19:33:58.784+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 30 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c5748d000 con 0x558c54c5bc00 2019-12-07T19:33:58.785+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e79340 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.785+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79340 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log log((0'0,21'11], crt=21'10) [] 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'11 (0'0) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:139 2019-12-07T19:33:58.773780+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.785+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d59b00 seq 21 2019-12-07T19:33:58.785+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79340 finish 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x35011~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x35011 0x35011~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x5011 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x11 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x482 and padding block with 0xb7e 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x35482 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.785+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000126164s (0.000000076s flush + 0.000126088s kv commit) 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00> 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.785+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:33:58.785+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.785+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:33:58.785+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=41, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558c55251400 con 0x558c54c5bc00 2019-12-07T19:33:58.785+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 31 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558c54d86800 con 0x558c54c5bc00 2019-12-07T19:33:58.785+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.785+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'11], crt=21'11) [21'12 (21'11) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:140 2019-12-07T19:33:58.773812+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] add_log_entry 21'12 (21'11) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:140 2019-12-07T19:33:58.773812+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.785+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740a300 seq 22 2019-12-07T19:33:58.785+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 3 keys 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 3 keys = 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes 0x558c575ba000 shared_blobs 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 950 (485 bytes onode + 2 bytes spanning blobs + 463 bytes inline extents) 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:33:58.785+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:33:58.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.785+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] op_applied version 21'12 2019-12-07T19:33:58.786+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:33:58.786+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1990 < min_flush_size 524288 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba000 had 1 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2013 < min_flush_size 524288 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2013 < min_flush_size 524288 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x35482~7dd to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x35482 0x35482~7dd to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x5482 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x482 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc5f and padding block with 0x3a1 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x35c5f 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.786+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000146455s (0.000000097s flush + 0.000146358s kv commit) 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.786+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a300> 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.786+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:33:58.786+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 0x558c575ba000 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.786+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:33:58.786+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.786+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.786+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=42, last_complete=21'12, committed=1, applied=1)) v2 -- 0x558c575b9000 con 0x558c54c5bc00 2019-12-07T19:33:58.786+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 32 ==== MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c575be000 con 0x558c54c5bc00 2019-12-07T19:33:58.788+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.788+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000082 MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s2 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'12], crt=21'11) [] 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] rollforward: entry=21'12 (21'11) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:140 2019-12-07T19:33:58.773812+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.788+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740ac00 seq 23 2019-12-07T19:33:58.788+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740ac00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740ac00 onodes shared_blobs 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740ac00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 prepare 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740ac00 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 io_done 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x35c5f~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x35c5f 0x35c5f~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x5c5f 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc5f 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd0 and padding block with 0xf30 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x360d0 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.788+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000097835s (0.000000116s flush + 0.000097719s kv commit) 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.788+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740ac00> 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.788+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 kv_submitted 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740ac00 2019-12-07T19:33:58.788+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 finishing 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740ac00 onodes 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740ac00 done 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:33:58.788+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740ac00 [] 2019-12-07T19:33:58.788+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.788+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=43, last_complete=21'12, committed=1, applied=1)) v2 -- 0x558c55251000 con 0x558c54c5bc00 2019-12-07T19:33:58.788+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 33 ==== MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:33:58.788+0000 7f8f23c56700 15 osd.7 21 enqueue_op 0x558c5740c840 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.788+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c54d5c000 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c840 prio 127 cost 0 latency 0.000046 MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s2 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.788+0000 7f8f0197c700 25 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.788+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~1000 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x30000 (196608) 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d3cb08 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54c9af30 space 0x558c54d3cb08 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~1000 = 4096 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] get_hash_info: Getting attr on 1:6ee4dfe7:::smithi13140796-47:head 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] get_hash_info: not in cache 1:6ee4dfe7:::smithi13140796-47:head 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] get_hash_info: found on disk, size 196608 2019-12-07T19:33:58.788+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key = 0 2019-12-07T19:33:58.788+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=44, attrs_read=0)) v2 -- 0x558c54ce2000 con 0x558c54c5bc00 2019-12-07T19:33:58.788+0000 7f8f0197c700 10 osd.7 21 dequeue_op 0x558c5740c840 finish 2019-12-07T19:33:58.788+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.797+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 25 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 1364+0+0 (unknown 0 0 0) 0x558c574a5000 con 0x558c54c5b400 2019-12-07T19:33:58.797+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740d080 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.797+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c5740d080 prio 127 cost 0 latency 0.000083 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'6], crt=21'6) [21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] add_log_entry 21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.797+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740a600 seq 14 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:c87708e9:::smithi13140796-17:head#7 key 0x848000000000000001c87708e921736d'ithi13140796-17!='0xfffffffffffffffe0000000000000007'o' 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) r -2 v.len 0 2019-12-07T19:33:58.797+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# -> 4#1:c87708e9:::smithi13140796-17:head#7 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# -> 4#1:c87708e9:::smithi13140796-17:head#7 = 0 2019-12-07T19:33:58.797+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes 0x558c54d2ca00 shared_blobs 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54d2cb08) update 4#1:c87708e9:::smithi13140796-17:head#7 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c87708e9:::smithi13140796-17:head#7 is 662 (485 bytes onode + 2 bytes spanning blobs + 175 bytes inline extents) 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.797+0000 7f8f0317f700 10 osd.7 21 dequeue_op 0x558c5740d080 finish 2019-12-07T19:33:58.797+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.797+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.797+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.797+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1354 < min_flush_size 524288 2019-12-07T19:33:58.797+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2ca00 had 1 2019-12-07T19:33:58.797+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45180 had 1 2019-12-07T19:33:58.797+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.797+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1377 < min_flush_size 524288 2019-12-07T19:33:58.797+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1377 < min_flush_size 524288 2019-12-07T19:33:58.797+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.797+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x360d0~561 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.797+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x360d0 0x360d0~561 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.797+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x60d0 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd0 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x631 and padding block with 0x9cf 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x36631 2019-12-07T19:33:58.798+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.798+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.798+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116007s (0.000000130s flush + 0.000115877s kv commit) 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a600> 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.798+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:33:58.798+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 0x558c54d2ca00 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.798+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:33:58.798+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.798+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=24, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558c55251200 con 0x558c54c5b400 2019-12-07T19:33:58.798+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.798+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 26 ==== MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c574a5800 con 0x558c54c5b400 2019-12-07T19:33:58.800+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.800+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.800+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.800+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.800+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558c5513e000 2019-12-07T19:33:58.800+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000068 MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.800+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.800+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'7], crt=21'6) [] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.800+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.800+0000 7f8f07187700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.800+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.800+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.800+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54d58000 seq 15 2019-12-07T19:33:58.800+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head#7 onode 0x558c54d2ca00 txc 0x558c54d58000 2019-12-07T19:33:58.800+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head#7 0x0 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9d50) put_ref 0x0~2000 Blob(0x558c54cf9d50 blob([0x73000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9ce0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9880) put_ref 0x0~2000 Blob(0x558c54cf9880 blob([0x75000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dce0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54d3dd50) put_ref 0x0~2000 Blob(0x558c54d3dd50 blob([0x77000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9dc0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9e30) put_ref 0x0~2000 Blob(0x558c54cf9e30 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9ea0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9f10) put_ref 0x0~2000 Blob(0x558c54cf9f10 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9f80 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e42000) put_ref 0x0~2000 Blob(0x558c54e42000 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e420e0) put_ref 0x0~2000 Blob(0x558c54e420e0 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42150 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e421c0) put_ref 0x0~2000 Blob(0x558c54e421c0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42230 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e422a0) put_ref 0x0~1000 Blob(0x558c54e422a0 blob([0x83000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.extentmap(0x558c54d2cb08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54cf9d50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf9ce0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x73000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9d50) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9cf8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x73000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54cf9ce0) put 0x558c54cf9ce0 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9cf8 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54cf9880 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3dce0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x75000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9880) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54d3dcf8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x75000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54d3dce0) put 0x558c54d3dce0 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54d3dcf8 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54d3dd50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf9dc0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x77000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54d3dd50) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9dd8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x77000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54cf9dc0) put 0x558c54cf9dc0 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9dd8 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c54cf9e30 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf9ea0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x79000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9e30) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9eb8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x79000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54cf9ea0) put 0x558c54cf9ea0 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9eb8 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c54cf9f10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf9f80 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7b000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54cf9f10) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9f98 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x7b000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54cf9f80) put 0x558c54cf9f80 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9f98 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c54e42000 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7d000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e42000) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x7d000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54e42070) put 0x558c54e42070 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c54e420e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42150 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7f000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e420e0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42168 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x7f000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54e42150) put 0x558c54e42150 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42168 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c54e421c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42230 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x81000~2000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e421c0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42248 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x81000~2000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54e42230) put 0x558c54e42230 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42248 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~1000: 0x0~1000 Blob(0x558c54e422a0 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x83000~1000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.blob(0x558c54e422a0) discard_unallocated 0x0~1000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x83000~1000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.sharedblob(0x558c54e42310) put 0x558c54e42310 removing self from set 0x558c54d0e850 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _clear 2019-12-07T19:33:58.800+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head#7 = 0 2019-12-07T19:33:58.800+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.800+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:33:58.800+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[73000~11000] 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x73000~11000 2019-12-07T19:33:58.800+0000 7f8f07187700 10 freelist release 0x73000~11000 2019-12-07T19:33:58.800+0000 7f8f07187700 20 freelist _xor first_key 0x0 last_key 0x80000 2019-12-07T19:33:58.800+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:33:58.800+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:33:58.800+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:33:58.800+0000 7f8f07187700 10 osd.7 21 dequeue_op 0x558c5740d1e0 finish 2019-12-07T19:33:58.800+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1279 < min_flush_size 524288 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2ca00 had 1 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x36631~516 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x36631 0x36631~516 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x6631 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x631 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb47 and padding block with 0x4b9 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x36b47 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.802+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000102249s (0.000000138s flush + 0.000102111s kv commit) 2019-12-07T19:33:58.802+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58000> 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.802+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:33:58.802+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:33:58.802+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [73000~11000] 2019-12-07T19:33:58.802+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.802+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x73000~11000 2019-12-07T19:33:58.802+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.802+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.802+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=26, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558c548cc600 con 0x558c54c5b400 2019-12-07T19:33:58.802+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.814+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 197376 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.819+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 57 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 1364+0+0 (crc 0 0 0) 0x558c574a2000 con 0x558c54c64000 2019-12-07T19:33:58.819+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740cdc0 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.819+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c5740cdc0 prio 127 cost 0 latency 0.000067 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'8 (0'0,21'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'8], crt=21'8) [21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] add_log_entry 21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.819+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0900 seq 20 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:e7f24557:::smithi13140796-34:head#9 key 0x848000000000000001e7f245'W!smithi13140796-34!='0xfffffffffffffffe0000000000000009'o' 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:33:58.819+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# -> 4#1:e7f24557:::smithi13140796-34:head#9 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head# -> 4#1:e7f24557:::smithi13140796-34:head#9 = 0 2019-12-07T19:33:58.819+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes 0x558c54e44500 shared_blobs 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44608) update 4#1:e7f24557:::smithi13140796-34:head#9 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:e7f24557:::smithi13140796-34:head#9 is 817 (485 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.819+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c5740cdc0 finish 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1509 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44500 had 1 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575baa00 had 1 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x36b47~5fc to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x36b47 0x36b47~5fc to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x6b47 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb47 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x143 and padding block with 0xebd 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x37143 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.819+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000094044s (0.000000129s flush + 0.000093915s kv commit) 2019-12-07T19:33:58.819+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0900> 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.819+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:33:58.819+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 0x558c54e44500 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.819+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:33:58.819+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.819+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.819+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=84, last_complete=21'9, committed=1, applied=1)) v2 -- 0x558c54d77c00 con 0x558c54c64000 2019-12-07T19:33:58.819+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 58 ==== MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e41000 con 0x558c54c64000 2019-12-07T19:33:58.819+0000 7f8f24457700 15 osd.7 21 enqueue_op 0x558c5740cb00 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.819+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x558c54d5f000 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c5740cb00 prio 127 cost 0 latency 0.000111 MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'9], crt=21'8) [] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] rollforward: entry=21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.820+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0f00 seq 21 2019-12-07T19:33:58.820+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head#9 onode 0x558c54e44500 txc 0x558c54ea0f00 2019-12-07T19:33:58.820+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head#9 0x0 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43a40) put_ref 0x0~2000 Blob(0x558c54e43a40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43ab0) put_ref 0x0~2000 Blob(0x558c54e43ab0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf99d0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e439d0) put_ref 0x0~2000 Blob(0x558c54e439d0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43b90) put_ref 0x0~2000 Blob(0x558c54e43b90 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43c00 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43c70) put_ref 0x0~2000 Blob(0x558c54e43c70 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e7e0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e850) put_ref 0x0~2000 Blob(0x558c5757e850 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e930) put_ref 0x0~2000 Blob(0x558c5757e930 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ea10) put_ref 0x0~2000 Blob(0x558c5757ea10 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eaf0) put_ref 0x0~2000 Blob(0x558c5757eaf0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3c0e0) put_ref 0x0~2000 Blob(0x558c54d3c0e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0d20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c574e30a0) put_ref 0x0~2000 Blob(0x558c574e30a0 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ec40) put_ref 0x0~2000 Blob(0x558c5757ec40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ecb0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ed20) put_ref 0x0~2000 Blob(0x558c5757ed20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ee00) put_ref 0x0~2000 Blob(0x558c5757ee00 blob([0x155000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eee0) put_ref 0x0~2000 Blob(0x558c5757eee0 blob([0x157000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ef50 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757efc0) put_ref 0x0~2000 Blob(0x558c5757efc0 blob([0x159000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f030 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757f0a0) put_ref 0x0~2000 Blob(0x558c5757f0a0 blob([0x15b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f110 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44608) dirty_range mark inline shard dirty 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54e43a40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x13b000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43a40) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x13b000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c54e43f80) put 0x558c54e43f80 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54e43ab0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf99d0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x13d000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43ab0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54cf99e8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x13d000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c54cf99d0) put 0x558c54cf99d0 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54cf99e8 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54e439d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43b20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x13f000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e439d0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43b38 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x13f000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c54e43b20) put 0x558c54e43b20 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43b38 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c54e43b90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43c00 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x141000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43b90) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43c18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x141000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c54e43c00) put 0x558c54e43c00 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e43c18 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c54e43c70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757e7e0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x143000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43c70) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e7f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x143000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757e7e0) put 0x558c5757e7e0 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e7f8 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c5757e850 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x145000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e850) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x145000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757e8c0) put 0x558c5757e8c0 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c5757e930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x147000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e930) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x147000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757e9a0) put 0x558c5757e9a0 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c5757ea10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x149000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ea10) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x149000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757ea80) put 0x558c5757ea80 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5757eaf0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x14b000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eaf0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x14b000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757eb60) put 0x558c5757eb60 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c54d3c0e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0d20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x14d000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3c0e0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c574e0d38 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x14d000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c574e0d20) put 0x558c574e0d20 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c574e0d38 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c574e30a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x14f000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c574e30a0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x14f000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757ebd0) put 0x558c5757ebd0 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c5757ec40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757ecb0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x151000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ec40) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ecc8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x151000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757ecb0) put 0x558c5757ecb0 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ecc8 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c5757ed20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x153000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ed20) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x153000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757ed90) put 0x558c5757ed90 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c5757ee00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x155000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757ee00) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ee88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x155000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757ee70) put 0x558c5757ee70 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ee88 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c5757eee0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757ef50 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x157000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757eee0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ef68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x157000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757ef50) put 0x558c5757ef50 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757ef68 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c5757efc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757f030 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x159000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757efc0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757f048 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x159000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757f030) put 0x558c5757f030 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757f048 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c5757f0a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757f110 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x15b000~2000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.blob(0x558c5757f0a0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757f128 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x15b000~2000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.sharedblob(0x558c5757f110) put 0x558c5757f110 removing self from set 0x558c54d39750 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5757f128 in 0x558c53a59880) _clear 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:e7f24557:::smithi13140796-34:head#9 = 0 2019-12-07T19:33:58.820+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[13b000~22000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x13b000~22000 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 freelist release 0x13b000~22000 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:33:58.820+0000 7f8f0117b700 10 osd.7 21 dequeue_op 0x558c5740cb00 finish 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1250 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44500 had 1 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x37143~4f9 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x37143 0x37143~4f9 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x7143 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x143 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x63c and padding block with 0x9c4 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3763c 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:33:58.820+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000070723s (0.000000103s flush + 0.000070620s kv commit) 2019-12-07T19:33:58.820+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.820+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:33:58.820+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:33:58.820+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [13b000~22000] 2019-12-07T19:33:58.820+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.820+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x13b000~22000 2019-12-07T19:33:58.820+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:33:58.820+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:33:58.820+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=85, last_complete=21'9, committed=1, applied=1)) v2 -- 0x558c548cc600 con 0x558c54c64000 2019-12-07T19:33:58.820+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 29 ==== MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=29, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57455680 con 0x558c54c19800 2019-12-07T19:33:58.822+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c5740c000 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=29, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.822+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=29, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.822+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=29, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.822+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=29, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.822+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c5740c000 prio 127 cost 0 latency 0.000096 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=29, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.822+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=29, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.822+0000 7f8f0217d700 25 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.823+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~2f000 size 0x2f000 (192512) 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need {<0x0, 0x1000> : [0x2e000:0~1000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~1000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574f14a8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9dd0 space 0x558c574f14a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5742b208 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9200 space 0x558c5742b208 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574f16d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9170 space 0x558c574f16d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fb7b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8480 space 0x558c574fb7b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fb5f8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d83f0 space 0x558c574fb5f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fb6d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9050 space 0x558c574fb6d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fbba8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8fc0 space 0x558c574fbba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fbc88 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8f30 space 0x558c574fbc88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fbd68 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d98c0 space 0x558c574fbd68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fbe48 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9830 space 0x558c574fbe48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fbf28 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c54c9aa20 space 0x558c574fbf28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc018 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c54c9b680 space 0x558c574fc018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc0f8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c54c9aea0 space 0x558c574fc0f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc1d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c54c9bcb0 space 0x558c574fc1d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc2b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c54c9b5f0 space 0x558c574fc2b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc398 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9cb0 space 0x558c574fc398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc478 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d82d0 space 0x558c574fc478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc558 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8bd0 space 0x558c574fc558 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc638 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8ab0 space 0x558c574fc638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc718 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d9c20 space 0x558c574fc718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc7f8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8360 space 0x558c574fc7f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc8d8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8240 space 0x558c574fc8d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2e000:0~1000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fc9b8 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d81b0 space 0x558c574fc9b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574f17b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c573d8120 space 0x558c574f17b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 = 192512 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=192512 r=192512 len=192512 2019-12-07T19:33:58.823+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.823+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.823+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.823+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 192512 2019-12-07T19:33:58.823+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key 2019-12-07T19:33:58.823+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key = 0 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read: Checking hash of 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.823+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 21/18 ECSubReadReply(tid=29, attrs_read=0)) v2 -- 0x558c57454f00 con 0x558c54c19800 2019-12-07T19:33:58.823+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c5740c000 finish 2019-12-07T19:33:58.823+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.824+0000 7f8f24c58700 0 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 msgr2=0x558c54c38100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 30 ==== MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=30, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57454f00 con 0x558c54c19800 2019-12-07T19:33:58.824+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c552b7b80 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=30, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.824+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=30, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 msgr2=0x558c54c38100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 90 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 msgr2=0x558c54c38100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.824+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=30, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 crc :-1 s=READY pgs=11 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.824+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=30, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 crc :-1 s=READY pgs=11 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.824+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c552b7b80 prio 127 cost 0 latency 0.000046 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=30, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.824+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=30, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.824+0000 7f8f06185700 25 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.824+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~2f000 size 0x2f000 (192512) 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.824+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.824+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 = 192512 2019-12-07T19:33:58.824+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=192512 r=192512 len=192512 2019-12-07T19:33:58.824+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.824+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.824+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.824+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 192512 2019-12-07T19:33:58.824+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key 2019-12-07T19:33:58.824+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key = 0 2019-12-07T19:33:58.824+0000 7f8f06185700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read: Checking hash of 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.824+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 21/18 ECSubReadReply(tid=30, attrs_read=0)) v2 -- 0x558c54ce6a00 con 0x558c54c19800 2019-12-07T19:33:58.824+0000 7f8f06185700 10 osd.7 21 dequeue_op 0x558c552b7b80 finish 2019-12-07T19:33:58.824+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 unknown :-1 s=STANDBY pgs=11 cs=1 l=0 rx=0 tx=0).connect 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54d43c00 0x558c54d05700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.824+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c54d43c00 0x558c54d05700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.824+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 unknown :-1 s=BANNER_CONNECTING pgs=11 cs=1 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.824+0000 7f8f24457700 10 osd.7 21 ms_handle_authentication new session 0x558c54c5dba0 con 0x558c54d43c00 entity osd.2 addr 2019-12-07T19:33:58.824+0000 7f8f24457700 10 osd.7 21 ms_handle_authentication session 0x558c54c5dba0 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.824+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54d43c00 0x558c54d05700 crc :-1 s=SESSION_ACCEPTING pgs=25 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect race detected, this connection loses to existing=0x558c54c19800 2019-12-07T19:33:58.825+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54d43c00 msgr2=0x558c54d05700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 98 2019-12-07T19:33:58.825+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54d43c00 msgr2=0x558c54d05700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.825+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54d43c00 0x558c54d05700 crc :-1 s=SESSION_ACCEPTING pgs=25 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.825+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54d43c00 0x558c54d05700 crc :-1 s=SESSION_ACCEPTING pgs=25 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.825+0000 7f8f1199c700 2 osd.7 21 ms_handle_reset con 0x558c54d43c00 session 0x558c54c5dba0 2019-12-07T19:33:58.825+0000 7f8f1199c700 10 osd.7 21 ms_handle_connect con 0x558c54c19800 2019-12-07T19:33:58.825+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558c54c19800 0x558c54c38100 crc :-1 s=READY pgs=11 cs=1 l=0 rx=0 tx=0).ready entity=osd.2 client_cookie=a3b801957c02aad7 server_cookie=22a5b791d0d72b65 in_seq=30 out_seq=28 2019-12-07T19:33:58.825+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 31 ==== MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=31, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c575baf00 con 0x558c54c19800 2019-12-07T19:33:58.825+0000 7f8f24c58700 15 osd.7 21 enqueue_op 0x558c5740c6e0 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=31, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.825+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=31, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=31, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=31, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558c55141000 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c5740c6e0 prio 127 cost 0 latency 0.000042 MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=31, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.ds2 21/18 ECSubRead(tid=31, to_read={1:be42858c:::smithi13140796-46:head=0,192512,0}, subchunks={1:be42858c:::smithi13140796-46:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.825+0000 7f8f0217d700 25 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.825+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~2f000 size 0x2f000 (192512) 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x0~2f000 = 192512 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=192512 r=192512 len=192512 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 192512 2019-12-07T19:33:58.825+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key = 0 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read: Checking hash of 1:be42858c:::smithi13140796-46:head 2019-12-07T19:33:58.825+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 21/18 ECSubReadReply(tid=31, attrs_read=0)) v2 -- 0x558c57455680 con 0x558c54c19800 2019-12-07T19:33:58.825+0000 7f8f0217d700 10 osd.7 21 dequeue_op 0x558c5740c6e0 finish 2019-12-07T19:33:58.825+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.864+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.914+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.964+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.015+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.045+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:33:59.065+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.115+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.165+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.215+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.265+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.315+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.365+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.371+0000 7f8f0097a700 5 osd.7 21 heartbeat osd_stat(store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,6] op hist []) 2019-12-07T19:33:59.371+0000 7f8f0097a700 20 osd.7 21 check_full_status cur ratio 2.07689e-06, physical ratio 2.07689e-06, new state none 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690476875s) v5 -- 0x558c54d77200 con 0x558c5476d400 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690476875s) v5 -- 0x558c54d77400 con 0x558c551b6800 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.688187843s) v5 -- 0x558c54d77600 con 0x558c54c9e000 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.688187843s) v5 -- 0x558c575b8600 con 0x558c54c9e400 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.674135283s) v5 -- 0x558c575b9a00 con 0x558c551b7800 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.674135283s) v5 -- 0x558c575b9c00 con 0x558c551b7c00 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.687659225s) v5 -- 0x558c575b9e00 con 0x558c552c2000 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.687659225s) v5 -- 0x558c54d76600 con 0x558c552c2400 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.695876832s) v5 -- 0x558c54ceb800 con 0x558c552c2800 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.695876832s) v5 -- 0x558c57518000 con 0x558c552c2c00 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.678770657s) v5 -- 0x558c57518200 con 0x558c54c65800 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.678770657s) v5 -- 0x558c57518400 con 0x558c54c65c00 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690547135s) v5 -- 0x558c57518600 con 0x558c551b7000 2019-12-07T19:33:59.371+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690547135s) v5 -- 0x558c57518800 con 0x558c551b7400 2019-12-07T19:33:59.371+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 2 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.839228836s delta_ub 0.031053974s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c65800 2019-12-07T19:33:59.371+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072687+0000 -> 2019-12-07T19:33:59.372987+0000 last_rx_front 2019-12-07T19:33:57.072733+0000 2019-12-07T19:33:59.371+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030802630s,0.031053974s]) 2019-12-07T19:33:59.371+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 2 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.839254823s delta_ub 0.031079961s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c65c00 2019-12-07T19:33:59.371+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.372987+0000 last_rx_front 2019-12-07T19:33:57.072733+0000 -> 2019-12-07T19:33:59.373022+0000 2019-12-07T19:33:59.371+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:59.372735+0000) and older pending ping(s) 2019-12-07T19:33:59.371+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030793114s,0.031079961s]) 2019-12-07T19:33:59.371+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.822211849s delta_ub 0.014036987s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c548cc600 con 0x558c552c2c00 2019-12-07T19:33:59.371+0000 7f8f24457700 25 osd.7 21 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072765+0000 last_rx_front 2019-12-07T19:33:57.072719+0000 -> 2019-12-07T19:33:59.373042+0000 2019-12-07T19:33:59.371+0000 7f8f24457700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013730541s,0.014036987s]) 2019-12-07T19:33:59.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.822128755s delta_ub 0.013953893s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76a00 con 0x558c552c2800 2019-12-07T19:33:59.371+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6820/36937 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827486051s delta_ub 0.019311189s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c548cc600 con 0x558c551b7400 2019-12-07T19:33:59.371+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072765+0000 -> 2019-12-07T19:33:59.373068+0000 last_rx_front 2019-12-07T19:33:59.373042+0000 2019-12-07T19:33:59.371+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:59.372735+0000) and older pending ping(s) 2019-12-07T19:33:59.371+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013621136s,0.013953893s]) 2019-12-07T19:33:59.371+0000 7f8f24457700 25 osd.7 21 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072706+0000 last_rx_front 2019-12-07T19:33:57.072836+0000 -> 2019-12-07T19:33:59.373092+0000 2019-12-07T19:33:59.371+0000 7f8f24457700 10 osd.7 21 maybe_share_map con 0x558c54c19400 v2:172.21.15.109:6818/36937 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:59.371+0000 7f8f24457700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018954448s,0.019311189s]) 2019-12-07T19:33:59.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6822/36937 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827435739s delta_ub 0.019260877s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76a00 con 0x558c551b7000 2019-12-07T19:33:59.371+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072706+0000 -> 2019-12-07T19:33:59.373111+0000 last_rx_front 2019-12-07T19:33:59.373092+0000 2019-12-07T19:33:59.371+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:59.372735+0000) and older pending ping(s) 2019-12-07T19:33:59.371+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018885212s,0.019260877s]) 2019-12-07T19:33:59.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 2 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.829998329s delta_ub 0.021823467s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76a00 con 0x558c54c9e400 2019-12-07T19:33:59.371+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072827+0000 last_rx_front 2019-12-07T19:33:57.072882+0000 -> 2019-12-07T19:33:59.373149+0000 2019-12-07T19:33:59.371+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021409745s,0.021823467s]) 2019-12-07T19:33:59.371+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.830622705s delta_ub 0.022447843s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c548cc600 con 0x558c552c2000 2019-12-07T19:33:59.371+0000 7f8f24457700 25 osd.7 21 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072787+0000 -> 2019-12-07T19:33:59.373171+0000 last_rx_front 2019-12-07T19:33:57.072868+0000 2019-12-07T19:33:59.371+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.844813918s delta_ub 0.036639056s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76a00 con 0x558c551b7c00 2019-12-07T19:33:59.371+0000 7f8f24457700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022012141s,0.022447843s]) 2019-12-07T19:33:59.371+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072797+0000 last_rx_front 2019-12-07T19:33:57.072809+0000 -> 2019-12-07T19:33:59.373186+0000 2019-12-07T19:33:59.372+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036188165s,0.036639056s]) 2019-12-07T19:33:59.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.830672902s delta_ub 0.022498040s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c552c2400 2019-12-07T19:33:59.372+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373171+0000 last_rx_front 2019-12-07T19:33:57.072868+0000 -> 2019-12-07T19:33:59.373218+0000 2019-12-07T19:33:59.372+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:59.372735+0000) and older pending ping(s) 2019-12-07T19:33:59.372+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022014979s,0.022498040s]) 2019-12-07T19:33:59.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 2 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.830090724s delta_ub 0.021915862s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c9e000 2019-12-07T19:33:59.372+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072827+0000 -> 2019-12-07T19:33:59.373244+0000 last_rx_front 2019-12-07T19:33:59.373149+0000 2019-12-07T19:33:59.372+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:59.372735+0000) and older pending ping(s) 2019-12-07T19:33:59.372+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021407012s,0.021915862s]) 2019-12-07T19:33:59.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827784287s delta_ub 0.019609425s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c5476d400 2019-12-07T19:33:59.372+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072893+0000 -> 2019-12-07T19:33:59.373260+0000 last_rx_front 2019-12-07T19:33:57.072855+0000 2019-12-07T19:33:59.372+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019084635s,0.019609425s]) 2019-12-07T19:33:59.372+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827884222s delta_ub 0.019709360s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76a00 con 0x558c551b6800 2019-12-07T19:33:59.372+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373260+0000 last_rx_front 2019-12-07T19:33:57.072855+0000 -> 2019-12-07T19:33:59.373276+0000 2019-12-07T19:33:59.372+0000 7f8f24c58700 25 osd.7 21 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:59.372735+0000) and older pending ping(s) 2019-12-07T19:33:59.372+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019168127s,0.019709360s]) 2019-12-07T19:33:59.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 6 ==== osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.844991132s delta_ub 0.036816270s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c551b7800 2019-12-07T19:33:59.372+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:57.072797+0000 -> 2019-12-07T19:33:59.373297+0000 last_rx_front 2019-12-07T19:33:59.373186+0000 2019-12-07T19:33:59.372+0000 7f8f23c56700 25 osd.7 21 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:59.372735+0000) and older pending ping(s) 2019-12-07T19:33:59.372+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036254207s,0.036816270s]) 2019-12-07T19:33:59.387+0000 7f8f21451700 10 osd.7 21 tick 2019-12-07T19:33:59.387+0000 7f8f21451700 10 osd.7 21 do_waiters -- start 2019-12-07T19:33:59.387+0000 7f8f21451700 10 osd.7 21 do_waiters -- finish 2019-12-07T19:33:59.387+0000 7f8f21451700 20 osd.7 21 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:33:59.415+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.450+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 5 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.908345690s delta_ub 0.021915862s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c19000 2019-12-07T19:33:59.450+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021310784s,0.021915862s]) 2019-12-07T19:33:59.450+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.887034906s delta_ub 18446744073.688240832s) v5 -- 0x558c57518400 con 0x558c54c19000 2019-12-07T19:33:59.450+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 5 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.908345690s delta_ub 0.021915862s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b9000 con 0x558c54c18800 2019-12-07T19:33:59.450+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021250419s,0.021915862s]) 2019-12-07T19:33:59.450+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.887095271s delta_ub 18446744073.688301197s) v5 -- 0x558c57518400 con 0x558c54c18800 2019-12-07T19:33:59.465+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.482+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:33:59.516+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.566+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.616+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 10 osd.7 21 tick_without_osd_lock 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fdd8000/0x40000000/0x1680000000, data 0x202da8/0x226000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 25 osd.7 21 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373260+0000 last_rx_front 2019-12-07T19:33:59.373276+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 25 osd.7 21 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373244+0000 last_rx_front 2019-12-07T19:33:59.373149+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 25 osd.7 21 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373297+0000 last_rx_front 2019-12-07T19:33:59.373186+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 25 osd.7 21 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373171+0000 last_rx_front 2019-12-07T19:33:59.373218+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 25 osd.7 21 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373068+0000 last_rx_front 2019-12-07T19:33:59.373042+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 25 osd.7 21 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.372987+0000 last_rx_front 2019-12-07T19:33:59.373022+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 25 osd.7 21 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373111+0000 last_rx_front 2019-12-07T19:33:59.373092+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 20 osd.7 21 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 20 osd.7 21 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 20 osd.7 21 scrub_load_below_threshold loadavg per cpu 0.0625 < max 5 = yes 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 20 osd.7 21 sched_scrub load_is_low=1 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 10 osd.7 21 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:33:59.623578+0000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 20 osd.7 21 sched_scrub done 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 10 osd.7 21 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 20 osd.7 21 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:59.622+0000 7f8f1fbbe700 10 osd.7 21 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:59.666+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.716+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.751+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 8 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.209920745s delta_ub 0.022498040s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c548cc600 con 0x558c552d1400 2019-12-07T19:33:59.751+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 8 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.209920745s delta_ub 0.022498040s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c18000 2019-12-07T19:33:59.751+0000 7f8f24457700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021774758s,0.022498040s]) 2019-12-07T19:33:59.751+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.188145987s delta_ub 18446744073.687776858s) v5 -- 0x558c57518800 con 0x558c552d1400 2019-12-07T19:33:59.751+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021749189s,0.022498040s]) 2019-12-07T19:33:59.751+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.188171556s delta_ub 18446744073.687802427s) v5 -- 0x558c57518400 con 0x558c54c18000 2019-12-07T19:33:59.766+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.816+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.308927630s delta_ub 0.014036987s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76a00 con 0x558c54c65400 2019-12-07T19:33:59.858+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013515830s,0.014036987s]) 2019-12-07T19:33:59.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.295411800s delta_ub 18446744073.696035786s) v5 -- 0x558c57518600 con 0x558c54c65400 2019-12-07T19:33:59.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.308927630s delta_ub 0.014036987s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c54c65000 2019-12-07T19:33:59.858+0000 7f8f24c58700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013453639s,0.014036987s]) 2019-12-07T19:33:59.858+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.295473991s delta_ub 18446744073.696097977s) v5 -- 0x558c57518600 con 0x558c54c65000 2019-12-07T19:33:59.866+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.916+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.966+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.017+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.067+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.117+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.167+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.217+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 195688 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.251+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 9 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.710352893s delta_ub 0.022389492s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77c00 con 0x558c552d1400 2019-12-07T19:34:00.251+0000 7f8f24457700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021910046s,0.022389492s]) 2019-12-07T19:34:00.251+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.688442847s delta_ub 18446744073.687641570s) v5 -- 0x558c57518800 con 0x558c552d1400 2019-12-07T19:34:00.251+0000 7f8f24457700 10 osd.7 21 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 21 -> 22 (as per caller) 2019-12-07T19:34:00.251+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 9 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.710352893s delta_ub 0.022389492s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b9000 con 0x558c54c18000 2019-12-07T19:34:00.251+0000 7f8f23c56700 20 osd.7 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021873421s,0.022389492s]) 2019-12-07T19:34:00.251+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.688479472s delta_ub 18446744073.687678195s) v5 -- 0x558c57518400 con 0x558c54c18000 2019-12-07T19:34:00.252+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 34 ==== osd_map(22..22 src has 1..22) v4 ==== 633+0+0 (unknown 0 0 0) 0x558c575d58c0 con 0x558c54c5bc00 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 osd.7 21 OSD::ms_dispatch: osd_map(22..22 src has 1..22) v4 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 osd.7 21 do_waiters -- start 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 osd.7 21 do_waiters -- finish 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 osd.7 21 _dispatch 0x558c575d58c0 osd_map(22..22 src has 1..22) v4 2019-12-07T19:34:00.252+0000 7f8f1199c700 3 osd.7 21 handle_osd_map epochs [22,22], i have 21, src has [1,22] 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 osd.7 21 handle_osd_map got inc map for epoch 22 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 osd.7 21 got_full_map 22, nothing requested 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 osd.7 21 handle_osd_map pg_num_history pg_num_history(e22 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {22={}} 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 osd.7 21 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e22 [1,22] lci=[13,22]) 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f7800 seq 13 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c89b972f:::inc_osdmap.22:0# key 0x7f7fffffffffffffffc89b97'/!inc_osdmap.22!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1100 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 1 blobs 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x73000~1000/1000,1000,0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x73000~1000] 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757fb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757fb20 blob([0x73000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f490 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c5757fb20) get_ref 0x0~24d Blob(0x558c5757fb20 blob([0x73000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f490 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c5757fb20) get_ref init 0x1000, 1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c5757fb20 blob([0x73000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5757f490 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5757f4a8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149288) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d = 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3760e826:::osdmap.22:0# key 0x7f7fffffffffffffff3760e8'&!osdmap.22!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3760e826:::osdmap.22:0# 0x0~140c 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1101 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3760e826:::osdmap.22:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~40c 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 2 blobs 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x74000~2000/1000,2000,0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x74000~2000] 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fdab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b3b0 blob([0x74000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdab0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c5742b3b0) get_ref 0x0~1000 Blob(0x558c5742b3b0 blob([0x74000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdab0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c5742b3b0) get_ref init 0x1000, 1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5742b3b0 blob([0x74000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fdab0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c574fdac8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fda40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fd880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fda40 blob([!~1000,0x75000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd880 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c574fda40) get_ref 0x1000~40c Blob(0x558c574fda40 blob([!~1000,0x75000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c574fd880 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c574fda40) get_ref init 0x2000, 1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558c574fda40 blob([!~1000,0x75000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558c574fd880 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c574fd898 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x140c 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149508) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3760e826:::osdmap.22:0# 0x0~140c = 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 0 blobs 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 0 blobs 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 23202 (4 ios * 4000 + 7202 bytes) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes 0x558c53a91900,0x558c55149180,0x558c55149400,0x558c55276c80 shared_blobs 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149288) update #-1:c89b972f:::inc_osdmap.22:0# 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c89b972f:::inc_osdmap.22:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149508) update #-1:3760e826:::osdmap.22:0# 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149508) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3760e826:::osdmap.22:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[73000~3000] released 0x[] 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 freelist allocate 0x73000~3000 2019-12-07T19:34:00.252+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:34:00.252+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f7800 2019-12-07T19:34:00.252+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 aio_wait 2019-12-07T19:34:00.252+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:34:00.252+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2500 < min_flush_size 524288 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55149180 had 1 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55149400 had 1 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860594688 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998728) bluestore 86 GiB free (0.959976), bluefs_ratio 0.0399525 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3763c~2277 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3763c 0x3763c~2277 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x763c 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x63c 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8b3 and padding block with 0x74d 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x398b3 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.252+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000161899s (0.000005377s flush + 0.000156522s kv commit) 2019-12-07T19:34:00.252+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7800> 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.252+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f7800 osr 0x558c54ea4b40 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 10 0x1b000~1000 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 11 0x1b000~1000 crc 2e955191 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 10 0x2000~1000 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 21 _committed_osd_maps 22..22 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 21 advance to epoch 22 (<= last 22 <= newest_map 22) 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 11 0x2000~1000 crc 79322a78 2019-12-07T19:34:00.252+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.252+0000 7f8f141a1700 7 osd.7 22 consume_map version 22 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 identify_splits_and_merges 1.0s3 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 identify_splits_and_merges 1.9s4 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 identify_splits_and_merges 1.3s4 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 identify_splits_and_merges 1.4s0 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 identify_splits_and_merges 1.ds2 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 identify_splits_and_merges 1.6s2 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 identify_splits_and_merges 1.7s4 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 22 prune_pg_created 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:0.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:1.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:2.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:3.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:4.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:5.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:6.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7:7.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:00.252+0000 7f8f141a1700 15 osd.7 22 enqueue_peering_evt 1.0s3 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f8f141a1700 15 osd.7 22 enqueue_peering_evt 1.9s4 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558c5500d000 2019-12-07T19:34:00.252+0000 7f8f141a1700 15 osd.7 22 enqueue_peering_evt 1.3s4 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f8f141a1700 15 osd.7 22 enqueue_peering_evt 1.4s0 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f8f141a1700 15 osd.7 22 enqueue_peering_evt 1.ds2 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f8f141a1700 15 osd.7 22 enqueue_peering_evt 1.6s2 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f8f141a1700 15 osd.7 22 enqueue_peering_evt 1.7s4 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 21 -> 22 2019-12-07T19:34:00.252+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 maybe_update_heartbeat_peers forcing update after 20.729860 seconds 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 reset_heartbeat_peers 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 21 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 maybe_update_heartbeat_peers updating 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 adding neighbor peer osd.0 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 adding neighbor peer osd.1 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 adding neighbor peer osd.6 2019-12-07T19:34:00.252+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 adding random peer osd.2 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 adding random peer osd.4 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 adding random peer osd.5 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f8f141a1700 10 osd.7 22 maybe_update_heartbeat_peers 7 peers, extras 0,1,2,3,4,5,6 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f8f141a1700 7 osd.7 22 activate_map version 22 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558c54d5c000 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558c5513e000 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 21 -> 22 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 21 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f8f0918b700 20 osd.7 22 agent_entry empty queue 2019-12-07T19:34:00.252+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 21 -> 22 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558c5500a000 2019-12-07T19:34:00.252+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558c548ab000 2019-12-07T19:34:00.252+0000 7f8f0317f700 10 osd.7 pg_epoch: 21 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558c55141000 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 21 -> 22 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.252+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f8f04181700 10 osd.7 pg_epoch: 21 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:34:00.252+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 21 -> 22 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:34:00.252+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 21 -> 22 2019-12-07T19:34:00.252+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f8f06185700 10 osd.7 pg_epoch: 21 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.252+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f8f06986700 10 osd.7 pg_epoch: 21 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:00.252+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f8f06185700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:00.252+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.252+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] state: Active advmap 2019-12-07T19:34:00.252+0000 7f8f06185700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.252+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:00.252+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] activate_map 2019-12-07T19:34:00.252+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] state: Started advmap 2019-12-07T19:34:00.252+0000 7f8f06185700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558c54d5f000 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f8f06185700 20 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f8f04181700 20 osd.7 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f8f0317f700 20 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.252+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:00.252+0000 7f8f06185700 20 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f8f06185700 20 osd.7 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 21 -> 22 2019-12-07T19:34:00.253+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f8f06185700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f8f06986700 20 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] hit_set_clear 2019-12-07T19:34:00.253+0000 7f8f06185700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 21 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:34:00.253+0000 7f8f04181700 20 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f06986700 20 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] agent_stop 2019-12-07T19:34:00.253+0000 7f8f06185700 20 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f8f06986700 20 osd.7 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f8f06185700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:00.253+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] activate_map 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f8f0117b700 20 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] state: Active: handling ActMap 2019-12-07T19:34:00.253+0000 7f8f0117b700 20 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f8f0117b700 20 osd.7 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] Active: kicking snap trim 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f8f0117b700 20 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f06986700 7 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] state: handle ActMap primary 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f8f06986700 20 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.253+0000 7f8f06986700 15 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] publish_stats_to_osd 21:16 2019-12-07T19:34:00.253+0000 7f8f06986700 20 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f06986700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.267+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.317+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.367+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.397+0000 7f8f21451700 10 osd.7 22 tick 2019-12-07T19:34:00.397+0000 7f8f21451700 10 osd.7 22 do_waiters -- start 2019-12-07T19:34:00.397+0000 7f8f21451700 10 osd.7 22 do_waiters -- finish 2019-12-07T19:34:00.397+0000 7f8f21451700 20 osd.7 22 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:00.417+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.441+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.909301593s delta_ub 0.031079961s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76a00 con 0x558c54c5a400 2019-12-07T19:34:00.442+0000 7f8f24c58700 20 osd.7 22 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030642158s,0.031079961s]) 2019-12-07T19:34:00.442+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.909301593s delta_ub 0.031079961s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b9000 con 0x558c54c5ac00 2019-12-07T19:34:00.442+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.878659435s delta_ub 18446744073.678909458s) v5 -- 0x558c57518600 con 0x558c54c5a400 2019-12-07T19:34:00.442+0000 7f8f24c58700 10 osd.7 22 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 21 -> 22 (as per caller) 2019-12-07T19:34:00.442+0000 7f8f23c56700 20 osd.7 22 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030614936s,0.031079961s]) 2019-12-07T19:34:00.442+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.878686657s delta_ub 18446744073.678936680s) v5 -- 0x558c57518400 con 0x558c54c5ac00 2019-12-07T19:34:00.467+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.517+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.520+0000 7f8f1219d700 20 osd.7 22 reports for 0 queries 2019-12-07T19:34:00.520+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c574ab500 con 0x558c5476d800 2019-12-07T19:34:00.520+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:00.520+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x1f1000/0x1f1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:00.520+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(1 pgs tid 0 v 0) v2 -- 0x558c574ab880 con 0x558c5476d800 2019-12-07T19:34:00.568+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 10 osd.7 22 tick_without_osd_lock 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fdd5000/0x40000000/0x1680000000, data 0x204401/0x229000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 25 osd.7 22 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373260+0000 last_rx_front 2019-12-07T19:33:59.373276+0000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 25 osd.7 22 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373244+0000 last_rx_front 2019-12-07T19:33:59.373149+0000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 25 osd.7 22 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373297+0000 last_rx_front 2019-12-07T19:33:59.373186+0000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 25 osd.7 22 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373171+0000 last_rx_front 2019-12-07T19:33:59.373218+0000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 25 osd.7 22 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373068+0000 last_rx_front 2019-12-07T19:33:59.373042+0000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 25 osd.7 22 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.372987+0000 last_rx_front 2019-12-07T19:33:59.373022+0000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 25 osd.7 22 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:33:59.372735+0000 last_rx_back 2019-12-07T19:33:59.373111+0000 last_rx_front 2019-12-07T19:33:59.373092+0000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 20 osd.7 22 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 10 osd.7 22 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 20 osd.7 22 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:00.577+0000 7f8f1fbbe700 10 osd.7 22 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:00.618+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:34:00.618+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:00.618+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 11 0x2000~1000 2019-12-07T19:34:00.618+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 79322a78 2019-12-07T19:34:00.618+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 11 0x1b000~1000 2019-12-07T19:34:00.618+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 2e955191 2019-12-07T19:34:00.618+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 199324 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:00.618+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:00.636+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 27 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c54e98000 con 0x558c54c5b400 2019-12-07T19:34:00.636+0000 7f8f24457700 15 osd.7 22 enqueue_peering_evt 1.3s4 epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:34:00.636+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.636+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.636+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.636+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.636+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) pg 0x558c5513e000 2019-12-07T19:34:00.636+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 23, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.636+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.640+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 35 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c574e7100 con 0x558c54c5bc00 2019-12-07T19:34:00.640+0000 7f8f23c56700 15 osd.7 22 enqueue_peering_evt 1.6s2 epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:34:00.640+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) pg 0x558c54d5c000 2019-12-07T19:34:00.640+0000 7f8f0197c700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 23, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 59 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e98000 con 0x558c54c64000 2019-12-07T19:34:00.641+0000 7f8f24457700 15 osd.7 22 enqueue_peering_evt 1.9s4 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:34:00.641+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) pg 0x558c5500a000 2019-12-07T19:34:00.641+0000 7f8f08189700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 23, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 60 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e98000 con 0x558c54c64000 2019-12-07T19:34:00.641+0000 7f8f24457700 15 osd.7 22 enqueue_peering_evt 1.7s4 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:34:00.641+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 61 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e98000 con 0x558c54c64000 2019-12-07T19:34:00.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) pg 0x558c54d5f000 2019-12-07T19:34:00.641+0000 7f8f05183700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 23, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.641+0000 7f8f24457700 15 osd.7 22 enqueue_peering_evt 1.0s3 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s) 2019-12-07T19:34:00.641+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e23) pg 0x558c5500d000 2019-12-07T19:34:00.641+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 23, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.646+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c548d2000 0x558c574b9180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:00.646+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c548d2000 0x558c574b9180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:00.646+0000 7f8f24457700 10 osd.7 22 ms_handle_authentication new session 0x558c54c5c340 con 0x558c548d2000 entity client.admin addr 2019-12-07T19:34:00.646+0000 7f8f24457700 10 osd.7 22 ms_handle_authentication session 0x558c54c5c340 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:00.647+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/890522420 conn(0x558c548d2000 0x558c574b9180 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=client.4352 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:00.647+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4352 172.21.15.131:0/890522420 1 ==== command(tid 15: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c575cc780 con 0x558c548d2000 2019-12-07T19:34:00.647+0000 7f8f164bb700 20 osd.7 22 OSD::ms_dispatch: command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:00.647+0000 7f8f164bb700 10 osd.7 22 do_waiters -- start 2019-12-07T19:34:00.647+0000 7f8f164bb700 10 osd.7 22 do_waiters -- finish 2019-12-07T19:34:00.647+0000 7f8f164bb700 20 osd.7 22 _dispatch 0x558c575cc780 command(tid 15: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:00.647+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/890522420 -- command_reply(tid 15: 0 ) v1 -- 0x558c53a6ee00 con 0x558c548d2000 2019-12-07T19:34:00.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 29 ==== MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 141592+0+0 (unknown 0 0 0) 0x558c54e3e800 con 0x558c54c5b400 2019-12-07T19:34:00.651+0000 7f8f24457700 15 osd.7 22 enqueue_op 0x558c5740c160 prio 127 cost 0 latency 0.000024 epoch 23 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.651+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {23=} 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) pg 0x558c5513e000 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 osd.7 22 dequeue_op 0x558c5740c160 prio 127 cost 0 latency 0.000114 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 osd.7 22 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0] 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.651+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740a900 seq 16 2019-12-07T19:34:00.651+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2 keys 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2 keys = 0 2019-12-07T19:34:00.651+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:00.651+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b000~22000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:cd22b62d:::smithi13140796-27:head# 0x1b000~22000 - have 0x1b000 (110592) bytes fadvise_flags 0x20 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x1b000~22000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740a900 17 blobs 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x22000/1000,22000,0 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x76000~e000/1000,22000,0 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x13b000~14000/1000,22000,0 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x76000~e000,0x13b000~14000] 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c9a0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5742b880) get_ref 0x0~2000 Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c9a0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5742b880) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54d3c9b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757fe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f19d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f19d0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757fe30) get_ref 0x0~2000 Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f19d0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757fe30) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574f19e8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9810 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c54e56000) get_ref 0x0~2000 Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9810 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c54e56000) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54cf9828 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e1c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1d50 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1c70) get_ref 0x0~2000 Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1d50 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1c70) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1d68 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e1dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1e30 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1dc0) get_ref 0x0~2000 Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1e30 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1e48 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e1ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1f10 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1ea0) get_ref 0x0~2000 Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1f10 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e1f28 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e1f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f810 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1f80) get_ref 0x0~2000 Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f810 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c574e1f80) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757f828 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757fce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f650 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757fce0) get_ref 0x0~2000 Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f650 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757fce0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757f668 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ecb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757fd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fd50 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757ecb0) get_ref 0x0~2000 Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fd50 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757ecb0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757fd68 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ed90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f6c0) get_ref 0x0~2000 Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ed20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757fdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fdc0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757ed20) get_ref 0x0~2000 Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fdc0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757ed20) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757fdd8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ea80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f730) get_ref 0x0~2000 Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f730) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ea10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757fb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fb90 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757ea10) get_ref 0x0~2000 Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fb90 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757ea10) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757fba8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757eb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f500) get_ref 0x0~2000 Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f500) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757eaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757fc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fc00 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757eaf0) get_ref 0x0~2000 Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fc00 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757eaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757fc18 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 28 ==== osd_map(22..23 src has 1..23) v4 ==== 1258+0+0 (unknown 0 0 0) 0x558c57580d80 con 0x558c54c5b400 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757fc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 osd.7 22 OSD::ms_dispatch: osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fc70 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f1199c700 10 osd.7 22 do_waiters -- start 2019-12-07T19:34:00.651+0000 7f8f1199c700 10 osd.7 22 do_waiters -- finish 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 osd.7 22 _dispatch 0x558c57580d80 osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f570) get_ref 0x0~2000 Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757fc70 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f570) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757fc88 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ebd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f5e0) get_ref 0x0~2000 Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.blob(0x558c5757f5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3d000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) dirty_range mark inline shard dirty 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b000~22000 = 0 2019-12-07T19:34:00.651+0000 7f8f1199c700 3 osd.7 22 handle_osd_map epochs [22,23], i have 22, src has [1,23] 2019-12-07T19:34:00.651+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:34:00.651+0000 7f8f1199c700 10 osd.7 22 handle_osd_map got inc map for epoch 23 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:00.651+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:00.651+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3760e826:::osdmap.22:0# 0x0~0 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000023.00000000000000000008' <- 0000000023.00000000000000000008 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b3b0 blob([0x74000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fdab0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fda40 blob([!~1000,0x75000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558c574fd880 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a900 cost 214050 (18 ios * 4000 + 142050 bytes) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a900 onodes 0x558c574a1680 shared_blobs 2019-12-07T19:34:00.651+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3760e826:::osdmap.22:0# 0x0~140c = 5132 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) update 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:00.651+0000 7f8f1199c700 10 osd.7 22 add_map_bl 22 5132 bytes 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) update inline shard 590 bytes from 31 extents 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:cd22b62d:::smithi13140796-27:head# is 1081 (485 bytes onode + 2 bytes spanning blobs + 594 bytes inline extents) 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a900 allocated 0x[76000~e000,13b000~14000] released 0x[] 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 freelist allocate 0x76000~e000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 freelist _xor first_key 0x0 last_key 0x80000 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 freelist allocate 0x13b000~14000 2019-12-07T19:34:00.651+0000 7f8f0317f700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 prepare 2019-12-07T19:34:00.651+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740a900 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 osd.7 22 got_full_map 23, nothing requested 2019-12-07T19:34:00.651+0000 7f8f1199c700 20 osd.7 22 handle_osd_map pg_num_history pg_num_history(e23 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:00.651+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {23={}} 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 osd.7 22 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e23 [1,23] lci=[13,23]) 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f6900 seq 14 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c57b972f:::inc_osdmap.23:0# key 0x7f7fffffffffffffffc57b97'/!inc_osdmap.23!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:00.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1102 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 1 blobs 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x14f000~1000/1000,1000,0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x14f000~1000] 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd2d0 blob([0x14f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e70 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c575bd2d0) get_ref 0x0~269 Blob(0x558c575bd2d0 blob([0x14f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e70 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c575bd2d0) get_ref init 0x1000, 1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c575bd2d0 blob([0x14f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c574e0e70 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c574e0e88 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149788) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 = 0 2019-12-07T19:34:00.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 23'8 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3de0e826:::osdmap.23:0# key 0x7f7fffffffffffffff3de0e8'&!osdmap.23!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.652+0000 7f8f0317f700 10 osd.7 22 dequeue_op 0x558c5740c160 finish 2019-12-07T19:34:00.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:00.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1103 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3de0e826:::osdmap.23:0# 0x0~1444 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~444 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbbc on front/back, now 0x0~1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 2 blobs 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x150000~2000/1000,2000,0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x150000~2000] 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552642a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264770 blob([0x150000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552642a0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c55264770) get_ref 0x0~1000 Blob(0x558c55264770 blob([0x150000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552642a0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c55264770) get_ref init 0x1000, 1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55264770 blob([0x150000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c552642a0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c552642b8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552659d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552647e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552659d0 blob([!~1000,0x151000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552647e0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c552659d0) get_ref 0x1000~444 Blob(0x558c552659d0 blob([!~1000,0x151000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c552647e0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c552659d0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~444: 0x1000~444 Blob(0x558c552659d0 blob([!~1000,0x151000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,444]) SharedBlob(0x558c552647e0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c552647f8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1444 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149a08) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 = 0 2019-12-07T19:34:00.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 0 blobs 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:00.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6900 0 blobs 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6900 cost 23286 (4 ios * 4000 + 7286 bytes) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6900 onodes 0x558c53a91900,0x558c55149680,0x558c55149900,0x558c55276c80 shared_blobs 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149788) update #-1:c57b972f:::inc_osdmap.23:0# 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149788) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c57b972f:::inc_osdmap.23:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149a08) update #-1:3de0e826:::osdmap.23:0# 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149a08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3de0e826:::osdmap.23:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6900 allocated 0x[14f000~3000] released 0x[] 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 freelist allocate 0x14f000~3000 2019-12-07T19:34:00.652+0000 7f8f1199c700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 prepare 2019-12-07T19:34:00.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6900 2019-12-07T19:34:00.652+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 aio_wait 2019-12-07T19:34:00.652+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a900 2019-12-07T19:34:00.652+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 io_done 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.652+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 aio_wait 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:00.652+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6900 2019-12-07T19:34:00.652+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 io_done 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3093 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1680 had 1 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3140 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3140 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x398b3~c44 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x398b3 0x398b3~c44 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x98b3 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8b3 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4f7 and padding block with 0xb09 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3a4f7 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000153587s (0.000005832s flush + 0.000147755s kv commit) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a900> 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c5525f680> 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 kv_submitted 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a900 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 finishing 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a900 onodes 0x558c574a1680 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9828 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d86c0 space 0x558c54cf9828 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c9b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d85a0 space 0x558c54d3c9b8 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1d68 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8750 space 0x558c574e1d68 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1e48 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d87e0 space 0x558c574e1e48 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1f28 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de000 space 0x558c574e1f28 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f19e8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d8630 space 0x558c574f19e8 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de360 space 0x558c5757ea98 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de480 space 0x558c5757eb78 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de630 space 0x558c5757ebe8 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de240 space 0x558c5757eda8 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f668 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de120 space 0x558c5757f668 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f828 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de090 space 0x558c5757f828 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757fba8 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de3f0 space 0x558c5757fba8 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757fc18 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de510 space 0x558c5757fc18 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55149680 had 1 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55149900 had 1 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757fc88 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de5a0 space 0x558c5757fc88 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757fd68 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de1b0 space 0x558c5757fd68 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757fdd8 in 0x558c53a596c0) _finish_write discard buffer(0x558c548de2d0 space 0x558c5757fdd8 0x0~2000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a900 done 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a900 [] 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 deferred_cleanup 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3a4f7~226c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 0x558c53a91900,0x558c54ce7400,0x558c54ce7680,0x558c54ce7900,0x558c55148f00,0x558c55276c80 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74088 in 0x558c53a59340) _finish_write discard buffer(0x558c551a95f0 space 0x558c54d74088 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3a4f7 0x3a4f7~226c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d741d8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9d40 space 0x558c54d741d8 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xa4f7 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4f7 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x763 and padding block with 0x89d 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d742b8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9dd0 space 0x558c54d742b8 0x1000~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74478 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a360 space 0x558c54d74478 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d745c8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a3f0 space 0x558c54d745c8 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d746a8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a480 space 0x558c54d746a8 0x1000~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 deferred_cleanup 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 deferred_cleanup 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 0x558c53a91900,0x558c55149180,0x558c55149400,0x558c55276c80 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fd898 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a750 space 0x558c574fd898 0x1000~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fdac8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a6c0 space 0x558c574fdac8 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f4a8 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a630 space 0x558c5757f4a8 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 kv_submitted 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 22/18 ECSubWriteReply(tid=36, last_complete=23'8, committed=1, applied=1)) v2 -- 0x558c54ceac00 con 0x558c54c5b400 2019-12-07T19:34:00.652+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3c763 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000140057s (0.000001376s flush + 0.000138681s kv commit) 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6900> 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 kv_submitted 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6900 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f6900 osr 0x558c54ea4b40 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 12 0x1b000~1000 crc 344afd6c 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 12 0x2000~1000 crc c2093e45 2019-12-07T19:34:00.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7 22 _committed_osd_maps 23..23 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7 22 advance to epoch 23 (<= last 23 <= newest_map 23) 2019-12-07T19:34:00.652+0000 7f8f141a1700 7 osd.7 23 consume_map version 23 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 identify_splits_and_merges 1.0s3 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 identify_splits_and_merges 1.9s4 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 identify_splits_and_merges 1.3s4 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 identify_splits_and_merges 1.4s0 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 identify_splits_and_merges 1.ds2 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 identify_splits_and_merges 1.6s2 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 identify_splits_and_merges 1.7s4 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 23 prune_pg_created 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:0.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:1.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:2.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:3.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s)) prio 255 cost 10 e23) pg 0x558c5500d000 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:4.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:5.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:6.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7:7.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 22 -> 23 2019-12-07T19:34:00.652+0000 7f8f141a1700 10 osd.7 23 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:00.652+0000 7f8f141a1700 15 osd.7 23 enqueue_peering_evt 1.0s3 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 22 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f141a1700 15 osd.7 23 enqueue_peering_evt 1.9s4 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f141a1700 15 osd.7 23 enqueue_peering_evt 1.3s4 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) pg 0x558c54d5c000 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7f8f141a1700 15 osd.7 23 enqueue_peering_evt 1.4s0 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:00.652+0000 7f8f141a1700 15 osd.7 23 enqueue_peering_evt 1.ds2 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7f8f141a1700 15 osd.7 23 enqueue_peering_evt 1.6s2 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7f8f141a1700 15 osd.7 23 enqueue_peering_evt 1.7s4 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7f8f141a1700 7 osd.7 23 activate_map version 23 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 22 -> 23 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 22 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) pg 0x558c5500a000 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558c55141000 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.0s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074625527s ub 36.074625527s int 16s) 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 22 -> 23 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 22 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558c548ab000 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 22 -> 23 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c5740b800 seq 13 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 22 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 22 -> 23 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 22 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:00.652+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b800 onodes shared_blobs 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b800 allocated 0x[] released 0x[] 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] state: Active advmap 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 prepare 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b800 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 io_done 2019-12-07T19:34:00.652+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] on_active_advmap new removed_snaps [2~1], snap_trimq now [2~1] 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f04181700 20 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] state: Started advmap 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558c5500d000 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:00.652+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f6300 seq 24 2019-12-07T19:34:00.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] hit_set_clear 2019-12-07T19:34:00.652+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:00.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] agent_stop 2019-12-07T19:34:00.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:00.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d59800 seq 13 2019-12-07T19:34:00.652+0000 7f8f0297e700 20 osd.7 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:00.652+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:00.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:00.652+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes shared_blobs 2019-12-07T19:34:00.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:00.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] activate_map 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:00.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3c763~475 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:00.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:00.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3c763 0x3c763~475 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59800 onodes shared_blobs 2019-12-07T19:34:00.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:00.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59800 allocated 0x[] released 0x[] 2019-12-07T19:34:00.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] Active: kicking snap trim 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xc763 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x763 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbd8 and padding block with 0x428 2019-12-07T19:34:00.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 prepare 2019-12-07T19:34:00.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59800 2019-12-07T19:34:00.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:00.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 io_done 2019-12-07T19:34:00.653+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558c5500a000 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) pg 0x558c54d5f000 2019-12-07T19:34:00.653+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:00.653+0000 7f8f04181700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] exit NotTrimming 2019-12-07T19:34:00.653+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f0918b700 20 osd.7 23 agent_entry empty queue 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] enter Trimming 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 22 -> 23 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] enter Trimming/WaitReservation 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 22 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) pg 0x558c5513e000 2019-12-07T19:34:00.653+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:00.653+0000 7f8f0297e700 15 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] publish_stats_to_osd 21:17 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f8f0297e700 7 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] state: handle ActMap primary 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 22 -> 23 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f8f0297e700 15 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] publish_stats_to_osd 22:18 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 22 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0000 seq 22 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] activate_map: Not dirtying info: last_persisted is 22 while current is 23 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54d58000 seq 17 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558c54d5f000 2019-12-07T19:34:00.653+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:00.653+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558c54d5c000 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f8f0197c700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558c5513e000 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f0b990700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:00.653+0000 7f8f0b990700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] SnapTrimmer state: NotTrimming: trimming 2 2019-12-07T19:34:00.653+0000 7f8f0b990700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[2~1]] exit Trimming/WaitReservation 2019-12-07T19:34:00.653+0000 7f8f0b990700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.653+0000 7f8f0b990700 15 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] publish_stats_to_osd 23:19 2019-12-07T19:34:00.653+0000 7f8f0b990700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:00.653+0000 7f8f0b990700 10 osd.7 23 queueing pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[2~1]] for snaptrim 2019-12-07T19:34:00.653+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=23) prio 5 cost 1048576 e23) 2019-12-07T19:34:00.653+0000 7f8f0b990700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.653+0000 7f8f0b990700 15 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[2~1]] publish_stats_to_osd 23:20 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3cbd8 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000408325s (0.000000120s flush + 0.000408205s kv commit) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b800> 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 kv_submitted 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b800 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 finishing 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b800 onodes 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b800 done 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b800 [] 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4472 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4495 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4495 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3cbd8~118f to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3cbd8 0x3cbd8~118f to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xcbd8 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbd8 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd67 and padding block with 0x299 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=23) prio 5 cost 1048576 e23) queued 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=23) prio 5 cost 1048576 e23) pg 0x558c548ab000 2019-12-07T19:34:00.653+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 30 ==== MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 96536+0+0 (unknown 0 0 0) 0x558c54e3f000 con 0x558c54c5b400 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[2~1]] snap_trimmer posting 2019-12-07T19:34:00.653+0000 7f8f24457700 15 osd.7 23 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000025 epoch 23 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.653+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 2 2019-12-07T19:34:00.653+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:00.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:00.653+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 0 2019-12-07T19:34:00.653+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000002_.0_0000000000000001.4) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3dd67 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[2~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim] SnapTrimmer state: adding snap 2 to purged_snaps 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000158947s (0.000000100s flush + 0.000158847s kv commit) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300,0x558c54d59800,0x558c54ea0000,0x558c54d58000> 2019-12-07T19:34:00.653+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 kv_submitted 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1]] SnapTrimmer state: purged_snaps now [2~1], snap_trimq now [] 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59800 2019-12-07T19:34:00.653+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 finishing 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59800 onodes 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59800 done 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d58c00 seq 10 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59800 [] 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:34:00.653+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.653+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.653+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:34:00.653+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:34:00.653+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1]] share_pg_info 2019-12-07T19:34:00.653+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558c548a1c00 con 0x558c54c64400 2019-12-07T19:34:00.653+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558c54e99c00 con 0x558c54c64000 2019-12-07T19:34:00.653+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558c548a3800 con 0x558c54c19800 2019-12-07T19:34:00.653+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558c57450000 con 0x558c54c5b400 2019-12-07T19:34:00.653+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558c57450700 con 0x558c54c19c00 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1]] exit Trimming 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:00.653+0000 7f8f06986700 15 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] publish_stats_to_osd 23:21 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] enter NotTrimming 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:00.653+0000 7f8f06986700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] snap_trimmer complete 2019-12-07T19:34:00.653+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.653+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) pg 0x558c5513e000 2019-12-07T19:34:00.653+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.653+0000 7f8f07187700 10 osd.7 23 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000337 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] append_log log((0'0,23'8], crt=21'7) [23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0] 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3dd67~475 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3dd67 0x3dd67~475 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xdd67 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd67 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1dc and padding block with 0xe24 2019-12-07T19:34:00.653+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] add_log_entry 23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.653+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log approx pg log length = 9 2019-12-07T19:34:00.653+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.653+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.653+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.653+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54d58600 seq 18 2019-12-07T19:34:00.653+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2 keys 2019-12-07T19:34:00.653+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2 keys = 0 2019-12-07T19:34:00.653+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:00.653+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:00.653+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d000~17000 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:cd22b62d:::smithi13140796-27:head# 0x3d000~17000 - have 0x3d000 (249856) bytes fadvise_flags 0x20 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.653+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x3d000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58600 12 blobs 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3e1dc 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138166s (0.000000113s flush + 0.000138053s kv commit) 2019-12-07T19:34:00.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:34:00.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.653+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x17000/1000,17000,0 2019-12-07T19:34:00.653+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x152000~b000/1000,17000,0 2019-12-07T19:34:00.653+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x25b000~c000/1000,17000,0 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x152000~b000,0x25b000~c000] 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1260 sbid 0x0)) 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore.blob(0x558c574f0e70) get_ref 0x0~2000 Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1260 sbid 0x0)) 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore.blob(0x558c574f0e70) get_ref init 0x2000, 1000 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574f1278 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e422a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore.blob(0x558c54e422a0) get_ref 0x0~2000 Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore.blob(0x558c54e422a0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.653+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e421c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e421c0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c574f0fc0) get_ref 0x0~2000 Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e421c0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c574f0fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e421d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42460 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42230) get_ref 0x0~2000 Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42460 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42230) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42478 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e423f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42380 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e423f0) get_ref 0x0~2000 Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42380 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e423f0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42398 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e420e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e420e0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42690) get_ref 0x0~2000 Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e420e0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42690) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e420f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42620 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42150) get_ref 0x0~2000 Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42620 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42150) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42638 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42000) get_ref 0x0~2000 Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e42000) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43a40) get_ref 0x0~2000 Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43a40) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43b90 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43c70) get_ref 0x0~2000 Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43b90 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43c70) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e43ba8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e439d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e439d0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43c00) get_ref 0x0~2000 Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e439d0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43c00) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x51000~2000: 0x0~2000 Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e439e8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e43b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43b20) get_ref 0x1000~1000 Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.blob(0x558c54e43b20) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x53000~1000: 0x1000~1000 Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e43ac8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x54000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a1788) dirty_range mark inline shard dirty 2019-12-07T19:34:00.654+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d000~17000 = 0 2019-12-07T19:34:00.654+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:34:00.654+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:00.654+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000023.00000000000000000009' <- 0000000023.00000000000000000009 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.654+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:00.654+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58600 cost 152186 (14 ios * 4000 + 96186 bytes) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58600 onodes 0x558c574a1680 shared_blobs 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a1788) update 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a1788) update inline shard 836 bytes from 43 extents 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:cd22b62d:::smithi13140796-27:head# is 1327 (485 bytes onode + 2 bytes spanning blobs + 840 bytes inline extents) 2019-12-07T19:34:00.654+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58600 allocated 0x[152000~b000,25b000~c000] released 0x[] 2019-12-07T19:34:00.654+0000 7f8f07187700 10 freelist allocate 0x152000~b000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:00.654+0000 7f8f07187700 10 freelist allocate 0x25b000~c000 2019-12-07T19:34:00.654+0000 7f8f07187700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:34:00.654+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 prepare 2019-12-07T19:34:00.654+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58600 2019-12-07T19:34:00.654+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] op_applied version 23'9 2019-12-07T19:34:00.654+0000 7f8f07187700 10 osd.7 23 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:00.654+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 aio_wait 2019-12-07T19:34:00.654+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58600 2019-12-07T19:34:00.654+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 io_done 2019-12-07T19:34:00.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2496 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1680 had 1 2019-12-07T19:34:00.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:00.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2519 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2519 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 31 ==== MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2294+0+0 (unknown 0 0 0) 0x558c54e41800 con 0x558c54c5b400 2019-12-07T19:34:00.655+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3e1dc~9d7 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.655+0000 7f8f24457700 15 osd.7 23 enqueue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000026 epoch 23 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.655+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3e1dc 0x3e1dc~9d7 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.655+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.655+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.655+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xe1dc 2019-12-07T19:34:00.655+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1dc 2019-12-07T19:34:00.655+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.655+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbb3 and padding block with 0x44d 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) pg 0x558c5513e000 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 23 dequeue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000080 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s4( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log log((0'0,23'9], crt=21'7) [23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] add_log_entry 23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.655+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c574ff200 seq 19 2019-12-07T19:34:00.655+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 3 keys 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 3 keys = 0 2019-12-07T19:34:00.655+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:00.655+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000023.00000000000000000010' <- 0000000023.00000000000000000010 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff200 cost 5944 (1 ios * 4000 + 1944 bytes) 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff200 onodes 0x558c574a1680 shared_blobs 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a1788) update 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:cd22b62d:::smithi13140796-27:head# is 1327 (485 bytes onode + 2 bytes spanning blobs + 840 bytes inline extents) 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff200 allocated 0x[] released 0x[] 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 prepare 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff200 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 io_done 2019-12-07T19:34:00.655+0000 7f8f24457700 0 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] op_applied version 23'10 2019-12-07T19:34:00.655+0000 7f8f0317f700 10 osd.7 23 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:34:00.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 92 2019-12-07T19:34:00.655+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:00.655+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=READY pgs=3 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:00.655+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=READY pgs=3 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:00.655+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3ebb3 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58600> 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 kv_submitted 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58600 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 finishing 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58600 onodes 0x558c574a1680 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _finish_write discard buffer(0x558c54c9abd0 space 0x558c54e42088 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e420f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c54c9bb00 space 0x558c54e420f8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e421d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d9320 space 0x558c54e421d8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001342274s (0.000047332s flush + 0.001294942s kv commit) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _finish_write discard buffer(0x558c551a9440 space 0x558c54e42328 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42398 in 0x558c53a596c0) _finish_write discard buffer(0x558c54c9bb90 space 0x558c54e42398 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42478 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d93b0 space 0x558c54e42478 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42638 in 0x558c53a596c0) _finish_write discard buffer(0x558c54c9ba70 space 0x558c54e42638 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 23/18 ECSubWriteReply(tid=37, last_complete=23'9, committed=1, applied=1)) v2 -- 0x558c54d77c00 con 0x558c54c5b400 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e439e8 in 0x558c53a596c0) _finish_write discard buffer(0x558c54c9b8c0 space 0x558c54e439e8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43ac8 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d9290 space 0x558c54e43ac8 0x1000~1000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2427 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43ba8 in 0x558c53a596c0) _finish_write discard buffer(0x558c54c9b560 space 0x558c54e43ba8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a596c0) _finish_write discard buffer(0x558c54c9b9e0 space 0x558c54e43f98 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1278 in 0x558c53a596c0) _finish_write discard buffer(0x558c573d90e0 space 0x558c574f1278 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 done 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 kv_queued 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58600 [] 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1680 had 1 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2450 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2450 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3ebb3~992 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3ebb3 0x3ebb3~992 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xebb3 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbb3 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x545 and padding block with 0xabb 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3f545 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132927s (0.000000096s flush + 0.000132831s kv commit) 2019-12-07T19:34:00.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff200> 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 kv_submitted 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff200 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 finishing 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff200 onodes 0x558c574a1680 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 done 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff200 [] 2019-12-07T19:34:00.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.656+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 23/18 ECSubWriteReply(tid=38, last_complete=23'10, committed=1, applied=1)) v2 -- 0x558c54ceac00 con 0x558c54c5b400 2019-12-07T19:34:00.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 32 ==== MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54897000 con 0x558c54c5b400 2019-12-07T19:34:00.657+0000 7f8f24457700 15 osd.7 23 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000022 epoch 23 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:00.657+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.657+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) pg 0x558c5513e000 2019-12-07T19:34:00.658+0000 7f8f07187700 10 osd.7 23 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.000079 MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 pg pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.658+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.3s4 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:00.658+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] append_log log((0'0,23'10], crt=21'7) [] 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] rollforward: entry=23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] rollforward: entry=23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] rollforward: entry=23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.658+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:00.658+0000 7f8f07187700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.658+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.658+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:00.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740bb00 seq 20 2019-12-07T19:34:00.658+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:00.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:00.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:00.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:00.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:34:00.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:00.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:00.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:00.658+0000 7f8f07187700 10 osd.7 23 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3f545~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3f545 0x3f545~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xf545 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x545 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9b6 and padding block with 0x64a 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3f9b6 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000115432s (0.000000122s flush + 0.000115310s kv commit) 2019-12-07T19:34:00.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740bb00> 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:00.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:00.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:34:00.658+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.658+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.658+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 23/18 ECSubWriteReply(tid=39, last_complete=23'10, committed=1, applied=1)) v2 -- 0x558c57518000 con 0x558c54c5b400 2019-12-07T19:34:00.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.658+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 32 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c548a3800 con 0x558c54c19800 2019-12-07T19:34:00.658+0000 7f8f24c58700 15 osd.7 23 enqueue_peering_evt 1.ds2 epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:34:00.658+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.658+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.658+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.658+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.658+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e23) pg 0x558c55141000 2019-12-07T19:34:00.658+0000 7f8f06185700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:34:00.658+0000 7f8f06185700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.659+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:00.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:00.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c574fe000 seq 16 2019-12-07T19:34:00.659+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:00.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:00.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:00.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:00.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:00.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe000 onodes shared_blobs 2019-12-07T19:34:00.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe000 allocated 0x[] released 0x[] 2019-12-07T19:34:00.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 prepare 2019-12-07T19:34:00.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe000 2019-12-07T19:34:00.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 io_done 2019-12-07T19:34:00.659+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3f9b6~475 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3f9b6 0x3f9b6~475 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xf9b6 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9b6 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe2b and padding block with 0x1d5 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3fe2b 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000128150s (0.000000121s flush + 0.000128029s kv commit) 2019-12-07T19:34:00.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe000> 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 kv_submitted 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe000 2019-12-07T19:34:00.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 finishing 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe000 onodes 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 done 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:00.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe000 [] 2019-12-07T19:34:00.659+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.659+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.663+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 36 ==== osd_map(23..23 src has 1..23) v4 ==== 661+0+0 (unknown 0 0 0) 0x558c575d5b00 con 0x558c54c5bc00 2019-12-07T19:34:00.663+0000 7f8f1199c700 20 osd.7 23 OSD::ms_dispatch: osd_map(23..23 src has 1..23) v4 2019-12-07T19:34:00.663+0000 7f8f1199c700 10 osd.7 23 do_waiters -- start 2019-12-07T19:34:00.663+0000 7f8f1199c700 10 osd.7 23 do_waiters -- finish 2019-12-07T19:34:00.663+0000 7f8f1199c700 20 osd.7 23 _dispatch 0x558c575d5b00 osd_map(23..23 src has 1..23) v4 2019-12-07T19:34:00.663+0000 7f8f1199c700 3 osd.7 23 handle_osd_map epochs [23,23], i have 23, src has [1,23] 2019-12-07T19:34:00.663+0000 7f8f1199c700 10 osd.7 23 no new maps here, dropping 2019-12-07T19:34:00.663+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 37 ==== MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 1360+0+0 (unknown 0 0 0) 0x558c575be800 con 0x558c54c5bc00 2019-12-07T19:34:00.663+0000 7f8f23c56700 15 osd.7 23 enqueue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000021 epoch 23 MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:34:00.663+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e23) pg 0x558c54d5c000 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 23 dequeue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000046 MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 23 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 22 -> 23 (as per caller) 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] append_log log((0'0,21'12], crt=21'12) [23'13 (21'3) delete 1:657ac0fe:::smithi13140796-4:head by client.4540.0:184 2019-12-07T19:34:00.663938+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 0, new_object: 0] 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] add_log_entry 23'13 (21'3) delete 1:657ac0fe:::smithi13140796-4:head by client.4540.0:184 2019-12-07T19:34:00.663938+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] append_log approx pg log length = 13 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.663+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.663+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.663+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f6000 seq 25 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:657ac0fe:::smithi13140796-4:head#d key 0x828000000000000001657ac0fe21736d'ithi13140796-4!='0xfffffffffffffffe000000000000000d'o' 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:00.663+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# -> 2#1:657ac0fe:::smithi13140796-4:head#d 2019-12-07T19:34:00.663+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head# -> 2#1:657ac0fe:::smithi13140796-4:head#d = 0 2019-12-07T19:34:00.663+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000023.00000000000000000013' <- 0000000023.00000000000000000013 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.663+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:00.663+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6000 cost 5801 (1 ios * 4000 + 1801 bytes) 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6000 onodes 0x558c54d96280 shared_blobs 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore.extentmap(0x558c54d96388) update 2#1:657ac0fe:::smithi13140796-4:head#d 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:657ac0fe:::smithi13140796-4:head#d is 530 (483 bytes onode + 2 bytes spanning blobs + 45 bytes inline extents) 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6000 allocated 0x[] released 0x[] 2019-12-07T19:34:00.663+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 prepare 2019-12-07T19:34:00.663+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6000 2019-12-07T19:34:00.663+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 io_done 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] op_applied version 23'13 2019-12-07T19:34:00.663+0000 7f8f05984700 10 osd.7 23 dequeue_op 0x558c5740c9a0 finish 2019-12-07T19:34:00.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 469 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2020 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96280 had 1 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455680 had 1 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2043 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2043 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3fe2b~7fb to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3fe2b 0x3fe2b~7fb to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.738550+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _allocate len 0x626 from 0 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _allocate len 0x626 from 1 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd170000 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd170000~10000/10000,10000,acd170000 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x50000 to include allocated 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 26 (was clean) 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0xfe2b 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe2b 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x626 and padding block with 0x9da 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x50000 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40626 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (26) on file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]), flushing log 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 26 len 0x49 crc 0xf5b33445) 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x19000~1000 to file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x19000 0x19000~1000 to file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x19000 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x1a000 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:00.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000018 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 26 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000362909s (0.000000119s flush + 0.000362790s kv commit) 2019-12-07T19:34:00.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6000> 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 kv_submitted 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6000 2019-12-07T19:34:00.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 finishing 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6000 onodes 0x558c54d96280 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 done 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:00.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6000 [] 2019-12-07T19:34:00.664+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.664+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.664+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 23/18 ECSubWriteReply(tid=47, last_complete=23'13, committed=1, applied=1)) v2 -- 0x558c575b9200 con 0x558c54c5bc00 2019-12-07T19:34:00.664+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.665+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 38 ==== MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558c575bf000 con 0x558c54c5bc00 2019-12-07T19:34:00.665+0000 7f8f23c56700 15 osd.7 23 enqueue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000022 epoch 23 MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 2019-12-07T19:34:00.665+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2) prio 127 cost 0 e23) pg 0x558c54d5c000 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 osd.7 23 dequeue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 pg pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] append_log log((0'0,23'13], crt=21'12) [] 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] rollforward: entry=23'13 (21'3) delete 1:657ac0fe:::smithi13140796-4:head by client.4540.0:184 2019-12-07T19:34:00.663938+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] append_log approx pg log length = 13 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.665+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740af00 seq 26 2019-12-07T19:34:00.665+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head#d onode 0x558c54d96280 txc 0x558c5740af00 2019-12-07T19:34:00.665+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head#d 0x0 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) put_ref 0x0~2000 Blob(0x558c54d750a0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552656c0 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75110) put_ref 0x0~2000 Blob(0x558c54d75110 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96388) dirty_range mark inline shard dirty 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d750a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c552656c0 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x37000~2000] 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c552656d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 _rm buffer(0x558c54c9b3b0 space 0x558c552656d8 0x0~1000 clean) 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x37000~2000 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.sharedblob(0x558c552656c0) put 0x558c552656c0 removing self from set 0x558c54d38c10 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c552656d8 in 0x558c53a58fc0) _clear 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d75110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x39000~2000] 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75110) discard_unallocated 0x0~2000 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x39000~2000 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.sharedblob(0x558c54d75180) put 0x558c54d75180 removing self from set 0x558c54d38c10 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a58fc0) _clear 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:657ac0fe:::smithi13140796-4:head#d = 0 2019-12-07T19:34:00.665+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[37000~4000] 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x37000~4000 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 freelist release 0x37000~4000 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:00.665+0000 7f8f0197c700 10 osd.7 23 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1249 < min_flush_size 524288 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96280 had 1 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40626~4f8 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40626 0x40626~4f8 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x10626 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x626 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb1e and padding block with 0x4e2 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40b1e 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:00.665+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000082114s (0.000000076s flush + 0.000082038s kv commit) 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:00.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00> 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:00.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:00.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [37000~4000] 2019-12-07T19:34:00.665+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x37000~4000 2019-12-07T19:34:00.665+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:00.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:00.665+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 23/18 ECSubWriteReply(tid=48, last_complete=23'13, committed=1, applied=1)) v2 -- 0x558c575b9000 con 0x558c54c5bc00 2019-12-07T19:34:00.665+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.668+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.677+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4352 172.21.15.131:0/890522420 2 ==== command(tid 16: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c575cc960 con 0x558c548d2000 2019-12-07T19:34:00.677+0000 7f8f164bb700 20 osd.7 23 OSD::ms_dispatch: command(tid 16: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:00.677+0000 7f8f164bb700 10 osd.7 23 do_waiters -- start 2019-12-07T19:34:00.677+0000 7f8f164bb700 10 osd.7 23 do_waiters -- finish 2019-12-07T19:34:00.677+0000 7f8f164bb700 20 osd.7 23 _dispatch 0x558c575cc960 command(tid 16: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:00.678+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/890522420 -- command_reply(tid 16: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c53a6fa40 con 0x558c548d2000 2019-12-07T19:34:00.680+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/890522420 conn(0x558c548d2000 msgr2=0x558c574b9180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 98 2019-12-07T19:34:00.680+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/890522420 conn(0x558c548d2000 msgr2=0x558c574b9180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:00.680+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/890522420 conn(0x558c548d2000 0x558c574b9180 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:00.680+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/890522420 conn(0x558c548d2000 0x558c574b9180 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:00.680+0000 7f8f164bb700 2 osd.7 23 ms_handle_reset con 0x558c548d2000 session 0x558c54c5c340 2019-12-07T19:34:00.680+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] reap_dead start 2019-12-07T19:34:00.718+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.768+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.818+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.868+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.919+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.969+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.019+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.069+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.071+0000 7f8f0097a700 5 osd.7 23 heartbeat osd_stat(store_statfs(0x163fdd5000/0x40000000/0x1680000000, data 0x204401/0x229000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,6] op hist []) 2019-12-07T19:34:01.071+0000 7f8f0097a700 20 osd.7 23 check_full_status cur ratio 2.3524e-05, physical ratio 2.3524e-05, new state none 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690383489s) v5 -- 0x558c57518a00 con 0x558c5476d400 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690383489s) v5 -- 0x558c57518c00 con 0x558c551b6800 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.688301197s) v5 -- 0x558c57518e00 con 0x558c54c9e000 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.688301197s) v5 -- 0x558c57519000 con 0x558c54c9e400 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.673297409s) v5 -- 0x558c57519200 con 0x558c551b7800 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.673297409s) v5 -- 0x558c57519400 con 0x558c551b7c00 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.687678195s) v5 -- 0x558c57519600 con 0x558c552c2000 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.687678195s) v5 -- 0x558c57519800 con 0x558c552c2400 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.696097977s) v5 -- 0x558c57519a00 con 0x558c552c2800 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.696097977s) v5 -- 0x558c54c58800 con 0x558c552c2c00 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.678936680s) v5 -- 0x558c54c58a00 con 0x558c54c65800 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.678936680s) v5 -- 0x558c54c59000 con 0x558c54c65c00 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690666404s) v5 -- 0x558c54c58400 con 0x558c551b7000 2019-12-07T19:34:01.071+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690666404s) v5 -- 0x558c54c59c00 con 0x558c551b7400 2019-12-07T19:34:01.072+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 3 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.539476211s delta_ub 0.031032696s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c65800 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.372987+0000 -> 2019-12-07T19:34:01.073229+0000 last_rx_front 2019-12-07T19:33:59.373022+0000 2019-12-07T19:34:01.072+0000 7f8f23c56700 10 osd.7 23 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 22 -> 23 (as per caller) 2019-12-07T19:34:01.072+0000 7f8f23c56700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030807151s,0.031032696s]) 2019-12-07T19:34:01.072+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.522382047s delta_ub 0.013938532s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c552c2800 2019-12-07T19:34:01.072+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 3 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.539499541s delta_ub 0.031056026s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c65c00 2019-12-07T19:34:01.072+0000 7f8f24c58700 25 osd.7 23 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373068+0000 -> 2019-12-07T19:34:01.073263+0000 last_rx_front 2019-12-07T19:33:59.373042+0000 2019-12-07T19:34:01.072+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.522428445s delta_ub 0.013984930s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77c00 con 0x558c552c2c00 2019-12-07T19:34:01.072+0000 7f8f24c58700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013679209s,0.013938532s]) 2019-12-07T19:34:01.072+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6822/36937 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.527675697s delta_ub 0.019232182s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c551b7000 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073229+0000 last_rx_front 2019-12-07T19:33:59.373022+0000 -> 2019-12-07T19:34:01.073286+0000 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:01.073004+0000) and older pending ping(s) 2019-12-07T19:34:01.072+0000 7f8f23c56700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030773692s,0.031056026s]) 2019-12-07T19:34:01.072+0000 7f8f24457700 25 osd.7 23 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073263+0000 last_rx_front 2019-12-07T19:33:59.373042+0000 -> 2019-12-07T19:34:01.073309+0000 2019-12-07T19:34:01.072+0000 7f8f24457700 25 osd.7 23 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:01.073004+0000) and older pending ping(s) 2019-12-07T19:34:01.072+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 3 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.530220535s delta_ub 0.021777020s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c9e000 2019-12-07T19:34:01.072+0000 7f8f24457700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013679458s,0.013984930s]) 2019-12-07T19:34:01.072+0000 7f8f24c58700 25 osd.7 23 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373111+0000 -> 2019-12-07T19:34:01.073339+0000 last_rx_front 2019-12-07T19:33:59.373092+0000 2019-12-07T19:34:01.072+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.6 v2:172.21.15.109:6820/36937 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.527722213s delta_ub 0.019278698s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77c00 con 0x558c551b7400 2019-12-07T19:34:01.072+0000 7f8f24c58700 10 osd.7 23 maybe_share_map con 0x558c54c19400 v2:172.21.15.109:6818/36937 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:01.072+0000 7f8f24c58700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018897227s,0.019232182s]) 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373244+0000 -> 2019-12-07T19:34:01.073353+0000 last_rx_front 2019-12-07T19:33:59.373149+0000 2019-12-07T19:34:01.072+0000 7f8f23c56700 10 osd.7 23 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:01.072+0000 7f8f23c56700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021428137s,0.021777020s]) 2019-12-07T19:34:01.072+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.545056755s delta_ub 0.036613240s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c551b7c00 2019-12-07T19:34:01.072+0000 7f8f24457700 25 osd.7 23 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073339+0000 last_rx_front 2019-12-07T19:33:59.373092+0000 -> 2019-12-07T19:34:01.073365+0000 2019-12-07T19:34:01.072+0000 7f8f24457700 25 osd.7 23 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:34:01.073004+0000) and older pending ping(s) 2019-12-07T19:34:01.072+0000 7f8f24457700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [0.018917564s,0.019278698s]) 2019-12-07T19:34:01.072+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.530808487s delta_ub 0.022364972s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c552c2400 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373171+0000 last_rx_front 2019-12-07T19:33:59.373218+0000 -> 2019-12-07T19:34:01.073380+0000 2019-12-07T19:34:01.072+0000 7f8f23c56700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021988749s,0.022364972s]) 2019-12-07T19:34:01.072+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.530833360s delta_ub 0.022389845s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77c00 con 0x558c552c2000 2019-12-07T19:34:01.072+0000 7f8f24c58700 25 osd.7 23 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373297+0000 last_rx_front 2019-12-07T19:33:59.373186+0000 -> 2019-12-07T19:34:01.073389+0000 2019-12-07T19:34:01.072+0000 7f8f24c58700 10 osd.7 23 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:01.072+0000 7f8f24c58700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036227737s,0.036613240s]) 2019-12-07T19:34:01.072+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.545023464s delta_ub 0.036579949s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c551b7800 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373297+0000 -> 2019-12-07T19:34:01.073398+0000 last_rx_front 2019-12-07T19:34:01.073389+0000 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:01.073004+0000) and older pending ping(s) 2019-12-07T19:34:01.072+0000 7f8f23c56700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036186067s,0.036579949s]) 2019-12-07T19:34:01.072+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.528032143s delta_ub 0.019588628s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c551b6800 2019-12-07T19:34:01.072+0000 7f8f24457700 25 osd.7 23 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373171+0000 -> 2019-12-07T19:34:01.073408+0000 last_rx_front 2019-12-07T19:34:01.073380+0000 2019-12-07T19:34:01.072+0000 7f8f24457700 25 osd.7 23 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:01.073004+0000) and older pending ping(s) 2019-12-07T19:34:01.072+0000 7f8f24457700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021985398s,0.022389845s]) 2019-12-07T19:34:01.072+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 7 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.527979159s delta_ub 0.019535644s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c5476d400 2019-12-07T19:34:01.072+0000 7f8f23c56700 25 osd.7 23 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:33:59.373260+0000 -> 2019-12-07T19:34:01.073421+0000 last_rx_front 2019-12-07T19:33:59.373276+0000 2019-12-07T19:34:01.072+0000 7f8f23c56700 10 osd.7 23 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:01.072+0000 7f8f23c56700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019118058s,0.019535644s]) 2019-12-07T19:34:01.072+0000 7f8f24c58700 25 osd.7 23 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073421+0000 last_rx_front 2019-12-07T19:33:59.373276+0000 -> 2019-12-07T19:34:01.073431+0000 2019-12-07T19:34:01.072+0000 7f8f24c58700 25 osd.7 23 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:34:01.073004+0000) and older pending ping(s) 2019-12-07T19:34:01.072+0000 7f8f24c58700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019161467s,0.019588628s]) 2019-12-07T19:34:01.072+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 3 ==== osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.530291901s delta_ub 0.021848386s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c54c9e400 2019-12-07T19:34:01.072+0000 7f8f24c58700 25 osd.7 23 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073353+0000 last_rx_front 2019-12-07T19:33:59.373149+0000 -> 2019-12-07T19:34:01.073454+0000 2019-12-07T19:34:01.072+0000 7f8f24c58700 25 osd.7 23 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:01.073004+0000) and older pending ping(s) 2019-12-07T19:34:01.072+0000 7f8f24c58700 20 osd.7 23 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021397955s,0.021848386s]) 2019-12-07T19:34:01.119+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.169+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.219+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.269+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.283+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36937 conn(0x558c53ae4c00 msgr2=0x558c53aa6680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:34:01.283+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36937 conn(0x558c53ae4c00 msgr2=0x558c53aa6680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:01.283+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36937 conn(0x558c53ae4c00 0x558c53aa6680 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.283+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36937 conn(0x558c53ae4c00 0x558c53aa6680 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36937 conn(0x558c552d0400 msgr2=0x558c552d6580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 72 2019-12-07T19:34:01.283+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 msgr2=0x558c552b2580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 64 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36937 conn(0x558c552d0400 msgr2=0x558c552d6580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:01.283+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 msgr2=0x558c552b2580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36937 conn(0x558c552d0400 0x558c552d6580 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.283+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c551b7400 msgr2=0x558c552b2b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 71 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36937 conn(0x558c552d0400 0x558c552d6580 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.283+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c551b7400 msgr2=0x558c552b2b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:01.283+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 0x558c552b2580 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.283+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c551b7400 0x558c552b2b00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.283+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c551b7400 0x558c552b2b00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.283+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 0x558c552b2580 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.283+0000 7f8f1099a700 20 osd.7 23 heartbeat_reset con 0x558c551b7400 s 0x558c552bc1a0 2019-12-07T19:34:01.283+0000 7f8f1099a700 10 osd.7 23 heartbeat_reset failed hb con 0x558c551b7400 for osd.6, reopening 2019-12-07T19:34:01.283+0000 7f8f1099a700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 msgr2=0x558c552b2580 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:34:01.283+0000 7f8f1099a700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.283+0000 7f8f1099a700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c54c9e800 0x558c54d03b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:01.283+0000 7f8f1099a700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c54d43400 0x558c575ff600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 msgr2=0x558c54c36b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 96 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 msgr2=0x558c54c36b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=READY pgs=9 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.283+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=READY pgs=9 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:01.283+0000 7f8f0e996700 20 osd.7 23 heartbeat_reset con 0x558c552d0400 s 0x558c552bd860 2019-12-07T19:34:01.283+0000 7f8f0e996700 10 osd.7 23 heartbeat_reset closing (old) failed hb con 0x558c552d0400 2019-12-07T19:34:01.283+0000 7f8f0d994700 20 osd.7 23 heartbeat_reset con 0x558c53ae4c00 s 0x558c552bdba0 2019-12-07T19:34:01.283+0000 7f8f0d994700 10 osd.7 23 heartbeat_reset closing (old) failed hb con 0x558c53ae4c00 2019-12-07T19:34:01.283+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c54c9e800 msgr2=0x558c54d03b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6822/36937 2019-12-07T19:34:01.283+0000 7f8f0f998700 20 osd.7 23 heartbeat_reset con 0x558c551b7000 s 0 2019-12-07T19:34:01.283+0000 7f8f0f998700 2 osd.7 23 ms_handle_refused con 0x558c54c9e800 session 0x558c552bc1a0 2019-12-07T19:34:01.283+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c54c9e800 0x558c54d03b80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:34:01.283+0000 7f8f0f998700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558c5742db00 con 0x558c5476d000 2019-12-07T19:34:01.284+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c54d43400 msgr2=0x558c575ff600 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6820/36937 2019-12-07T19:34:01.284+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c54d43400 0x558c575ff600 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:34:01.284+0000 7f8f1099a700 2 osd.7 23 ms_handle_refused con 0x558c54d43400 session 0x558c552bc1a0 2019-12-07T19:34:01.284+0000 7f8f1099a700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558c57580480 con 0x558c5476d000 2019-12-07T19:34:01.320+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.357+0000 7f8f21451700 10 osd.7 23 tick 2019-12-07T19:34:01.357+0000 7f8f21451700 10 osd.7 23 do_waiters -- start 2019-12-07T19:34:01.357+0000 7f8f21451700 10 osd.7 23 do_waiters -- finish 2019-12-07T19:34:01.357+0000 7f8f21451700 20 osd.7 23 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:01.370+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.420+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.460+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:01.470+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.484+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c54c9e800 msgr2=0x558c54d03b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6822/36937 2019-12-07T19:34:01.484+0000 7f8f0f998700 2 osd.7 23 ms_handle_refused con 0x558c54c9e800 session 0x558c552bc1a0 2019-12-07T19:34:01.484+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c54c9e800 0x558c54d03b80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.400000 2019-12-07T19:34:01.484+0000 7f8f0f998700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558c575c6900 con 0x558c5476d000 2019-12-07T19:34:01.485+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c54d43400 msgr2=0x558c575ff600 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6820/36937 2019-12-07T19:34:01.485+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c54d43400 0x558c575ff600 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.400000 2019-12-07T19:34:01.485+0000 7f8f1099a700 2 osd.7 23 ms_handle_refused con 0x558c54d43400 session 0x558c552bc1a0 2019-12-07T19:34:01.485+0000 7f8f1099a700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558c575c66c0 con 0x558c5476d000 2019-12-07T19:34:01.520+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.570+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 10 osd.7 23 tick_without_osd_lock 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fd9d000/0x40000000/0x1680000000, data 0x23aaae/0x261000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 25 osd.7 23 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073421+0000 last_rx_front 2019-12-07T19:34:01.073431+0000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 25 osd.7 23 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073353+0000 last_rx_front 2019-12-07T19:34:01.073454+0000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 25 osd.7 23 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073398+0000 last_rx_front 2019-12-07T19:34:01.073389+0000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 25 osd.7 23 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073408+0000 last_rx_front 2019-12-07T19:34:01.073380+0000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 25 osd.7 23 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073263+0000 last_rx_front 2019-12-07T19:34:01.073309+0000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 25 osd.7 23 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073229+0000 last_rx_front 2019-12-07T19:34:01.073286+0000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 25 osd.7 23 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073339+0000 last_rx_front 2019-12-07T19:34:01.073365+0000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 20 osd.7 23 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 10 osd.7 23 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 20 osd.7 23 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:01.609+0000 7f8f1fbbe700 10 osd.7 23 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:01.620+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258835 2019-12-07T19:34:01.620+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 211079 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.640+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 15 ==== osd_map(23..24 src has 1..24) v4 ==== 1589+0+0 (crc 0 0 0) 0x558c575c66c0 con 0x558c5476d000 2019-12-07T19:34:01.640+0000 7f8f164bb700 20 osd.7 23 OSD::ms_dispatch: osd_map(23..24 src has 1..24) v4 2019-12-07T19:34:01.640+0000 7f8f164bb700 10 osd.7 23 do_waiters -- start 2019-12-07T19:34:01.640+0000 7f8f164bb700 10 osd.7 23 do_waiters -- finish 2019-12-07T19:34:01.640+0000 7f8f164bb700 20 osd.7 23 _dispatch 0x558c575c66c0 osd_map(23..24 src has 1..24) v4 2019-12-07T19:34:01.640+0000 7f8f164bb700 3 osd.7 23 handle_osd_map epochs [23,24], i have 23, src has [1,24] 2019-12-07T19:34:01.640+0000 7f8f164bb700 10 osd.7 23 handle_osd_map got inc map for epoch 24 2019-12-07T19:34:01.640+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3de0e826:::osdmap.23:0# 0x0~0 2019-12-07T19:34:01.640+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1444 size 0x1444 (5188) 2019-12-07T19:34:01.640+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:01.640+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55264770 blob([0x150000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c552642a0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:01.640+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552659d0 blob([!~1000,0x151000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,444]) SharedBlob(0x558c552647e0 sbid 0x0)) need 0x1000~444 cache has 0x[1000~444] 2019-12-07T19:34:01.640+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 = 5188 2019-12-07T19:34:01.640+0000 7f8f164bb700 10 osd.7 23 add_map_bl 23 5188 bytes 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 osd.7 23 got_full_map 24, nothing requested 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 osd.7 23 handle_osd_map pg_num_history pg_num_history(e24 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {24={}} 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 osd.7 23 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e24 [1,24] lci=[13,24]) 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54d20900 seq 15 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:ce7b972f:::inc_osdmap.24:0# key 0x7f7fffffffffffffffce7b97'/!inc_osdmap.24!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1104 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~398 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc68 on front/back, now 0x0~1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20900 1 blobs 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x37000~1000/1000,1000,0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x37000~1000] 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265ce0 blob([0x37000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c230 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c55265ce0) get_ref 0x0~398 Blob(0x558c55265ce0 blob([0x37000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c230 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c55265ce0) get_ref init 0x1000, 1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~398: 0x0~398 Blob(0x558c55265ce0 blob([0x37000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x398) SharedBlob(0x558c54d3c230 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54d3c248 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x398 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d508) dirty_range mark inline shard dirty 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 = 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:32e0e826:::osdmap.24:0# key 0x7f7fffffffffffffff32e0e8'&!osdmap.24!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:32e0e826:::osdmap.24:0# 0x0~1428 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1105 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:32e0e826:::osdmap.24:0# 0x0~1428 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~428 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbd8 on front/back, now 0x0~1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20900 2 blobs 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x38000~2000/1000,2000,0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x38000~2000] 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55265340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265730 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265340 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c55265730) get_ref 0x0~1000 Blob(0x558c55265730 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265340 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c55265730) get_ref init 0x1000, 1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55265730 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265340 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55265358 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55279880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55279880 blob([!~1000,0x39000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677a40 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c55279880) get_ref 0x1000~428 Blob(0x558c55279880 blob([!~1000,0x39000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c54677a40 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c55279880) get_ref init 0x2000, 1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~428: 0x1000~428 Blob(0x558c55279880 blob([!~1000,0x39000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558c54677a40 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54677a58 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1428 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d788) dirty_range mark inline shard dirty 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:32e0e826:::osdmap.24:0# 0x0~1428 = 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20900 0 blobs 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20900 0 blobs 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20900 cost 23561 (4 ios * 4000 + 7561 bytes) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20900 onodes 0x558c53a91900,0x558c54d2d400,0x558c54d2d680,0x558c55276c80 shared_blobs 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d508) update #-1:ce7b972f:::inc_osdmap.24:0# 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d508) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:ce7b972f:::inc_osdmap.24:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d788) update #-1:32e0e826:::osdmap.24:0# 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2d788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:32e0e826:::osdmap.24:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20900 allocated 0x[37000~3000] released 0x[] 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 freelist allocate 0x37000~3000 2019-12-07T19:34:01.641+0000 7f8f164bb700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 prepare 2019-12-07T19:34:01.641+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d20900 2019-12-07T19:34:01.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 aio_wait 2019-12-07T19:34:01.641+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20900 2019-12-07T19:34:01.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 io_done 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2d400 had 1 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2d680 had 1 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860529152 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998711) bluestore 86 GiB free (0.959974), bluefs_ratio 0.039952 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40b1e~226c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40b1e 0x40b1e~226c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x10b1e 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb1e 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd8a and padding block with 0x276 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x42d8a 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000137960s (0.000005143s flush + 0.000132817s kv commit) 2019-12-07T19:34:01.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20900> 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 kv_submitted 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20900 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54d20900 osr 0x558c54ea4b40 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 12 0x1b000~1000 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7 23 _committed_osd_maps 24..24 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7 23 advance to epoch 24 (<= last 24 <= newest_map 24) 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 13 0x1b000~1000 crc a6a9dc77 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 12 0x2000~1000 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 13 0x2000~1000 crc 2619c538 2019-12-07T19:34:01.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.641+0000 7f8f141a1700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] mark_down_addrs [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- 0x558c54c19400 2019-12-07T19:34:01.641+0000 7f8f141a1700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558c54c19400 0x558c54c36b00 unknown :-1 s=STANDBY pgs=9 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:34:01.641+0000 7f8f141a1700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c54c9e800 msgr2=0x558c54d03b80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2019-12-07T19:34:01.641+0000 7f8f1199c700 2 osd.7 23 ms_handle_reset con 0x558c54c19400 session 0x558c54c5c1a0 2019-12-07T19:34:01.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] reap_dead start 2019-12-07T19:34:01.641+0000 7f8f141a1700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558c54c9e800 0x558c54d03b80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.641+0000 7f8f141a1700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c54d43400 msgr2=0x558c575ff600 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2019-12-07T19:34:01.641+0000 7f8f141a1700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558c54d43400 0x558c575ff600 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.641+0000 7f8f141a1700 7 osd.7 24 consume_map version 24 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 identify_splits_and_merges 1.0s3 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 identify_splits_and_merges 1.9s4 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 identify_splits_and_merges 1.3s4 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 identify_splits_and_merges 1.4s0 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 identify_splits_and_merges 1.ds2 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 identify_splits_and_merges 1.6s2 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 identify_splits_and_merges 1.7s4 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 24 prune_pg_created 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:0.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:1.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:2.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:3.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:4.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:5.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:6.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7:7.consume_map 24 (was 23) 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:01.641+0000 7f8f141a1700 10 osd.7 24 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:01.641+0000 7f8f141a1700 15 osd.7 24 enqueue_peering_evt 1.0s3 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.641+0000 7f8f141a1700 15 osd.7 24 enqueue_peering_evt 1.9s4 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.641+0000 7f8f141a1700 15 osd.7 24 enqueue_peering_evt 1.3s4 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558c5500d000 2019-12-07T19:34:01.641+0000 7f8f141a1700 15 osd.7 24 enqueue_peering_evt 1.4s0 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.641+0000 7f8f141a1700 15 osd.7 24 enqueue_peering_evt 1.ds2 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.641+0000 7f8f141a1700 15 osd.7 24 enqueue_peering_evt 1.6s2 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558c5513e000 2019-12-07T19:34:01.641+0000 7f8f141a1700 15 osd.7 24 enqueue_peering_evt 1.7s4 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 23 -> 24 2019-12-07T19:34:01.641+0000 7f8f141a1700 7 osd.7 24 activate_map version 24 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 23 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.641+0000 7f8f0918b700 20 osd.7 24 agent_entry empty queue 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558c54d5f000 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558c5500a000 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558c55141000 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 23 -> 24 2019-12-07T19:34:01.641+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 23 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] handle_advance_map [1,2147483647,4,0,7,2]/[1,2147483647,4,0,7,2] -- 1/1 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 23 -> 24 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 23 -> 24 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] new interval newup [1,2147483647,4,0,7,2] newacting [1,2147483647,4,0,7,2] 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 23 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 23 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,2147483647]/[1,4,5,0,7,2147483647] -- 1/1 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.641+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.642991+0000 15 0.000131 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:01.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,2147483647] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.641+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.643005+0000 0 0.000000 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.641+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.641+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.643014+0000 0 0.000000 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.641+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.643013+0000 15 0.000164 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558c548ab000 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.641+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.641+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.643027+0000 0 0.000000 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.641+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.643037+0000 0 0.000000 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] new interval newup [1,2147483647,4,0,7,2] newacting [1,2147483647,4,0,7,2] 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.641+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 23 -> 24 2019-12-07T19:34:01.641+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=4 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 23 -> 24 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 23 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:01.641+0000 7f8f05183700 15 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 23 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,2147483647]/[4,1,2,5,7,2147483647] -- 4/4 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active advmap 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.641+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [1,6,4,0,7,2](1) acting [1,6,4,0,7,2](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [1,6,4,0,7,2](1) acting [1,6,4,0,7,2](1) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,2147483647] newacting [4,1,2,5,7,2147483647] 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,2147483647] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] agent_stop 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.641+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.643105+0000 16 0.000175 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.641+0000 7f8f08189700 15 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.641+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.643116+0000 0 0.000000 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:01.641+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.643126+0000 0 0.000000 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.054714444s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.054714444s (mnow 25.078567276s + 10.976147168s) 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] Active: kicking snap trim 2019-12-07T19:34:01.641+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.641+0000 7f8f0297e700 7 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [1,4,5,0,7,6](1) acting [1,4,5,0,7,6](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [1,4,5,0,7,6](1) acting [1,4,5,0,7,6](1) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.054714444s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=24 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.641+0000 7f8f0297e700 15 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] publish_stats_to_osd 23:22 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4))) 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=24 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.641+0000 7f8f05183700 1 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.054714444s@ mbc={} ps=[2~1]] start_peering_interval up [1,6,4,0,7,2] -> [1,2147483647,4,0,7,2], acting [1,6,4,0,7,2] -> [1,2147483647,4,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=24 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.641+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.641+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=24 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,2147483647] newacting [4,1,2,5,7,2147483647] 2019-12-07T19:34:01.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:01.641+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=24 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.641+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:01.641+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=4 lpr=24 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.053267305s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.053267305s (mnow 25.078623339s + 10.974643966s) 2019-12-07T19:34:01.641+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.641+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f0317f700 15 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.053267305s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.642+0000 7f8f08189700 1 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.053267305s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,6] -> [1,4,5,0,7,2147483647], acting [1,4,5,0,7,6] -> [1,4,5,0,7,2147483647], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [4,1,2,5,7,6](4) acting [4,1,2,5,7,6](4)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [4,1,2,5,7,6](4) acting [4,1,2,5,7,6](4) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.054714444s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558c54d5c000 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active pruub 36.050572280s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.050572280s (mnow 25.078691494s + 10.971880786s) 2019-12-07T19:34:01.642+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active pruub 36.050572280s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 23 -> 24 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.642+0000 7f8f0317f700 1 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'10 active pruub 36.050572280s@ mbc={} ps=[2~1]] start_peering_interval up [4,1,2,5,7,6] -> [4,1,2,5,7,2147483647], acting [4,1,2,5,7,6] -> [4,1,2,5,7,2147483647], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 23 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] handle_advance_map [3,2147483647,7,1,2,5]/[3,2147483647,7,1,2,5] -- 3/3 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.976147168s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] new interval newup [3,2147483647,7,1,2,5] newacting [3,2147483647,7,1,2,5] 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.643287+0000 1 0.000292 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.643289+0000 15 0.000172 2019-12-07T19:34:01.642+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.643296+0000 0 0.000000 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.642+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.643302+0000 0 0.000000 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f05183700 1 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.642+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.643309+0000 0 0.000000 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f05183700 5 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=24 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.053267305s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown pruub 36.050572280s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=24 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=24 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.642+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:01.642+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=24 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] new interval newup [3,2147483647,7,1,2,5] newacting [3,2147483647,7,1,2,5] 2019-12-07T19:34:01.642+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=24 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=2 lpr=24 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.642+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 24 send_incremental_map 23 -> 24 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:01.642+0000 7f8f0197c700 15 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.642+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~0 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971880786s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~398 size 0x398 (920) 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265ce0 blob([0x37000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x398) SharedBlob(0x558c54d3c230 sbid 0x0)) need 0x0~398 cache has 0x[0~398] 2019-12-07T19:34:01.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 = 920 2019-12-07T19:34:01.642+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.643357+0000 1 0.000262 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 24 add_map_inc_bl 24 920 bytes 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [3,6,7,1,2,5](3) acting [3,6,7,1,2,5](3)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [3,6,7,1,2,5](3) acting [3,6,7,1,2,5](3) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.642+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.642+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(24..24 src has 1..24) v4 -- 0x558c573778c0 con 0x558c54c64000 2019-12-07T19:34:01.642+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974643966s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=1(3),2(4),3(0),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2))) 2019-12-07T19:34:01.642+0000 7f8f0317f700 1 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.642+0000 7f8f05183700 10 osd.7 24 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 23 -> 24 (shared) 2019-12-07T19:34:01.642+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.643376+0000 0 0.000000 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.642+0000 7f8f0317f700 5 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c54ea3ce0] 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.642+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.643381+0000 1 0.000345 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active pruub 36.053997442s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.053997442s (mnow 25.078825103s + 10.975172339s) 2019-12-07T19:34:01.642+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:24 sent:24 1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) 4->0 ([18,23] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e24/24) v1 -- 0x558c54e9aa00 con 0x558c54c64000 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active pruub 36.053997442s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.642+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59200 seq 23 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.642+0000 7f8f0197c700 1 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 luod=0'0 crt=23'13 active pruub 36.053997442s@ mbc={} ps=[2~1]] start_peering_interval up [3,6,7,1,2,5] -> [3,2147483647,7,1,2,5], acting [3,6,7,1,2,5] -> [3,2147483647,7,1,2,5], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 2 -> 2, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 24 send_incremental_map 23 -> 24 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:01.642+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:01.642+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(24..24 src has 1..24) v4 -- 0x558c5742d440 con 0x558c54c5b400 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] release_backoffs [1:60000000::::0,1:70000000::::head) 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 osd.7 24 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 23 -> 24 (shared) 2019-12-07T19:34:01.642+0000 7f8f08189700 1 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:01.642+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:24 sent:24 1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) 4->0 ([18,23] all_participants=1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4)))) e24/24) v1 -- 0x558c54e98e00 con 0x558c54c5b400 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.642+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.643422+0000 0 0.000000 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:01.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:01.642+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740a600 seq 21 2019-12-07T19:34:01.642+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:01.642+0000 7f8f08189700 5 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.642+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:01.642+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes shared_blobs 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:01.642+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown pruub 36.053997442s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.642+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x42d8a~50c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x42d8a 0x42d8a~50c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x12d8a 2019-12-07T19:34:01.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd8a 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x296 and padding block with 0xd6a 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.642+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:24 sent:24 1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) 4->0 ([18,23] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e24/24) v1 -- 0x558c54e99500 con 0x558c54c64000 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975172339s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d20600 seq 14 2019-12-07T19:34:01.642+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:01.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.643526+0000 1 0.000212 2019-12-07T19:34:01.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:01.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:01.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes shared_blobs 2019-12-07T19:34:01.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x43296 2019-12-07T19:34:01.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f0197c700 1 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000113312s (0.000000096s flush + 0.000113216s kv commit) 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.642+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.643569+0000 0 0.000000 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:01.642+0000 7f8f0197c700 5 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2538 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2561 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2561 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x43296~a01 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x43296 0x43296~a01 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x13296 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x296 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc97 and padding block with 0x369 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 24 send_incremental_map 23 -> 24 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:01.642+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(24..24 src has 1..24) v4 -- 0x558c5742c480 con 0x558c54c5bc00 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 osd.7 24 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 23 -> 24 (shared) 2019-12-07T19:34:01.642+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:24 sent:24 1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) 2->0 ([18,23] all_participants=1(3),2(4),3(0),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e24/24) v1 -- 0x558c54e9b800 con 0x558c54c5bc00 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740b200 seq 27 2019-12-07T19:34:01.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x43c97 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b200 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b200 onodes shared_blobs 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000123104s (0.000000088s flush + 0.000123016s kv commit) 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b200 allocated 0x[] released 0x[] 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a600,0x558c54d20600> 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 prepare 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b200 2019-12-07T19:34:01.642+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:01.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 io_done 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:01.642+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.642+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x43c97~50c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x43c97 0x43c97~50c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x13c97 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc97 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1a3 and padding block with 0xe5d 2019-12-07T19:34:01.642+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x441a3 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000130045s (0.000000077s flush + 0.000129968s kv commit) 2019-12-07T19:34:01.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b200> 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 kv_submitted 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b200 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 finishing 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b200 onodes 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b200 done 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b200 [] 2019-12-07T19:34:01.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.642+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.644+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 39 ==== pg_query2(1.6s2 1.6s2 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c54ce2000 con 0x558c54c5bc00 2019-12-07T19:34:01.644+0000 7f8f23c56700 15 osd.7 24 enqueue_peering_evt 1.6s2 epoch_sent: 24 epoch_requested: 24 MQuery 1.6s2 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.644+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.6s2 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.644+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 40 ==== osd_map(24..24 src has 1..24) v4 ==== 964+0+0 (unknown 0 0 0) 0x558c5742c480 con 0x558c54c5bc00 2019-12-07T19:34:01.644+0000 7f8f1199c700 20 osd.7 24 OSD::ms_dispatch: osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.644+0000 7f8f1199c700 10 osd.7 24 do_waiters -- start 2019-12-07T19:34:01.644+0000 7f8f1199c700 10 osd.7 24 do_waiters -- finish 2019-12-07T19:34:01.644+0000 7f8f1199c700 20 osd.7 24 _dispatch 0x558c5742c480 osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.644+0000 7f8f1199c700 3 osd.7 24 handle_osd_map epochs [24,24], i have 24, src has [1,24] 2019-12-07T19:34:01.644+0000 7f8f1199c700 10 osd.7 24 no new maps here, dropping 2019-12-07T19:34:01.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.6s2 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.6s2 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558c54d5c000 2019-12-07T19:34:01.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975036316s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.6s2 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.644+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:01.644+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:24 sent:24 1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) 2->0 ([18,23] all_participants=1(3),2(4),3(0),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e24/24) v1 -- 0x558c54e98700 con 0x558c54c5bc00 2019-12-07T19:34:01.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 33 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c54e45400 con 0x558c54c5b400 2019-12-07T19:34:01.644+0000 7f8f24457700 15 osd.7 24 enqueue_peering_evt 1.3s4 epoch_sent: 24 epoch_requested: 24 MQuery 1.3s4 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.644+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.3s4 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.3s4 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.3s4 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558c5513e000 2019-12-07T19:34:01.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971778990s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.3s4 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.644+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:01.644+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:24 sent:24 1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) 4->0 ([18,23] all_participants=1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4)))) e24/24) v1 -- 0x558c574e7800 con 0x558c54c5b400 2019-12-07T19:34:01.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.644+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 34 ==== osd_map(24..24 src has 1..24) v4 ==== 964+0+0 (unknown 0 0 0) 0x558c5742d440 con 0x558c54c5b400 2019-12-07T19:34:01.644+0000 7f8f1199c700 20 osd.7 24 OSD::ms_dispatch: osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.644+0000 7f8f1199c700 10 osd.7 24 do_waiters -- start 2019-12-07T19:34:01.644+0000 7f8f1199c700 10 osd.7 24 do_waiters -- finish 2019-12-07T19:34:01.644+0000 7f8f1199c700 20 osd.7 24 _dispatch 0x558c5742d440 osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.644+0000 7f8f1199c700 3 osd.7 24 handle_osd_map epochs [24,24], i have 24, src has [1,24] 2019-12-07T19:34:01.644+0000 7f8f1199c700 10 osd.7 24 no new maps here, dropping 2019-12-07T19:34:01.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 62 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54e45400 con 0x558c54c64000 2019-12-07T19:34:01.649+0000 7f8f24457700 15 osd.7 24 enqueue_peering_evt 1.9s4 epoch_sent: 24 epoch_requested: 24 MQuery 1.9s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.649+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 63 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54e45400 con 0x558c54c64000 2019-12-07T19:34:01.649+0000 7f8f24457700 15 osd.7 24 enqueue_peering_evt 1.7s4 epoch_sent: 24 epoch_requested: 24 MQuery 1.7s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.649+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558c54d5f000 2019-12-07T19:34:01.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558c5500a000 2019-12-07T19:34:01.649+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 64 ==== osd_map(24..24 src has 1..24) v4 ==== 964+0+0 (crc 0 0 0) 0x558c573778c0 con 0x558c54c64000 2019-12-07T19:34:01.649+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.975993832s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.7s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.649+0000 7f8f1199c700 20 osd.7 24 OSD::ms_dispatch: osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.649+0000 7f8f1199c700 10 osd.7 24 do_waiters -- start 2019-12-07T19:34:01.649+0000 7f8f1199c700 10 osd.7 24 do_waiters -- finish 2019-12-07T19:34:01.649+0000 7f8f1199c700 20 osd.7 24 _dispatch 0x558c573778c0 osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.649+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.974454636s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.9s4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.649+0000 7f8f1199c700 3 osd.7 24 handle_osd_map epochs [24,24], i have 24, src has [1,24] 2019-12-07T19:34:01.649+0000 7f8f1199c700 10 osd.7 24 no new maps here, dropping 2019-12-07T19:34:01.649+0000 7f8f05183700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.649+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.649+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:01.649+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:01.649+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:24 sent:24 1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) 4->0 ([18,23] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e24/24) v1 -- 0x558c54e9a300 con 0x558c54c64000 2019-12-07T19:34:01.649+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:24 sent:24 1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) 4->0 ([18,23] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e24/24) v1 -- 0x558c548a0000 con 0x558c54c64000 2019-12-07T19:34:01.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 65 ==== MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=95, reqid=client.4540.0:200, at_version=24'6, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 3135+0+0 (crc 0 0 0) 0x558c54895800 con 0x558c54c64000 2019-12-07T19:34:01.654+0000 7f8f24457700 15 osd.7 24 enqueue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000032 epoch 24 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=95, reqid=client.4540.0:200, at_version=24'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:34:01.654+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=95, reqid=client.4540.0:200, at_version=24'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.654+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=95, reqid=client.4540.0:200, at_version=24'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) queued 2019-12-07T19:34:01.654+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.654+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=95, reqid=client.4540.0:200, at_version=24'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) pg 0x558c5500d000 2019-12-07T19:34:01.654+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=95, reqid=client.4540.0:200, at_version=24'6, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] 2019-12-07T19:34:01.654+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=95, reqid=client.4540.0:200, at_version=24'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:34:01.654+0000 7f8f04982700 20 snap_mapper.add_oid 1:0064fe1d:::smithi13140796-44:4 4 2019-12-07T19:34:01.654+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:01.654+0000 7f8f04982700 20 snap_mapper.get_snaps 1:0064fe1d:::smithi13140796-44:4 got.empty() 2019-12-07T19:34:01.654+0000 7f8f04982700 20 snap_mapper.set_snaps 1:0064fe1d:::smithi13140796-44:4 4 2019-12-07T19:34:01.654+0000 7f8f04982700 20 snap_mapper.set_snaps set OBJ_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:01.654+0000 7f8f04982700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:01.654+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] append_log log((0'0,21'4], crt=21'4) [24'5 (21'4) clone 1:64fe1d00:::smithi13140796-44:4 by unknown.0.0:0 2019-12-07T19:33:58.756457+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,24'6 (21'4) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:200 2019-12-07T19:34:01.653983+0000 0 ObjectCleanRegions clean_offsets: [0~737280,737280~18446744073708814335], clean_omap: 1, new_object: 0] 2019-12-07T19:34:01.654+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'5 (0'0,24'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[2~1]] add_log_entry 24'5 (21'4) clone 1:64fe1d00:::smithi13140796-44:4 by unknown.0.0:0 2019-12-07T19:33:58.756457+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:01.654+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'6 (0'0,24'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[2~1]] add_log_entry 24'6 (21'4) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:200 2019-12-07T19:34:01.653983+0000 0 ObjectCleanRegions clean_offsets: [0~737280,737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:34:01.654+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'6 (0'0,24'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[2~1]] append_log approx pg log length = 6 2019-12-07T19:34:01.654+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'6 (0'0,24'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:01.654+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:01.654+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 24'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c574ff800 seq 14 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:0064fe1d:::smithi13140796-44:4# key 0x8380000000000000010064fe1d21736d'ithi13140796-44!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) r -2 v.len 0 2019-12-07T19:34:01.654+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:4# 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1106 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.onode(0x558c57455900).flush flush done 2019-12-07T19:34:01.654+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# 0x0 2019-12-07T19:34:01.654+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:4# 0x0~2d000 -> 0x0~2d000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x0~2000: 0x0~2000 Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10241 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5757ff10 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5757ff10 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c5742b810 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b810) get_ref 0x0~2000 Blob(0x558c5742b810 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b810) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c5742b810 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c575833b0 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10242 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c575833b0 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c575833b0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c5742b8f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b8f0) get_ref 0x0~2000 Blob(0x558c5742b8f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c5742b8f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57583490 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10243 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c57583490 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c57583490 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c574f1c00 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c574f1c00) get_ref 0x0~2000 Blob(0x558c574f1c00 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c574f1c00) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c574f1c00 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c5742ba40 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10244 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5742ba40 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5742ba40 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c5742b960 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b960) get_ref 0x0~2000 Blob(0x558c5742b960 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b960) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5742b960 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c5742bb20 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10245 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5742bb20 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5742bb20 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c5742b7a0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b7a0) get_ref 0x0~2000 Blob(0x558c5742b7a0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5742b7a0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c5742bc00 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10246 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5742bc00 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5742bc00 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c5742b730 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b730) get_ref 0x0~2000 Blob(0x558c5742b730 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5742b730) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5742b730 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c5742bce0 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10247 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5742bce0 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5742bce0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c57538770 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c57538770) get_ref 0x0~2000 Blob(0x558c57538770 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c57538770) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57538770 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c5742bdc0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10248 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5742bdc0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5742bdc0 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c54ea3260 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54ea3260) get_ref 0x0~2000 Blob(0x558c54ea3260 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54ea3260) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c54ea3260 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c5742bea0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10249 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5742bea0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5742bea0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c54677490 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54677490) get_ref 0x0~2000 Blob(0x558c54677490 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54677490) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c54677490 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c5742bf80 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10250 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c5742bf80 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c5742bf80 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c54677340 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54677340) get_ref 0x0~2000 Blob(0x558c54677340 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54677340) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c54677340 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c574fa070 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10251 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa070 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa070 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c54676000 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54676000) get_ref 0x0~2000 Blob(0x558c54676000 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54676000) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c54676000 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c574fa150 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10252 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa150 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa150 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c546761c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c546761c0) get_ref 0x0~2000 Blob(0x558c546761c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c546761c0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c546761c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c574fa230 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10253 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa230 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa230 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c575bd730 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c575bd730) get_ref 0x0~2000 Blob(0x558c575bd730 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c575bd730) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c575bd730 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c574fa310 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10254 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa310 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa310 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c575bdab0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c575bdab0) get_ref 0x0~2000 Blob(0x558c575bdab0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c575bdab0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c575bdab0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c574fa3f0 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10255 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa3f0 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa3f0 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c55279570 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55279570) get_ref 0x0~2000 Blob(0x558c55279570 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55279570) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c55279570 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c574fa4d0 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10256 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa4d0 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa4d0 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c55278930 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55278930) get_ref 0x0~2000 Blob(0x558c55278930 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55278930) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c55278930 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x20000~2000: 0x0~2000 Blob(0x558c574fa5b0 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10257 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa5b0 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa5b0 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c55278e00 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55278e00) get_ref 0x0~2000 Blob(0x558c55278e00 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55278e00) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c55278e00 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x22000~2000: 0x0~2000 Blob(0x558c574fa690 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10258 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa690 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa690 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c574f1d50 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c574f1d50) get_ref 0x0~2000 Blob(0x558c574f1d50 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c574f1d50) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c574f1d50 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x24000~2000: 0x0~2000 Blob(0x558c574fa770 blob([0x1f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10259 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa770 blob([0x1f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa770 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c5523a9a0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5523a9a0) get_ref 0x0~2000 Blob(0x558c5523a9a0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c5523a9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c5523a9a0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x26000~2000: 0x0~2000 Blob(0x558c574fa850 blob([0x1f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10260 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa850 blob([0x1f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa850 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c54ea3570 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54ea3570) get_ref 0x0~2000 Blob(0x558c54ea3570 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c54ea3570) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c54ea3570 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x28000~2000: 0x0~2000 Blob(0x558c574fa930 blob([0x1f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10261 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574fa930 blob([0x1f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574fa930 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c552652d0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c552652d0) get_ref 0x0~2000 Blob(0x558c552652d0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c552652d0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c552652d0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c574faa10 blob([0x1f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10262 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574faa10 blob([0x1f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574faa10 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c55264690 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55264690) get_ref 0x0~2000 Blob(0x558c55264690 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.blob(0x558c55264690) get_ref init 0x2000, 1000 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c55264690 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup src 0x2c000~1000: 0x0~1000 Blob(0x558c574faaf0 blob([0x1fb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10263 2019-12-07T19:34:01.654+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared Blob(0x558c574faaf0 blob([0x1fb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 sbid 0x0)) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) make_blob_shared now Blob(0x558c574faaf0 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=1)))) 2019-12-07T19:34:01.654+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup new Blob(0x558c55264150 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264150) get_ref 0x0~1000 Blob(0x558c55264150 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264150) get_ref init 0x1000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dup dst 0x2c000~1000: 0x0~1000 Blob(0x558c55264150 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dirty_range mark inline shard dirty 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dirty_range mark inline shard dirty 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:4# = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# snapset 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# snapset = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# 1 keys 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# 1 keys = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# hinfo_key (42 bytes) 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.0062F78B.4.smithi13140796-44..' <- OBJ_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.3_0000000000000001.0062F78B.4.smithi13140796-44..' <- SNA_1_0000000000000004_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000024.00000000000000000005' <- 0000000024.00000000000000000005 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000024.00000000000000000006' <- 0000000024.00000000000000000006 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff800 cost 8180 (1 ios * 4000 + 4180 bytes) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff800 onodes 0x558c548f2280,0x558c54e45b80,0x558c57455900 shared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c548f2388) update #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c0371625:::snapmapper:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) update 3#1:0064fe1d:::smithi13140796-44:4# 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) update inline shard 621 bytes from 23 extents 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:4# is 1039 (412 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) update inline shard 621 bytes from 23 extents 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 1184 (557 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff800 allocated 0x[] released 0x[] 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 prepare 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff800 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 io_done 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'6 (0'0,24'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[2~1]] op_applied version 24'6 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5417 < min_flush_size 524288 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:34:01.655+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45b80 had 1 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455900 had 1 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5440 < min_flush_size 524288 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5440 < min_flush_size 524288 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x441a3~1540 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x441a3 0x441a3~1540 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x141a3 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1a3 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6e3 and padding block with 0x91d 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x456e3 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000120974s (0.000000091s flush + 0.000120883s kv commit) 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff800> 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff800 2019-12-07T19:34:01.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 finishing 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff800 onodes 0x558c548f2280,0x558c54e45b80,0x558c57455900 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 done 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:01.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff800 [] 2019-12-07T19:34:01.655+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.655+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.655+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 24/18 ECSubWriteReply(tid=95, last_complete=24'6, committed=1, applied=1)) v2 -- 0x558c575b8200 con 0x558c54c64000 2019-12-07T19:34:01.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 66 ==== MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=96, reqid=client.4540.0:201, at_version=24'7, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 2116+0+0 (crc 0 0 0) 0x558c54693000 con 0x558c54c64000 2019-12-07T19:34:01.655+0000 7f8f24457700 15 osd.7 24 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000021 epoch 24 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=96, reqid=client.4540.0:201, at_version=24'7, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:34:01.655+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=96, reqid=client.4540.0:201, at_version=24'7, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.655+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=96, reqid=client.4540.0:201, at_version=24'7, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) queued 2019-12-07T19:34:01.655+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.655+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=96, reqid=client.4540.0:201, at_version=24'7, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) pg 0x558c5500d000 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=96, reqid=client.4540.0:201, at_version=24'7, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.0s3( v 24'6 (0'0,24'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'6 (0'0,24'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=96, reqid=client.4540.0:201, at_version=24'7, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'6 (0'0,24'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] append_log log((0'0,24'6], crt=21'4) [24'7 (24'6) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:201 2019-12-07T19:34:01.653997+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 0, new_object: 0] 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'7 (0'0,24'7] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'6 crt=21'4 lcod 24'6 active mbc={} ps=[2~1]] add_log_entry 24'7 (24'6) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:201 2019-12-07T19:34:01.653997+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 0, new_object: 0 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'7 (0'0,24'7] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'6 crt=21'4 lcod 24'6 active mbc={} ps=[2~1]] append_log approx pg log length = 7 2019-12-07T19:34:01.655+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'7 (0'0,24'7] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'6 crt=21'4 lcod 24'6 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:01.655+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:01.655+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 24'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c574ffb00 seq 15 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:0064fe1d:::smithi13140796-44:head#7 key 0x8380000000000000010064fe1d21736d'ithi13140796-44!='0xfffffffffffffffe0000000000000007'o' 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) r -2 v.len 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:head#7 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:head#7 = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# -> 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1107 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.onode(0x558c54e45b80).flush flush done 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# -> 3#1:0064fe1d:::smithi13140796-44:head# 0x0~2d000 -> 0x0~2d000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x0~2000: 0x0~2000 Blob(0x558c5742b810 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c54cf9880 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c54cf9880) get_ref 0x0~2000 Blob(0x558c54cf9880 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c54cf9880) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c54cf9880 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c5742b8f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c55279420 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55279420) get_ref 0x0~2000 Blob(0x558c55279420 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55279420) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c55279420 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c574f1c00 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c55279030 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55279030) get_ref 0x0~2000 Blob(0x558c55279030 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55279030) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c55279030 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c5742b960 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c55264310 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264310) get_ref 0x0~2000 Blob(0x558c55264310 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264310) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c55264310 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c5742b7a0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c55264230 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264230) get_ref 0x0~2000 Blob(0x558c55264230 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264230) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c55264230 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c5742b730 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c55264000 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264000) get_ref 0x0~2000 Blob(0x558c55264000 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55264000) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c55264000 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57538770 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c55265260 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55265260) get_ref 0x0~2000 Blob(0x558c55265260 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c55265260) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c55265260 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c54ea3260 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566000 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566000) get_ref 0x0~2000 Blob(0x558c57566000 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566000) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57566000 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c54677490 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566070 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566070) get_ref 0x0~2000 Blob(0x558c57566070 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566070) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57566070 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x12000~2000: 0x0~2000 Blob(0x558c54677340 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c575660e0 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575660e0) get_ref 0x0~2000 Blob(0x558c575660e0 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575660e0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c575660e0 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x14000~2000: 0x0~2000 Blob(0x558c54676000 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566150 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566150) get_ref 0x0~2000 Blob(0x558c57566150 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566150) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57566150 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x16000~2000: 0x0~2000 Blob(0x558c546761c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c575661c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575661c0) get_ref 0x0~2000 Blob(0x558c575661c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575661c0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c575661c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x18000~2000: 0x0~2000 Blob(0x558c575bd730 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566230 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566230) get_ref 0x0~2000 Blob(0x558c57566230 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566230) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57566230 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c575bdab0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c575662a0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575662a0) get_ref 0x0~2000 Blob(0x558c575662a0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575662a0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c575662a0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c55279570 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566310 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566310) get_ref 0x0~2000 Blob(0x558c57566310 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566310) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57566310 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c55278930 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566380 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566380) get_ref 0x0~2000 Blob(0x558c57566380 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566380) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c57566380 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x20000~2000: 0x0~2000 Blob(0x558c55278e00 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c575663f0 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575663f0) get_ref 0x0~2000 Blob(0x558c575663f0 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575663f0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c575663f0 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x22000~2000: 0x0~2000 Blob(0x558c574f1d50 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566460 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566460) get_ref 0x0~2000 Blob(0x558c57566460 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566460) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57566460 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x24000~2000: 0x0~2000 Blob(0x558c5523a9a0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c575664d0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575664d0) get_ref 0x0~2000 Blob(0x558c575664d0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575664d0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c575664d0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x26000~2000: 0x0~2000 Blob(0x558c54ea3570 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566540 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566540) get_ref 0x0~2000 Blob(0x558c57566540 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566540) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c57566540 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x28000~2000: 0x0~2000 Blob(0x558c552652d0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c575665b0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575665b0) get_ref 0x0~2000 Blob(0x558c575665b0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c575665b0) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c575665b0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c55264690 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566620 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566620) get_ref 0x0~2000 Blob(0x558c57566620 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566620) get_ref init 0x2000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c57566620 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup src 0x2c000~1000: 0x0~1000 Blob(0x558c55264150 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup new Blob(0x558c57566690 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566690) get_ref 0x0~1000 Blob(0x558c57566690 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.blob(0x558c57566690) get_ref init 0x1000, 1000 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dup dst 0x2c000~1000: 0x0~1000 Blob(0x558c57566690 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=3)))) 2019-12-07T19:34:01.655+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) dirty_range mark inline shard dirty 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# -> 3#1:0064fe1d:::smithi13140796-44:head# = 0 2019-12-07T19:34:01.655+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:01.655+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:01.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:01.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000024.00000000000000000007' <- 0000000024.00000000000000000007 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ffb00 cost 5766 (1 ios * 4000 + 1766 bytes) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ffb00 onodes 0x558c57455900,0x558c575bb900 shared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) update 3#1:0064fe1d:::smithi13140796-44:head#7 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head#7 is 1184 (557 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update inline shard 621 bytes from 23 extents 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 1184 (557 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=3))) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ffb00 allocated 0x[] released 0x[] 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 prepare 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ffb00 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 io_done 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'7 (0'0,24'7] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'6 crt=21'4 lcod 24'6 active mbc={} ps=[2~1]] op_applied version 24'7 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:01.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3721 < min_flush_size 524288 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455900 had 1 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3744 < min_flush_size 524288 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3744 < min_flush_size 524288 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x456e3~ea0 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x456e3 0x456e3~ea0 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x156e3 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6e3 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x583 and padding block with 0xa7d 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46583 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000134120s (0.000000127s flush + 0.000133993s kv commit) 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ffb00> 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 kv_submitted 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ffb00 2019-12-07T19:34:01.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 finishing 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ffb00 onodes 0x558c57455900,0x558c575bb900 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ffb00 done 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:01.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ffb00 [] 2019-12-07T19:34:01.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.656+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 24/18 ECSubWriteReply(tid=96, last_complete=24'7, committed=1, applied=1)) v2 -- 0x558c575b8400 con 0x558c54c64000 2019-12-07T19:34:01.656+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 67 ==== MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=97, reqid=client.4540.0:202, at_version=24'8, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 2309+0+0 (crc 0 0 0) 0x558c575be000 con 0x558c54c64000 2019-12-07T19:34:01.656+0000 7f8f24457700 15 osd.7 24 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000023 epoch 24 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=97, reqid=client.4540.0:202, at_version=24'8, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:34:01.656+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=97, reqid=client.4540.0:202, at_version=24'8, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=97, reqid=client.4540.0:202, at_version=24'8, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) queued 2019-12-07T19:34:01.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=97, reqid=client.4540.0:202, at_version=24'8, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e24) pg 0x558c5500d000 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=97, reqid=client.4540.0:202, at_version=24'8, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.0s3( v 24'7 (0'0,24'7] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'7 (0'0,24'7] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=97, reqid=client.4540.0:202, at_version=24'8, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'7 (0'0,24'7] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] append_log log((0'0,24'7], crt=21'4) [24'8 (24'7) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:202 2019-12-07T19:34:01.654004+0000 0 ObjectCleanRegions clean_offsets: [0~737280,737280~18446744073708814335], clean_omap: 1, new_object: 0] 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'7 crt=21'4 lcod 24'7 active mbc={} ps=[2~1]] add_log_entry 24'8 (24'7) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:202 2019-12-07T19:34:01.654004+0000 0 ObjectCleanRegions clean_offsets: [0~737280,737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'7 crt=21'4 lcod 24'7 active mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'7 crt=21'4 lcod 24'7 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:01.656+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:01.656+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 24'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d59b00 seq 16 2019-12-07T19:34:01.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:01.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:01.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000024.00000000000000000008' <- 0000000024.00000000000000000008 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5959 (1 ios * 4000 + 1959 bytes) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes 0x558c575bb900 shared_blobs 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 1184 (557 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:01.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:01.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:01.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.656+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'7 crt=21'4 lcod 24'7 active mbc={} ps=[2~1]] op_applied version 24'8 2019-12-07T19:34:01.657+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:34:01.657+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2299 < min_flush_size 524288 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2322 < min_flush_size 524288 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2322 < min_flush_size 524288 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46583~912 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46583 0x46583~912 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x16583 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x583 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe95 and padding block with 0x16b 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46e95 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000097273s (0.000000092s flush + 0.000097181s kv commit) 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00> 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:01.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 0x558c575bb900 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:01.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:01.657+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.657+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.657+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 24/18 ECSubWriteReply(tid=97, last_complete=24'8, committed=1, applied=1)) v2 -- 0x558c57519e00 con 0x558c54c64000 2019-12-07T19:34:01.657+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.659+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 68 ==== MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=98, reqid=client.4540.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=24'8)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c575be800 con 0x558c54c64000 2019-12-07T19:34:01.659+0000 7f8f24457700 15 osd.7 24 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000027 epoch 24 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=98, reqid=client.4540.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=24'8)) v2 2019-12-07T19:34:01.659+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=98, reqid=client.4540.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=24'8)) v2) prio 127 cost 0 e24) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=98, reqid=client.4540.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=24'8)) v2) prio 127 cost 0 e24) queued 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=98, reqid=client.4540.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=24'8)) v2) prio 127 cost 0 e24) pg 0x558c5500d000 2019-12-07T19:34:01.659+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=98, reqid=client.4540.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=24'8)) v2 pg pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] 2019-12-07T19:34:01.659+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.0s3 24/18 ECSubWrite(tid=98, reqid=client.4540.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=24'8)) v2 2019-12-07T19:34:01.659+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] append_log log((0'0,24'8], crt=21'4) [] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] rollforward: entry=24'5 (21'4) clone 1:64fe1d00:::smithi13140796-44:4 by unknown.0.0:0 2019-12-07T19:33:58.756457+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] rollforward: entry=24'6 (21'4) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:200 2019-12-07T19:34:01.653983+0000 0 ObjectCleanRegions clean_offsets: [0~737280,737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] rollforward: entry=24'7 (24'6) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:201 2019-12-07T19:34:01.653997+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 0, new_object: 0 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] rollforward: entry=24'8 (24'7) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:202 2019-12-07T19:34:01.654004+0000 0 ObjectCleanRegions clean_offsets: [0~737280,737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:34:01.659+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:01.659+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:01.659+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:01.659+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:01.659+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d59800 seq 17 2019-12-07T19:34:01.659+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head#7 onode 0x558c57455900 txc 0x558c54d59800 2019-12-07T19:34:01.659+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head#7 0x0 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5757ff10) put_ref 0x0~2000 Blob(0x558c5757ff10 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c575833b0) put_ref 0x0~2000 Blob(0x558c575833b0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c57583490) put_ref 0x0~2000 Blob(0x558c57583490 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5742ba40) put_ref 0x0~2000 Blob(0x558c5742ba40 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5742bb20) put_ref 0x0~2000 Blob(0x558c5742bb20 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5742bc00) put_ref 0x0~2000 Blob(0x558c5742bc00 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5742bce0) put_ref 0x0~2000 Blob(0x558c5742bce0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5742bdc0) put_ref 0x0~2000 Blob(0x558c5742bdc0 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5742bea0) put_ref 0x0~2000 Blob(0x558c5742bea0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c5742bf80) put_ref 0x0~2000 Blob(0x558c5742bf80 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa070) put_ref 0x0~2000 Blob(0x558c574fa070 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa150) put_ref 0x0~2000 Blob(0x558c574fa150 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa230) put_ref 0x0~2000 Blob(0x558c574fa230 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa310) put_ref 0x0~2000 Blob(0x558c574fa310 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa3f0) put_ref 0x0~2000 Blob(0x558c574fa3f0 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa4d0) put_ref 0x0~2000 Blob(0x558c574fa4d0 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa5b0) put_ref 0x0~2000 Blob(0x558c574fa5b0 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa690) put_ref 0x0~2000 Blob(0x558c574fa690 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa770) put_ref 0x0~2000 Blob(0x558c574fa770 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa850) put_ref 0x0~2000 Blob(0x558c574fa850 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574fa930) put_ref 0x0~2000 Blob(0x558c574fa930 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574faa10) put_ref 0x0~2000 Blob(0x558c574faa10 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.blob(0x558c574faaf0) put_ref 0x0~1000 Blob(0x558c574faaf0 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore.extentmap(0x558c57455a08) dirty_range mark inline shard dirty 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c5757ff10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1cf000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c575833b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d1000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57583490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d3000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c5742ba40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d5000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c5742bb20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d7000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c5742bc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d9000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c5742bce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1db000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c5742bdc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1dd000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5742bea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1df000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c5742bf80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e1000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c574fa070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e3000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c574fa150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e5000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c574fa230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e7000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c574fa310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e9000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c574fa3f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1eb000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c574fa4d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1ed000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c574fa5b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1ef000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c574fa690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f1000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c574fa770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f3000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c574fa850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f5000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c574fa930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f7000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c574faa10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f9000~2000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~1000: 0x0~1000 Blob(0x558c574faaf0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=3)))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1fb000~1000] 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head#7 = 0 2019-12-07T19:34:01.659+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:01.659+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:01.659+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59800 onodes shared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2))) 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59800 allocated 0x[] released 0x[] 2019-12-07T19:34:01.659+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 prepare 2019-12-07T19:34:01.659+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59800 2019-12-07T19:34:01.659+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 io_done 2019-12-07T19:34:01.659+0000 7f8f04982700 10 osd.7 24 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:01.659+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1773 < min_flush_size 524288 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455900 had 1 2019-12-07T19:34:01.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1796 < min_flush_size 524288 2019-12-07T19:34:01.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1796 < min_flush_size 524288 2019-12-07T19:34:01.659+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46e95~704 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.659+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46e95 0x46e95~704 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x16e95 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe95 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.659+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x599 and padding block with 0xa67 2019-12-07T19:34:01.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x47599 2019-12-07T19:34:01.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:01.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:01.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:01.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000155996s (0.000000089s flush + 0.000155907s kv commit) 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:01.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59800> 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 kv_submitted 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59800 2019-12-07T19:34:01.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 finishing 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59800 onodes 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59800 done 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:01.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59800 [] 2019-12-07T19:34:01.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:01.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:01.660+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.660+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 24/18 ECSubWriteReply(tid=98, last_complete=24'8, committed=1, applied=1)) v2 -- 0x558c57568000 con 0x558c54c64000 2019-12-07T19:34:01.660+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:01.670+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.720+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 6 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 0.019588628s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b9000 con 0x558c54c18400 2019-12-07T19:34:01.752+0000 7f8f23c56700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019070210s,0.019588628s]) 2019-12-07T19:34:01.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.189269897s delta_ub 18446744073.690481406s) v5 -- 0x558c551aec00 con 0x558c54c18400 2019-12-07T19:34:01.752+0000 7f8f23c56700 10 osd.7 24 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 23 -> 24 (as per caller) 2019-12-07T19:34:01.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 6 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 0.019588628s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c18c00 2019-12-07T19:34:01.752+0000 7f8f23c56700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019016564s,0.019588628s]) 2019-12-07T19:34:01.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.189323543s delta_ub 18446744073.690535052s) v5 -- 0x558c551aec00 con 0x558c54c18c00 2019-12-07T19:34:01.771+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.821+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.870+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.921+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.952+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 10 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.410599692s delta_ub 0.022389845s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57568000 con 0x558c552d1400 2019-12-07T19:34:01.952+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 10 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.410599692s delta_ub 0.022389845s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b9000 con 0x558c54c18000 2019-12-07T19:34:01.952+0000 7f8f24457700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021827431s,0.022389845s]) 2019-12-07T19:34:01.952+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.388772261s delta_ub 18446744073.687724185s) v5 -- 0x558c57519e00 con 0x558c552d1400 2019-12-07T19:34:01.952+0000 7f8f23c56700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021780322s,0.022389845s]) 2019-12-07T19:34:01.952+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.388819370s delta_ub 18446744073.687771294s) v5 -- 0x558c551aec00 con 0x558c54c18000 2019-12-07T19:34:01.971+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.021+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.071+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.121+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.159+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.609358819s delta_ub 0.013984930s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c65400 2019-12-07T19:34:02.159+0000 7f8f24c58700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013437209s,0.013984930s]) 2019-12-07T19:34:02.159+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.595921610s delta_ub 18446744073.696114407s) v5 -- 0x558c54c58400 con 0x558c54c65400 2019-12-07T19:34:02.159+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.609358819s delta_ub 0.013984930s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c54c65000 2019-12-07T19:34:02.159+0000 7f8f24c58700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013384803s,0.013984930s]) 2019-12-07T19:34:02.159+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.595974016s delta_ub 18446744073.696166813s) v5 -- 0x558c54c58400 con 0x558c54c65000 2019-12-07T19:34:02.171+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.222+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 7 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 0.019422854s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c18c00 2019-12-07T19:34:02.253+0000 7f8f23c56700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019091235s,0.019422854s]) 2019-12-07T19:34:02.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.689545969s delta_ub 18446744073.690460381s) v5 -- 0x558c551aec00 con 0x558c54c18c00 2019-12-07T19:34:02.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 7 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 0.019422854s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b9000 con 0x558c54c18400 2019-12-07T19:34:02.253+0000 7f8f23c56700 20 osd.7 24 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019050605s,0.019422854s]) 2019-12-07T19:34:02.253+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.689586599s delta_ub 18446744073.690501011s) v5 -- 0x558c551aec00 con 0x558c54c18400 2019-12-07T19:34:02.272+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.322+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.331+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:02.371+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.401+0000 7f8f21451700 10 osd.7 24 tick 2019-12-07T19:34:02.401+0000 7f8f21451700 10 osd.7 24 do_waiters -- start 2019-12-07T19:34:02.401+0000 7f8f21451700 10 osd.7 24 do_waiters -- finish 2019-12-07T19:34:02.401+0000 7f8f21451700 20 osd.7 24 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:02.422+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.432+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:02.472+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.522+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.572+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 10 osd.7 24 tick_without_osd_lock 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fd9a000/0x40000000/0x1680000000, data 0x26926e/0x264000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 25 osd.7 24 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073421+0000 last_rx_front 2019-12-07T19:34:01.073431+0000 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 25 osd.7 24 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073353+0000 last_rx_front 2019-12-07T19:34:01.073454+0000 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 25 osd.7 24 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073398+0000 last_rx_front 2019-12-07T19:34:01.073389+0000 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 25 osd.7 24 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073408+0000 last_rx_front 2019-12-07T19:34:01.073380+0000 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 25 osd.7 24 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073263+0000 last_rx_front 2019-12-07T19:34:01.073309+0000 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 25 osd.7 24 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:01.073004+0000 last_rx_back 2019-12-07T19:34:01.073229+0000 last_rx_front 2019-12-07T19:34:01.073286+0000 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 20 osd.7 24 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 10 osd.7 24 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 20 osd.7 24 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:02.611+0000 7f8f1fbbe700 10 osd.7 24 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:02.622+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 224515 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.646+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c548d2000 0x558c574b9180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:02.646+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c548d2000 0x558c574b9180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:02.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 35 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c54e9aa00 con 0x558c54c5b400 2019-12-07T19:34:02.646+0000 7f8f24457700 15 osd.7 24 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.646+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 36 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558c57556900 con 0x558c54c5b400 2019-12-07T19:34:02.646+0000 7f8f1199c700 20 osd.7 24 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.646+0000 7f8f1199c700 10 osd.7 24 do_waiters -- start 2019-12-07T19:34:02.646+0000 7f8f1199c700 10 osd.7 24 do_waiters -- finish 2019-12-07T19:34:02.646+0000 7f8f1199c700 20 osd.7 24 _dispatch 0x558c57556900 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.646+0000 7f8f1199c700 3 osd.7 24 handle_osd_map epochs [25,25], i have 24, src has [1,25] 2019-12-07T19:34:02.646+0000 7f8f1199c700 10 osd.7 24 handle_osd_map got inc map for epoch 25 2019-12-07T19:34:02.646+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:32e0e826:::osdmap.24:0# 0x0~0 2019-12-07T19:34:02.646+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1428 size 0x1428 (5160) 2019-12-07T19:34:02.646+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:02.646+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265730 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265340 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:02.646+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279880 blob([!~1000,0x39000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558c54677a40 sbid 0x0)) need 0x1000~428 cache has 0x[1000~428] 2019-12-07T19:34:02.646+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:32e0e826:::osdmap.24:0# 0x0~1428 = 5160 2019-12-07T19:34:02.646+0000 7f8f1199c700 10 osd.7 24 add_map_bl 24 5160 bytes 2019-12-07T19:34:02.646+0000 7f8f1199c700 20 osd.7 24 got_full_map 25, nothing requested 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 osd.7 24 handle_osd_map pg_num_history pg_num_history(e25 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {25={}} 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 osd.7 24 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e25 [1,25] lci=[13,25]) 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54ea1b00 seq 16 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c0fb972f:::inc_osdmap.25:0# key 0x7f7fffffffffffffffc0fb97'/!inc_osdmap.25!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1108 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~156 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xeaa on front/back, now 0x0~1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1b00 1 blobs 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3a000~1000/1000,1000,0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3a000~1000] 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bd810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1ce0 blob([0x3a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd810 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c574f1ce0) get_ref 0x0~156 Blob(0x558c574f1ce0 blob([0x3a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd810 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c574f1ce0) get_ref init 0x1000, 1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~156: 0x0~156 Blob(0x558c574f1ce0 blob([0x3a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x156) SharedBlob(0x558c575bd810 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c575bd828 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x156 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149c88) dirty_range mark inline shard dirty 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 = 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3820e826:::osdmap.25:0# key 0x7f7fffffffffffffff3820e8'&!osdmap.25!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3820e826:::osdmap.25:0# 0x0~1428 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1109 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3820e826:::osdmap.25:0# 0x0~1428 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~428 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbd8 on front/back, now 0x0~1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1b00 2 blobs 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x267000~2000/1000,2000,0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x267000~2000] 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757fab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757fab0 blob([0x267000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c5757fab0) get_ref 0x0~1000 Blob(0x558c5757fab0 blob([0x267000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c5757fab0) get_ref init 0x1000, 1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5757fab0 blob([0x267000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5757ee70 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5757ee88 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757efc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757efc0 blob([!~1000,0x268000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f420 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c5757efc0) get_ref 0x1000~428 Blob(0x558c5757efc0 blob([!~1000,0x268000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c5757f420 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c5757efc0) get_ref init 0x2000, 1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~428: 0x1000~428 Blob(0x558c5757efc0 blob([!~1000,0x268000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558c5757f420 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5757f438 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1428 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e108) dirty_range mark inline shard dirty 2019-12-07T19:34:02.647+0000 7f8f24457700 10 osd.7 24 ms_handle_authentication new session 0x558c54c5d520 con 0x558c548d2000 entity osd.5 addr 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3820e826:::osdmap.25:0# 0x0~1428 = 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:02.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:02.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:02.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 map 25 > 24 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:02.647+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 25, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:02.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7f8f24457700 10 osd.7 24 ms_handle_authentication session 0x558c54c5d520 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1b00 0 blobs 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1b00 0 blobs 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 22983 (4 ios * 4000 + 6983 bytes) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes 0x558c53a91900,0x558c55149b80,0x558c55276c80,0x558c5756e000 shared_blobs 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149c88) update #-1:c0fb972f:::inc_osdmap.25:0# 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55149c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c0fb972f:::inc_osdmap.25:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e108) update #-1:3820e826:::osdmap.25:0# 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e108) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3820e826:::osdmap.25:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[3a000~1000,267000~2000] released 0x[] 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 freelist allocate 0x3a000~1000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 freelist allocate 0x267000~2000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea1b00 2019-12-07T19:34:02.647+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c548d2000 0x558c574b9180 crc :-1 s=SESSION_ACCEPTING pgs=11 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558c54c19c00 2019-12-07T19:34:02.647+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c548d2000 0x558c574b9180 crc :-1 s=SESSION_ACCEPTING pgs=11 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:34:02.647+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=READY pgs=11 cs=1 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=e020c6588efc571e server_cookie=4bffdf0ee71c7be4 in_seq=10 out_seq=12 2019-12-07T19:34:02.647+0000 7f8f1199c700 2 osd.7 24 ms_handle_reset con 0x558c548d2000 session 0x558c54c5d520 2019-12-07T19:34:02.647+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 aio_wait 2019-12-07T19:34:02.647+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:34:02.647+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2663 < min_flush_size 524288 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8829 < min_flush_size 524288 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55149b80 had 1 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756e000 had 1 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860529152 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998711) bluestore 86 GiB free (0.959973), bluefs_ratio 0.039952 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8852 < min_flush_size 524288 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8852 < min_flush_size 524288 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x47599~2294 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x47599 0x47599~2294 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x17599 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x599 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x82d and padding block with 0x7d3 2019-12-07T19:34:02.647+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 69 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54e45400 con 0x558c54c64000 2019-12-07T19:34:02.647+0000 7f8f24457700 15 osd.7 24 enqueue_peering_evt 1.9s4 epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.647+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.647+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 70 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (crc 0 0 0) 0x558c57556b40 con 0x558c54c64000 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 osd.7 24 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 osd.7 24 do_waiters -- start 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 osd.7 24 do_waiters -- finish 2019-12-07T19:34:02.647+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 11 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1231+0+0 (unknown 0 0 0) 0x558c54e9aa00 con 0x558c54c19c00 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 osd.7 24 _dispatch 0x558c57556b40 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.647+0000 7f8f1199c700 3 osd.7 24 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 osd.7 24 no new maps here, dropping 2019-12-07T19:34:02.647+0000 7f8f24457700 15 osd.7 24 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.647+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.647+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.647+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558c5500a000 2019-12-07T19:34:02.647+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 25, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.647+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 37 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c54e45400 con 0x558c54c5b400 2019-12-07T19:34:02.647+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 12 ==== osd_map(24..25 src has 1..25) v4 ==== 1314+0+0 (unknown 0 0 0) 0x558c57556d80 con 0x558c54c19c00 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 osd.7 24 OSD::ms_dispatch: osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 osd.7 24 do_waiters -- start 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 osd.7 24 do_waiters -- finish 2019-12-07T19:34:02.647+0000 7f8f1199c700 20 osd.7 24 _dispatch 0x558c57556d80 osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.647+0000 7f8f24457700 15 osd.7 24 enqueue_peering_evt 1.bs3 epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.647+0000 7f8f1199c700 3 osd.7 24 handle_osd_map epochs [24,25], i have 25, src has [1,25] 2019-12-07T19:34:02.647+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.647+0000 7f8f1199c700 10 osd.7 24 no new maps here, dropping 2019-12-07T19:34:02.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {25=0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)>} 2019-12-07T19:34:02.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.647+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {25=0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)>} 2019-12-07T19:34:02.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.647+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 map 25 > 24 2019-12-07T19:34:02.647+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7f8f0297e700 20 osd.7 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 25, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.647+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 map 25 > 24 2019-12-07T19:34:02.647+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 25, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.647+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4982d 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:02.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000005 2019-12-07T19:34:02.648+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:02.648+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000448684s (0.000059367s flush + 0.000389317s kv commit) 2019-12-07T19:34:02.648+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1b00> 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.648+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54ea1b00 osr 0x558c54ea4b40 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 13 0x1b000~1000 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 14 0x1b000~1000 crc bc76708a 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7 24 _committed_osd_maps 25..25 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 13 0x2000~1000 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7 24 advance to epoch 25 (<= last 25 <= newest_map 25) 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 14 0x2000~1000 crc 9d22d105 2019-12-07T19:34:02.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:02.648+0000 7f8f141a1700 7 osd.7 25 consume_map version 25 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 identify_splits_and_merges 1.0s3 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 identify_splits_and_merges 1.9s4 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs3 has no pg and waiting_for_split 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 identify_splits_and_merges 1.3s4 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:4.identify_splits_and_merges identify_splits_and_merges slot 1.cs0 has no pg and waiting_for_split 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 identify_splits_and_merges 1.4s0 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 identify_splits_and_merges 1.ds2 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 identify_splits_and_merges 1.6s2 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 identify_splits_and_merges 1.7s4 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7 25 prune_pg_created 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:0.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:1.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 25 <= 25, requeueing 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {25=} 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:2.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:3.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 pending_peering first epoch 25 <= 25, requeueing 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {25=} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:4.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 pending_peering first epoch 25 <= 25, requeueing 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.cs0 to_process <> waiting <> waiting_peering {25=0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25),OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)>} 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:5.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:6.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:02.648+0000 7f8f141a1700 10 osd.7:7.consume_map 25 (was 24) 2019-12-07T19:34:02.648+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.648+0000 7f8f0317f700 10 osd.7 25 handle_pg_query_nopg 1.bs3 2019-12-07T19:34:02.648+0000 7f8f0317f700 10 osd.7 25 pg 1.bs3 dne 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558c5500a000 2019-12-07T19:34:02.648+0000 7f8f0317f700 10 osd.7 25 send_incremental_map 24 -> 25 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:02.648+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~0 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~156 size 0x156 (342) 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1ce0 blob([0x3a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x156) SharedBlob(0x558c575bd810 sbid 0x0)) need 0x0~156 cache has 0x[0~156] 2019-12-07T19:34:02.648+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 = 342 2019-12-07T19:34:02.648+0000 7f8f0317f700 10 osd.7 25 add_map_inc_bl 25 342 bytes 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 24 -> 25 2019-12-07T19:34:02.648+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(25..25 src has 1..25) v4 -- 0x558c5742d680 con 0x558c54c5b400 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 24 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:02.648+0000 7f8f0317f700 10 osd.7 25 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.648+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:25 sent:25 1.bs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 25) v7 -- 0x558c54d0e5a0 con 0x558c54c5b400 2019-12-07T19:34:02.648+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 no pg, should create on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 25 _make_pg 1.cs0 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.c(unlocked)] enter Initial 2019-12-07T19:34:02.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 38 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c54e45400 con 0x558c54c5b400 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.649576+0000 3 0.000082 2019-12-07T19:34:02.648+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.3s4 epoch_sent: 25 epoch_requested: 25 MQuery 1.3s4 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.3s4 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.649586+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7f8f06986700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f06986700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:02.648+0000 7f8f06986700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.3s4 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.3s4 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558c5513e000 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:02.648+0000 7f8f06986700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:02.648+0000 7f8f06986700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.c(unlocked)] enter NotTrimming 2019-12-07T19:34:02.648+0000 7f8f08189700 15 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 24 -> 25 2019-12-07T19:34:02.648+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.cs0_head fresh osr 0x558c5525e000 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 24 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 0 up [7,5,4,1,2,3] acting [7,5,4,1,2,3] history ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s past_intervals ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0))) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [1,4,5,0,7,2147483647](1) acting [1,4,5,0,7,2147483647](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [1,4,5,0,7,2147483647](1) acting [1,4,5,0,7,2147483647](1)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,0] 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967283954s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4))) 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] init_hb_stamps now [0x558c54677f10,0x558c5523a3f0,0x558c5523a230,0x558c5523a150,0x558c54ea3ce0] 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.967283954s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 26.085134629s + 0s) 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.967283954s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.649692+0000 3 0.000047 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.967283954s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.649706+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.053267305s (mnow 26.085147186s + 9.968120119s) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=24 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=25 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f08189700 1 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,2147483647] -> [1,4,5,0,7,3], acting [1,4,5,0,7,2147483647] -> [1,4,5,0,7,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=25 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=25 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:02.649755+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=25 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,0] 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=0 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=25 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,2147483647]p4(0) r=4 lpr=25 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f07187700 15 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,24)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_activate_map: 25 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] activate_map 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] update_heartbeat_peers -> 1,2,3,4,5,7 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 25 need_heartbeat_peer_update 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [4,1,2,5,7,2147483647](4) acting [4,1,2,5,7,2147483647](4)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [4,1,2,5,7,2147483647](4) acting [4,1,2,5,7,2147483647](4)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:02.649815+0000 1 0.000033 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.969495434s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Start 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.648+0000 7f8f06986700 1 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: transitioning to Primary 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.969495434s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Start 2019-12-07T19:34:02.649838+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.969495434s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968120119s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.969495434s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary/Peering 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.050572280s (mnow 26.085292556s + 9.965279724s) 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.649853+0000 1 0.000373 2019-12-07T19:34:02.648+0000 7f8f06986700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior all_probe 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.648+0000 7f8f08189700 1 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior maybe_rw interval:18, acting: 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:34:02.648+0000 7f8f07187700 1 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] start_peering_interval up [4,1,2,5,7,2147483647] -> [4,1,2,5,7,0], acting [4,1,2,5,7,2147483647] -> [4,1,2,5,7,0], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior prior osd.6 is down 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.649881+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.648+0000 7f8f08189700 5 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior final: probe 1(3),2(4),3(5),4(2),5(1),7(0) down 6 blocked_by {} 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] up_thru 18 < same_since 25, must notify monitor 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.1(3) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.2(4) 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.3(5) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.9s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.4(2) 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.053267305s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.5(1) 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] check_prior_readable_down_osds prior_readable_down_osds osd.6 is dead as of epoch 25 2019-12-07T19:34:02.648+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 25 send_incremental_map 24 -> 25 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.648+0000 7f8f06986700 15 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:34:02.648+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(25..25 src has 1..25) v4 -- 0x558c5524c900 con 0x558c54c64000 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.648+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] activate_map: Not dirtying info: last_persisted is 25 while current is 25 2019-12-07T19:34:02.648+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7f8f08189700 10 osd.7 25 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.cs3 1.cs3 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558c574a1900 con 0x558c54c64000 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 25 send_incremental_map 23 -> 25 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:02.648+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:25 sent:25 1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967980480s) 4->0 ([18,24] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e25/25) v1 -- 0x558c57450700 con 0x558c54c64000 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(24..25 src has 1..25) v4 -- 0x558c57377680 con 0x558c54c19800 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 25 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 23 -> 25 (shared) 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.cs4 1.cs4 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558c54e44780 con 0x558c54c19800 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 25 send_incremental_map 24 -> 25 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(25..25 src has 1..25) v4 -- 0x558c5742c480 con 0x558c54c5bc00 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965279724s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.648+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:25 sent:25 1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e25/25) v1 -- 0x558c54e98700 con 0x558c54c64000 2019-12-07T19:34:02.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 71 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54e45400 con 0x558c54c64000 2019-12-07T19:34:02.648+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d21b00 seq 15 2019-12-07T19:34:02.648+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.7s4 epoch_sent: 25 epoch_requested: 25 MQuery 1.7s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.7s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.650032+0000 1 0.000312 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.7s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.7s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558c54d5f000 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:02.648+0000 7f8f07187700 1 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.648+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:02.648+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.650065+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:02.648+0000 7f8f07187700 5 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.648+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 6494 (1 ios * 4000 + 2494 bytes) 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes shared_blobs 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 24 -> 25 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 24 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:02.648+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:02.648+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:02.648+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 25 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.3s4 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.cs5 1.cs5 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558c574a0a00 con 0x558c54c5bc00 2019-12-07T19:34:02.648+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7f8f07187700 20 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY pruub 36.050572280s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.648+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.cs2 1.cs2 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558c5756f180 con 0x558c54c5b400 2019-12-07T19:34:02.648+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.650120+0000 3 0.000061 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 25 send_incremental_map 23 -> 25 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(24..25 src has 1..25) v4 -- 0x558c575d5b00 con 0x558c54c19c00 2019-12-07T19:34:02.648+0000 7f8f06986700 10 osd.7 25 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 23 -> 25 (shared) 2019-12-07T19:34:02.648+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.650133+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.cs1 1.cs1 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558c5756f400 con 0x558c54c19c00 2019-12-07T19:34:02.648+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d58600 seq 1 2019-12-07T19:34:02.648+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.648+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.cs0_head bits 4 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.cs0_head bits 4 = 0 2019-12-07T19:34:02.648+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:02.648+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:25 sent:25 1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.965107618s) 4->0 ([18,24] all_participants=1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4)))) e25/25) v1 -- 0x558c57452300 con 0x558c54c5b400 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:30000000::::head# key 0x8080000000000000013000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:02.648+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1110 2019-12-07T19:34:02.648+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.648+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:02.648+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:25 sent:25 1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4)))) e25/25) v1 -- 0x558c57452a00 con 0x558c54c5b400 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._infover' <- _infover 2019-12-07T19:34:02.648+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:02.648+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:02.648+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:02.648+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:02.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:02.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740a300 seq 22 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:02.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.648+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:02.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:02.648+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,2147483647,4,0,7,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:02.649+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.649+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58600 cost 5923 (1 ios * 4000 + 1923 bytes) 2019-12-07T19:34:02.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58600 onodes 0x558c5756f680 shared_blobs 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore.extentmap(0x558c5756f788) update 0#1:30000000::::head# 2019-12-07T19:34:02.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore.extentmap(0x558c5756f788) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:02.649+0000 7f8f0117b700 15 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:30000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:02.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2269 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58600 allocated 0x[] released 0x[] 2019-12-07T19:34:02.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:02.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:02.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 6494 (1 ios * 4000 + 2494 bytes) 2019-12-07T19:34:02.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes shared_blobs 2019-12-07T19:34:02.649+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 prepare 2019-12-07T19:34:02.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:34:02.649+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58600 2019-12-07T19:34:02.649+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 io_done 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2292 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2292 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:34:02.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:34:02.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [1,2147483647,4,0,7,2](1) acting [1,2147483647,4,0,7,2](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [1,2147483647,4,0,7,2](1) acting [1,2147483647,4,0,7,2](1)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 25 handle_pg_create_info new pg pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4982d~8f4 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7:4._attach_pg 1.cs0 0x558c57676000 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4982d 0x4982d~8f4 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 25 identify_splits_and_merges 1.cs0 e25 to e25 pg_nums {18=16} 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7:4._prime_splits 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1982d 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x82d 2019-12-07T19:34:02.649+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x121 and padding block with 0xedf 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.968739338s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.968739338s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.968739338s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.4(2) 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.968739338s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:13.021043+0000, must = 0 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.054714444s (mnow 26.085713045s + 9.969001399s) 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.649+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 25 queue_want_up_thru now 25 (was 18), currently 18 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 25 send_alive up_thru currently 18 want 25 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 25 send_alive want 25 2019-12-07T19:34:02.649+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_alive(want up_thru 25 have 25) v1 -- 0x558c54d9e3c0 con 0x558c5476d000 2019-12-07T19:34:02.649+0000 7f8f0117b700 1 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] start_peering_interval up [1,2147483647,4,0,7,2] -> [1,4,3,0,7,2], acting [1,2147483647,4,0,7,2] -> [1,4,3,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.5(1) 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:24.862108+0000, must = 0 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.649+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.0s3 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.9s4 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.3s4 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969001399s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 13 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1231+0+0 (unknown 0 0 0) 0x558c57452300 con 0x558c54c19c00 2019-12-07T19:34:02.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.4s0 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c5500a000 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.ds2 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c5500d000 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.6s2 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f08189700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.650426+0000 1 0.000303 2019-12-07T19:34:02.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f141a1700 15 osd.7 25 enqueue_peering_evt 1.7s4 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 maybe_update_heartbeat_peers updating 2019-12-07T19:34:02.649+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.649+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 24 -> 25 2019-12-07T19:34:02.649+0000 7f8f0117b700 1 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.649+0000 7f8f04982700 10 osd.7 pg_epoch: 24 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 adding neighbor peer osd.0 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 adding neighbor peer osd.1 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 adding neighbor peer osd.5 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 adding random peer osd.2 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 adding random peer osd.3 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 adding random peer osd.4 2019-12-07T19:34:02.649+0000 7f8f141a1700 10 osd.7 25 maybe_update_heartbeat_peers 6 peers, extras 0,1,2,3,4,5 2019-12-07T19:34:02.649+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.650457+0000 0 0.000000 2019-12-07T19:34:02.649+0000 7f8f141a1700 7 osd.7 25 activate_map version 25 2019-12-07T19:34:02.649+0000 7f8f04982700 10 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.649+0000 7f8f0918b700 20 osd.7 25 agent_entry empty queue 2019-12-07T19:34:02.649+0000 7f8f0117b700 5 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.649+0000 7f8f04982700 10 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.649+0000 7f8f04982700 10 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.649+0000 7f8f04982700 10 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c548ab000 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.649+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f04982700 10 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.7s4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.649+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 24 -> 25 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.054714444s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 24 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active advmap 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c55141000 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f06185700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c54d5c000 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 24 -> 25 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] Active: kicking snap trim 2019-12-07T19:34:02.649+0000 7f8f06185700 10 osd.7 pg_epoch: 24 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:02.649+0000 7f8f0297e700 7 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:02.649+0000 7f8f06185700 10 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:02.649+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:25 sent:25 1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.968854147s) 4->0 ([18,24] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e25/25) v1 -- 0x558c574e4000 con 0x558c54c64000 2019-12-07T19:34:02.649+0000 7f8f06185700 10 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.649+0000 7f8f0297e700 15 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] publish_stats_to_osd 23:23 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 25 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f06185700 10 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.649+0000 7f8f06185700 10 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 24 -> 25 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 25 2019-12-07T19:34:02.649+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:25 sent:25 1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e25/25) v1 -- 0x558c574e4700 con 0x558c54c64000 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 5(1) notify: (query:25 sent:25 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 24 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:02.649+0000 7f8f06185700 10 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.5(1) info 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25), identical to ours 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.649+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 11 ==== pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54ce2000 con 0x558c54c64400 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:02.649+0000 7f8f23c56700 15 osd.7 25 enqueue_peering_evt 1.2s4 epoch_sent: 25 epoch_requested: 25 MQuery 1.2s4 from 0(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1200 seq 24 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] new interval newup [3,2,7,1,4,5] newacting [3,2,7,1,4,5] 2019-12-07T19:34:02.649+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.2s4 from 0(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:02.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:02.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.2s4 from 0(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.650608+0000 3 0.000086 2019-12-07T19:34:02.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.2s4 from 0(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:02.649+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:02.649+0000 7f8f07988700 10 osd.7 25 handle_pg_query_nopg 1.2s4 2019-12-07T19:34:02.649+0000 7f8f07988700 10 osd.7 25 pg 1.2s4 dne 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.650617+0000 0 0.000000 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 6494 (1 ios * 4000 + 2494 bytes) 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4a121 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=24 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:02.649+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 12 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (crc 0 0 0) 0x558c575c6900 con 0x558c54c64400 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:02.649+0000 7f8f1199c700 20 osd.7 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=25 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.649+0000 7f8f1199c700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:02.649+0000 7f8f1199c700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:02.649+0000 7f8f1199c700 20 osd.7 25 _dispatch 0x558c575c6900 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f07988700 10 osd.7 25 send_incremental_map 24 -> 25 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:34:02.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=25 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000444102s (0.000000103s flush + 0.000443999s kv commit) 2019-12-07T19:34:02.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(25..25 src has 1..25) v4 -- 0x558c575c7d40 con 0x558c54c64400 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:02.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=25 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c5513e000 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1517 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=25 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] new interval newup [3,2,7,1,4,5] newacting [3,2,7,1,4,5] 2019-12-07T19:34:02.649+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f07988700 10 osd.7 25 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.649+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=25 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_notify((query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 25) v7 -- 0x558c575cd2c0 con 0x558c54c64400 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2147483647,7,1,2,5]p3(0) r=2 lpr=25 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3786 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:02.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558c54d5f000 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6055 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f05984700 15 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,24)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:02.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6078 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6078 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f05183700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4a121~17be to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4a121 0x4a121~17be to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [3,2147483647,7,1,2,5](3) acting [3,2147483647,7,1,2,5](3)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [3,2147483647,7,1,2,5](3) acting [3,2147483647,7,1,2,5](3)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f1199c700 3 osd.7 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1a121 2019-12-07T19:34:02.649+0000 7f8f1199c700 10 osd.7 25 no new maps here, dropping 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x121 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8df and padding block with 0x721 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973261905s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=1(3),2(4),3(0),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2))) 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.973261905s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.973261905s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.973261905s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558c54ea3ce0] 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.053997442s (mnow 26.086197648s + 9.967799794s) 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:02.649+0000 7f8f05984700 1 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] start_peering_interval up [3,2147483647,7,1,2,5] -> [3,2,7,1,4,5], acting [3,2147483647,7,1,2,5] -> [3,2,7,1,4,5], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 2 -> 2, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.649+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] release_backoffs [1:60000000::::0,1:70000000::::head) 2019-12-07T19:34:02.649+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4b8df 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000160888s (0.000000096s flush + 0.000160792s kv commit) 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58600,0x558c5740a300,0x558c54ea1200> 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 kv_submitted 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58600 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 finishing 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58600 onodes 0x558c5756f680 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 done 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58600 [] 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.649+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967799794s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.650896+0000 1 0.000290 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 72 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c57452300 con 0x558c54c64000 2019-12-07T19:34:02.649+0000 7f8f05984700 1 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.650925+0000 0 0.000000 2019-12-07T19:34:02.649+0000 7f8f05984700 5 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.649+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 73 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c54e9aa00 con 0x558c54c64000 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.1(3) 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 2019-12-07T19:34:02.649+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:11.368974+0000, must = 0 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:25 sent:25 1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) 2->0 ([18,24] all_participants=1(3),2(4),3(0),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e25/25) v1 -- 0x558c57450000 con 0x558c54c5bc00 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.649+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.649+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f7b00 seq 28 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 1(3) notify: (query:25 sent:25 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:02.649+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.1(3) info 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25), identical to ours 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f0297e700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.649+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:02.649+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:02.649+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:02.649+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.649+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes shared_blobs 2019-12-07T19:34:02.649+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:02.649+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:02.649+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:02.649+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4b8df~50c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4b8df 0x4b8df~50c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1b8df 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8df 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdeb and padding block with 0x215 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4bdeb 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:02.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000067608s (0.000000082s flush + 0.000067526s kv commit) 2019-12-07T19:34:02.649+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:02.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7b00> 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:02.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:02.649+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:02.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 39 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c574e7100 con 0x558c54c5b400 2019-12-07T19:34:02.649+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 4(2) notify: (query:25 sent:25 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.4(2) info 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25), identical to ours 2019-12-07T19:34:02.649+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7f8f06986700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.649+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.650+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 33 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c548a3800 con 0x558c54c19800 2019-12-07T19:34:02.650+0000 7f8f24c58700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.650+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.650+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.650+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 34 ==== osd_map(24..25 src has 1..25) v4 ==== 1314+0+0 (crc 0 0 0) 0x558c57377680 con 0x558c54c19800 2019-12-07T19:34:02.650+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.2(4) 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 2019-12-07T19:34:02.650+0000 7f8f1199c700 20 osd.7 25 OSD::ms_dispatch: osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.650+0000 7f8f1199c700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:02.650+0000 7f8f1199c700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:02.650+0000 7f8f1199c700 20 osd.7 25 _dispatch 0x558c57377680 osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.650+0000 7f8f1199c700 3 osd.7 25 handle_osd_map epochs [24,25], i have 25, src has [1,25] 2019-12-07T19:34:02.650+0000 7f8f1199c700 10 osd.7 25 no new maps here, dropping 2019-12-07T19:34:02.650+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:24.168649+0000, must = 0 2019-12-07T19:34:02.650+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:02.650+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.650+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:02.650+0000 7f8f0297e700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.650+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 35 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c548a3800 con 0x558c54c19800 2019-12-07T19:34:02.651+0000 7f8f24c58700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.651+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.651+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.651+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.651+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 2(4) notify: (query:25 sent:25 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.651+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.2(4) info 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25), identical to ours 2019-12-07T19:34:02.651+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:02.651+0000 7f8f06986700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.651+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.652+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 41 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57450000 con 0x558c54c5bc00 2019-12-07T19:34:02.652+0000 7f8f23c56700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.652+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.652+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 42 ==== pg_notify2(1.cs0 1.cs0 (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57450000 con 0x558c54c5bc00 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.652+0000 7f8f23c56700 15 osd.7 25 enqueue_peering_evt 1.cs0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.652+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.3(5) 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 2019-12-07T19:34:02.652+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 43 ==== pg_query2(1.6s2 1.6s2 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c574a0a00 con 0x558c54c5bc00 2019-12-07T19:34:02.652+0000 7f8f23c56700 15 osd.7 25 enqueue_peering_evt 1.6s2 epoch_sent: 25 epoch_requested: 25 MQuery 1.6s2 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.652+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.6s2 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:15.425896+0000, must = 0 2019-12-07T19:34:02.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:02.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.6s2 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.6s2 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558c54d5c000 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:02.652+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.967667479s) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.6s2 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.652+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 44 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558c5742c480 con 0x558c54c5bc00 2019-12-07T19:34:02.652+0000 7f8f1199c700 20 osd.7 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.652+0000 7f8f1199c700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:02.652+0000 7f8f1199c700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:02.652+0000 7f8f1199c700 20 osd.7 25 _dispatch 0x558c5742c480 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.652+0000 7f8f05984700 20 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY pruub 36.053997442s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.652+0000 7f8f0297e700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:02.653843+0000 10 0.000301 2019-12-07T19:34:02.652+0000 7f8f05984700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.652+0000 7f8f1199c700 3 osd.7 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.652+0000 7f8f1199c700 10 osd.7 25 no new maps here, dropping 2019-12-07T19:34:02.652+0000 7f8f0297e700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:02.652+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 45 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558c575c7d40 con 0x558c54c5bc00 2019-12-07T19:34:02.652+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:02.652+0000 7f8f1199c700 20 osd.7 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.652+0000 7f8f1199c700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:02.652+0000 7f8f1199c700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:02.652+0000 7f8f1199c700 20 osd.7 25 _dispatch 0x558c575c7d40 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.652+0000 7f8f1199c700 3 osd.7 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.652+0000 7f8f1199c700 10 osd.7 25 no new maps here, dropping 2019-12-07T19:34:02.652+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:25 sent:25 1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=1(3),2(4),3(0),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e25/25) v1 -- 0x558c57453800 con 0x558c54c5bc00 2019-12-07T19:34:02.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.1(3) 1.cs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966084358s) 2019-12-07T19:34:02.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0900 seq 29 2019-12-07T19:34:02.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.2(4) 1.cs4( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964735259s) 2019-12-07T19:34:02.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:02.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.3(5) 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963623646s) 2019-12-07T19:34:02.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes shared_blobs 2019-12-07T19:34:02.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[] released 0x[] 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.4(2) 1.cs2( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.969145425s) 2019-12-07T19:34:02.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:34:02.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:34:02.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.5(1) 1.cs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2019-12-07T19:34:02.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.7(0) 1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 5(1) For position 2: selecting up[i]: 4(2) For position 3: selecting up[i]: 1(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4bdeb~416 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4bdeb 0x4bdeb~416 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1bdeb 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdeb 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x201 and padding block with 0xdff 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_async_recovery_ec candidates by cost are: 12,7(0) 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_async_recovery_ec result want=[2147483647,5,4,1,2,3] async_recovery=7(0) 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting want [2147483647,5,4,1,2,3] != acting [7,5,4,1,2,3], requesting pg_temp change 2019-12-07T19:34:02.652+0000 7f8f0297e700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:02.653981+0000 0 0.000000 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c201 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Leaving Peering 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:02.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:02.652+0000 7f8f0297e700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 peering mbc={}] exit Started/Primary/Peering 2019-12-07T19:34:02.653997+0000 0 0.000000 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000089630s (0.000000119s flush + 0.000089511s kv commit) 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:02.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0900> 2019-12-07T19:34:02.652+0000 7f8f0297e700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started/Primary 2019-12-07T19:34:02.654005+0000 0 0.000000 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:34:02.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:02.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:34:02.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] release_backoffs [1:30000000::::0,1:40000000::::head) 2019-12-07T19:34:02.652+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:02.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.652+0000 7f8f0297e700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary 2019-12-07T19:34:02.652+0000 7f8f0297e700 5 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary/WaitActingChange 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.652+0000 7f8f0297e700 10 osd.7 25 send_pg_temp {1.c=[2147483647,5,4,1,2,3]} 2019-12-07T19:34:02.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558c57676000 2019-12-07T19:34:02.652+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_pgtemp(e25 {1.c=[2147483647,5,4,1,2,3]} v25) v2 -- 0x558c54d2ed80 con 0x558c5476d000 2019-12-07T19:34:02.652+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.cs0 from 3(5) notify: (query:25 sent:25 1.cs5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=1(3),2(4),3(5),4(2),5(1),6(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.652+0000 7f8f06986700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: In WaitActingChange, ignoring MNotifyRec 2019-12-07T19:34:02.652+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:02.652+0000 7f8f06986700 10 osd.7 25 queue_want_up_thru want 25 <= queued 25, currently 18 2019-12-07T19:34:02.652+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.672+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.722+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.772+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.822+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.837+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:02.872+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.923+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.973+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.016+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c5a000 0x558c54c37600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:03.016+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c5a000 0x558c54c37600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:03.016+0000 7f8f24457700 10 osd.7 25 ms_handle_authentication new session 0x558c54c5d860 con 0x558c54c5a000 entity client.admin addr 2019-12-07T19:34:03.016+0000 7f8f24457700 10 osd.7 25 ms_handle_authentication session 0x558c54c5d860 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:03.016+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1006309048 conn(0x558c54c5a000 0x558c54c37600 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4591 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:03.016+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4591 172.21.15.131:0/1006309048 1 ==== command(tid 14: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c54d0e5a0 con 0x558c54c5a000 2019-12-07T19:34:03.016+0000 7f8f164bb700 20 osd.7 25 OSD::ms_dispatch: command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:03.016+0000 7f8f164bb700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:03.016+0000 7f8f164bb700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:03.016+0000 7f8f164bb700 20 osd.7 25 _dispatch 0x558c54d0e5a0 command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:03.017+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1006309048 -- command_reply(tid 14: 0 ) v1 -- 0x558c54ce08c0 con 0x558c54c5a000 2019-12-07T19:34:03.023+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.040+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4591 172.21.15.131:0/1006309048 2 ==== command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c575ccb40 con 0x558c54c5a000 2019-12-07T19:34:03.040+0000 7f8f164bb700 20 osd.7 25 OSD::ms_dispatch: command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:03.040+0000 7f8f164bb700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:03.040+0000 7f8f164bb700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:03.040+0000 7f8f164bb700 20 osd.7 25 _dispatch 0x558c575ccb40 command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:03.041+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1006309048 -- command_reply(tid 15: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce0e00 con 0x558c54c5a000 2019-12-07T19:34:03.044+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1006309048 conn(0x558c54c5a000 msgr2=0x558c54c37600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 64 2019-12-07T19:34:03.044+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1006309048 conn(0x558c54c5a000 msgr2=0x558c54c37600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:03.044+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1006309048 conn(0x558c54c5a000 0x558c54c37600 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:03.044+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1006309048 conn(0x558c54c5a000 0x558c54c37600 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:03.044+0000 7f8f164bb700 2 osd.7 25 ms_handle_reset con 0x558c54c5a000 session 0x558c54c5d860 2019-12-07T19:34:03.073+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.123+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.173+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.223+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.274+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.323+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.372+0000 7f8f0097a700 5 osd.7 25 heartbeat osd_stat(store_statfs(0x163fd9a000/0x40000000/0x1680000000, data 0x26926e/0x264000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5] op hist []) 2019-12-07T19:34:03.372+0000 7f8f0097a700 20 osd.7 25 check_full_status cur ratio 2.60247e-05, physical ratio 2.60247e-05, new state none 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.690501011s) v5 -- 0x558c54c58e00 con 0x558c5476d400 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.690501011s) v5 -- 0x558c54c59400 con 0x558c551b6800 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.688153661s) v5 -- 0x558c54c58200 con 0x558c54c9e000 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.688153661s) v5 -- 0x558c54c59e00 con 0x558c54c9e400 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.673365549s) v5 -- 0x558c54c59200 con 0x558c551b7800 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.673365549s) v5 -- 0x558c54c59800 con 0x558c551b7c00 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.687771294s) v5 -- 0x558c54c58c00 con 0x558c552c2000 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.687771294s) v5 -- 0x558c54c58000 con 0x558c552c2400 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.696166813s) v5 -- 0x558c54c58600 con 0x558c552c2800 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.696166813s) v5 -- 0x558c54c59600 con 0x558c552c2c00 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.678777924s) v5 -- 0x558c5768b000 con 0x558c54c65800 2019-12-07T19:34:03.372+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.678777924s) v5 -- 0x558c5768b200 con 0x558c54c65c00 2019-12-07T19:34:03.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 4 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.839810202s delta_ub 0.031071774s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c54c65800 2019-12-07T19:34:03.372+0000 7f8f23c56700 25 osd.7 25 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:01.073229+0000 -> 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:01.073286+0000 2019-12-07T19:34:03.372+0000 7f8f23c56700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030814720s,0.031071774s]) 2019-12-07T19:34:03.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 4 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.839836017s delta_ub 0.031097589s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c54c65c00 2019-12-07T19:34:03.372+0000 7f8f23c56700 25 osd.7 25 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:01.073286+0000 -> 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:03.372+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.822707675s delta_ub 0.013969247s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c552c2800 2019-12-07T19:34:03.372+0000 7f8f23c56700 25 osd.7 25 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:03.373299+0000) and older pending ping(s) 2019-12-07T19:34:03.372+0000 7f8f23c56700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030814786s,0.031097589s]) 2019-12-07T19:34:03.372+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.822759404s delta_ub 0.014020976s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b8200 con 0x558c552c2c00 2019-12-07T19:34:03.372+0000 7f8f24c58700 25 osd.7 25 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:01.073263+0000 -> 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:01.073309+0000 2019-12-07T19:34:03.372+0000 7f8f24c58700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013673423s,0.013969247s]) 2019-12-07T19:34:03.372+0000 7f8f24457700 25 osd.7 25 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:01.073309+0000 -> 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:03.372+0000 7f8f24457700 25 osd.7 25 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:03.373299+0000) and older pending ping(s) 2019-12-07T19:34:03.372+0000 7f8f24457700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013703655s,0.014020976s]) 2019-12-07T19:34:03.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.845388659s delta_ub 0.036650231s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c551b7800 2019-12-07T19:34:03.372+0000 7f8f23c56700 25 osd.7 25 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:01.073398+0000 -> 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:01.073389+0000 2019-12-07T19:34:03.372+0000 7f8f23c56700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036208311s,0.036650231s]) 2019-12-07T19:34:03.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.831178684s delta_ub 0.022440256s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c552c2400 2019-12-07T19:34:03.372+0000 7f8f23c56700 25 osd.7 25 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:01.073408+0000 last_rx_front 2019-12-07T19:34:01.073380+0000 -> 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:03.372+0000 7f8f23c56700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021982715s,0.022440256s]) 2019-12-07T19:34:03.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 4 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.830598699s delta_ub 0.021860271s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c54c9e000 2019-12-07T19:34:03.372+0000 7f8f23c56700 25 osd.7 25 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:01.073353+0000 -> 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:01.073454+0000 2019-12-07T19:34:03.372+0000 7f8f23c56700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021375528s,0.021860271s]) 2019-12-07T19:34:03.372+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.828357455s delta_ub 0.019619027s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c5476d400 2019-12-07T19:34:03.372+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.831263039s delta_ub 0.022524611s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b8200 con 0x558c552c2000 2019-12-07T19:34:03.372+0000 7f8f23c56700 25 osd.7 25 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:01.073421+0000 -> 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:01.073431+0000 2019-12-07T19:34:03.372+0000 7f8f23c56700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019116636s,0.019619027s]) 2019-12-07T19:34:03.372+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.845482767s delta_ub 0.036744339s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c551b7c00 2019-12-07T19:34:03.372+0000 7f8f24457700 25 osd.7 25 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:01.073408+0000 -> 2019-12-07T19:34:03.373810+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:03.372+0000 7f8f24457700 25 osd.7 25 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:03.373299+0000) and older pending ping(s) 2019-12-07T19:34:03.372+0000 7f8f24457700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022013251s,0.022524611s]) 2019-12-07T19:34:03.372+0000 7f8f24c58700 25 osd.7 25 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:01.073389+0000 -> 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:03.372+0000 7f8f24c58700 25 osd.7 25 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:03.373299+0000) and older pending ping(s) 2019-12-07T19:34:03.372+0000 7f8f24c58700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036210850s,0.036744339s]) 2019-12-07T19:34:03.372+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 8 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.828479463s delta_ub 0.019741035s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c551b6800 2019-12-07T19:34:03.372+0000 7f8f24c58700 25 osd.7 25 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:01.073431+0000 -> 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:03.372+0000 7f8f24c58700 25 osd.7 25 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:34:03.373299+0000) and older pending ping(s) 2019-12-07T19:34:03.372+0000 7f8f24c58700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019173437s,0.019741035s]) 2019-12-07T19:34:03.372+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 4 ==== osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.830689455s delta_ub 0.021951027s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c9e400 2019-12-07T19:34:03.372+0000 7f8f24c58700 25 osd.7 25 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:01.073454+0000 -> 2019-12-07T19:34:03.373892+0000 2019-12-07T19:34:03.372+0000 7f8f24c58700 25 osd.7 25 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:03.373299+0000) and older pending ping(s) 2019-12-07T19:34:03.372+0000 7f8f24c58700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021357758s,0.021951027s]) 2019-12-07T19:34:03.373+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.407+0000 7f8f21451700 10 osd.7 25 tick 2019-12-07T19:34:03.407+0000 7f8f21451700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:03.407+0000 7f8f21451700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:03.407+0000 7f8f21451700 20 osd.7 25 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:03.424+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.442+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 7 ==== osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.914843615s delta_ub 0.036744339s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b8200 con 0x558c552d1c00 2019-12-07T19:34:03.442+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 7 ==== osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.914843615s delta_ub 0.036744339s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c552d1800 2019-12-07T19:34:03.442+0000 7f8f24457700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036118565s,0.036744339s]) 2019-12-07T19:34:03.442+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.878725050s delta_ub 18446744073.673433051s) v5 -- 0x558c54c59600 con 0x558c552d1c00 2019-12-07T19:34:03.442+0000 7f8f24c58700 20 osd.7 25 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036085548s,0.036744339s]) 2019-12-07T19:34:03.442+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.878758067s delta_ub 18446744073.673466068s) v5 -- 0x558c54c58600 con 0x558c552d1800 2019-12-07T19:34:03.467+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:03.474+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.524+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.574+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 228873 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 16 ==== osd_map(25..26 src has 1..26) v4 ==== 1263+0+0 (crc 0 0 0) 0x558c54d2ed80 con 0x558c5476d000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 osd.7 25 OSD::ms_dispatch: osd_map(25..26 src has 1..26) v4 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 osd.7 25 _dispatch 0x558c54d2ed80 osd_map(25..26 src has 1..26) v4 2019-12-07T19:34:03.600+0000 7f8f164bb700 3 osd.7 25 handle_osd_map epochs [25,26], i have 25, src has [1,26] 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 osd.7 25 handle_osd_map got inc map for epoch 26 2019-12-07T19:34:03.600+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3820e826:::osdmap.25:0# 0x0~0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1428 size 0x1428 (5160) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fab0 blob([0x267000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5757ee70 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757efc0 blob([!~1000,0x268000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558c5757f420 sbid 0x0)) need 0x1000~428 cache has 0x[1000~428] 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3820e826:::osdmap.25:0# 0x0~1428 = 5160 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 osd.7 25 add_map_bl 25 5160 bytes 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 osd.7 25 got_full_map 26, nothing requested 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 osd.7 25 handle_osd_map pg_num_history pg_num_history(e26 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {26={}} 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 osd.7 25 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e26 [1,26] lci=[13,26]) 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c574ff200 seq 17 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cb3b972f:::inc_osdmap.26:0# key 0x7f7fffffffffffffffcb3b97'/!inc_osdmap.26!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1111 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~365 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc9b on front/back, now 0x0~1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574ff200 1 blobs 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x269000~1000/1000,1000,0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x269000~1000] 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1b90 blob([0x269000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1a40 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c574f1b90) get_ref 0x0~365 Blob(0x558c574f1b90 blob([0x269000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1a40 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c574f1b90) get_ref init 0x1000, 1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~365: 0x0~365 Blob(0x558c574f1b90 blob([0x269000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x365) SharedBlob(0x558c574f1a40 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c574f1a58 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x365 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2da08) dirty_range mark inline shard dirty 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 = 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:35a0e826:::osdmap.26:0# key 0x7f7fffffffffffffff35a0e8'&!osdmap.26!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:35a0e826:::osdmap.26:0# 0x0~1671 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1112 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:35a0e826:::osdmap.26:0# 0x0~1671 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~671 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x98f on front/back, now 0x0~1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574ff200 2 blobs 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x26a000~2000/1000,2000,0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x26a000~2000] 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bdb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265c00 blob([0x26a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bdb90 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c55265c00) get_ref 0x0~1000 Blob(0x558c55265c00 blob([0x26a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bdb90 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c55265c00) get_ref init 0x1000, 1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55265c00 blob([0x26a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575bdb90 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c575bdba8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264700 blob([!~1000,0x26b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677030 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c55264700) get_ref 0x1000~671 Blob(0x558c55264700 blob([!~1000,0x26b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c54677030 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c55264700) get_ref init 0x2000, 1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~671: 0x1000~671 Blob(0x558c55264700 blob([!~1000,0x26b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558c54677030 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54677048 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1671 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2dc88) dirty_range mark inline shard dirty 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:35a0e826:::osdmap.26:0# 0x0~1671 = 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574ff200 0 blobs 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574ff200 0 blobs 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff200 cost 24095 (4 ios * 4000 + 8095 bytes) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff200 onodes 0x558c53a91900,0x558c54d2d900,0x558c54d2db80,0x558c55276c80 shared_blobs 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2da08) update #-1:cb3b972f:::inc_osdmap.26:0# 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2da08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cb3b972f:::inc_osdmap.26:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2dc88) update #-1:35a0e826:::osdmap.26:0# 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2dc88) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:35a0e826:::osdmap.26:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff200 allocated 0x[269000~3000] released 0x[] 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 freelist allocate 0x269000~3000 2019-12-07T19:34:03.600+0000 7f8f164bb700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 prepare 2019-12-07T19:34:03.600+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c574ff200 2019-12-07T19:34:03.601+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 aio_wait 2019-12-07T19:34:03.601+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff200 2019-12-07T19:34:03.601+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 io_done 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:03.601+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 40 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c5756f180 con 0x558c54c5b400 2019-12-07T19:34:03.601+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.bs3 epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.601+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.601+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.601+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 map 26 > 25 2019-12-07T19:34:03.601+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 26, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.601+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 41 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558c575d5b00 con 0x558c54c5b400 2019-12-07T19:34:03.601+0000 7f8f1199c700 20 osd.7 25 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.601+0000 7f8f1199c700 10 osd.7 25 do_waiters -- start 2019-12-07T19:34:03.601+0000 7f8f1199c700 10 osd.7 25 do_waiters -- finish 2019-12-07T19:34:03.601+0000 7f8f1199c700 20 osd.7 25 _dispatch 0x558c575d5b00 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.601+0000 7f8f1199c700 3 osd.7 25 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.601+0000 7f8f1199c700 10 osd.7 25 no new maps here, dropping 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2d900 had 1 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2db80 had 1 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c201~226c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c201 0x4c201~226c to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1c201 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x201 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x46d and padding block with 0xb93 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4e46d 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:03.601+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:03.601+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 42 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c5756f180 con 0x558c54c5b400 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000736301s (0.000577475s flush + 0.000158826s kv commit) 2019-12-07T19:34:03.601+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:03.601+0000 7f8f24457700 15 osd.7 25 enqueue_peering_evt 1.3s4 epoch_sent: 26 epoch_requested: 26 MQuery 1.3s4 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.601+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s4 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff200> 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.601+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 kv_submitted 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff200 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c574ff200 osr 0x558c54ea4b40 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 14 0x1b000~1000 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 15 0x1b000~1000 crc 9316858d 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) _discard drop 14 0x2000~1000 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5525f680) prepare_write seq 15 0x2000~1000 crc 55839bb3 2019-12-07T19:34:03.601+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7 25 _committed_osd_maps 26..26 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7 25 advance to epoch 26 (<= last 26 <= newest_map 26) 2019-12-07T19:34:03.601+0000 7f8f141a1700 7 osd.7 26 consume_map version 26 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.0s3 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.9s4 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:2.identify_splits_and_merges identify_splits_and_merges slot 1.2s4 has no pg and waiting_for_split 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs3 has no pg and waiting_for_split 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.3s4 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.cs0 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.4s0 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.ds2 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.6s2 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 identify_splits_and_merges 1.7s4 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 26 prune_pg_created 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:0.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:1.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:2.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 empty, pruning 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:3.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 pending_peering first epoch 26 <= 26, requeueing 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {26=} 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:4.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:5.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:6.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7:7.consume_map 26 (was 25) 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:03.601+0000 7f8f141a1700 10 osd.7 26 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.0s3 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.9s4 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.3s4 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.cs0 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c5500d000 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.4s0 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.ds2 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.6s2 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f141a1700 15 osd.7 26 enqueue_peering_evt 1.7s4 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 25 -> 26 2019-12-07T19:34:03.601+0000 7f8f141a1700 7 osd.7 26 activate_map version 26 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 osd.7 pg_epoch: 25 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:03.601+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.601+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.601+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c5500a000 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.601+0000 7f8f04181700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.601+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 25 -> 26 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 26 2019-12-07T19:34:03.601+0000 7f8f04181700 10 osd.7 pg_epoch: 25 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,2147483647] -- 1/1 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.601+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:03.601+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.601+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:03.601+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.601+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:03.601+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 26 handle_pg_query_nopg 1.bs3 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 26 pg 1.bs3 dne 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.603207+0000 3 0.000067 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 26 send_incremental_map 25 -> 26 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.603216+0000 0 0.000000 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.602+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~0 2019-12-07T19:34:03.602+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~365 size 0x365 (869) 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.602+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.602+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1b90 blob([0x269000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x365) SharedBlob(0x558c574f1a40 sbid 0x0)) need 0x0~365 cache has 0x[0~365] 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.602+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 = 869 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 26 add_map_inc_bl 26 869 bytes 2019-12-07T19:34:03.602+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(26..26 src has 1..26) v4 -- 0x558c575806c0 con 0x558c54c5b400 2019-12-07T19:34:03.602+0000 7f8f04181700 15 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 26 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.602+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:26 sent:26 1.bs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 26) v7 -- 0x558c54d390e0 con 0x558c54c5b400 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,3](1)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,3](1)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s4 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s4 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558c5513e000 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4))) 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 25 -> 26 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.038774203s + 0s) 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 25 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,2147483647] -- 4/4 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.602+0000 7f8f04181700 1 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,3] -> [1,4,5,0,7,3], acting [1,4,5,0,7,3] -> [1,4,5,0,7,2147483647], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,2147483647] 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.603378+0000 3 0.000074 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.603390+0000 0 0.000000 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=25 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=26 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=26 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=26 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=26 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,2147483647] 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=26 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]p4(0) r=4 lpr=26 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.602+0000 7f8f07187700 15 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,25)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.603456+0000 1 0.000259 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.602+0000 7f8f04181700 1 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.603477+0000 0 0.000000 2019-12-07T19:34:03.602+0000 7f8f04181700 5 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,0](4)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,0](4)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.602+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.602+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 26 send_incremental_map 25 -> 26 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.602+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(26..26 src has 1..26) v4 -- 0x558c55195200 con 0x558c54c64000 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:03.602+0000 7f8f04181700 10 osd.7 26 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.038960500s + 0s) 2019-12-07T19:34:03.602+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:26 sent:26 1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e26/26) v1 -- 0x558c548a2300 con 0x558c54c64000 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.602+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:03.602+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d59200 seq 16 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.602+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:03.602+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:03.602+0000 7f8f07187700 1 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [4,1,2,5,7,0] -> [4,1,2,5,7,0], acting [4,1,2,5,7,0] -> [4,1,2,5,7,2147483647], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.602+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:03.602+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.602+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:03.602+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:03.602+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:03.602+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:03.602+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:03.602+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:03.602+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.602+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c57676000 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_advance_map: 26 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 25 -> 26 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_advance_map [7,5,4,1,2,3]/[2147483647,5,4,1,2,3] -- 7/5 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4e46d~517 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4e46d 0x4e46d~517 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1e46d 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: verifying no want_acting [] targets didn't go down 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x46d 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x984 and padding block with 0x67c 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] new interval newup [7,5,4,1,2,3] newacting [2147483647,5,4,1,2,3] 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started/Primary/WaitActingChange 2019-12-07T19:34:03.603632+0000 2 0.000253 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started/Primary 2019-12-07T19:34:03.603637+0000 0 0.000000 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] release_backoffs [1:30000000::::0,1:40000000::::head) 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started 2019-12-07T19:34:03.603649+0000 0 0.000000 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=25 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=26 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4e984 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=26 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=26 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: Reset advmap 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=26 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] new interval newup [7,5,4,1,2,3] newacting [2147483647,5,4,1,2,3] 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.603664+0000 1 0.000296 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=26 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000087328s (0.000000065s flush + 0.000087263s kv commit) 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]p7(0) r=0 lpr=26 pi=[18,25)/1 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 26 clear_ready_to_merge 1.cs0 2019-12-07T19:34:03.602+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:03.602+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:03.602+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:03.602+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:03.602+0000 7f8f0297e700 15 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=0 lpr=26 pi=[18,25)/1 crt=0'0 unknown mbc={}] clear_stats 2019-12-07T19:34:03.602+0000 7f8f07187700 1 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.602+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:03.602+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.603694+0000 0 0.000000 2019-12-07T19:34:03.602+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7f8f07187700 5 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [7,5,4,1,2,3](7) acting [7,5,4,1,2,3](7)) up_thru 18 up_from 13 last_epoch_clean 19 interval(25-25 up [7,5,4,1,2,3](7) acting [7,5,4,1,2,3](7)) : primary up 13-18 does not include interval 2019-12-07T19:34:03.602+0000 7f8f07187700 20 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] noting past ([18,25] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0))) 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] on_new_interval 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.602+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] init_hb_stamps now [] 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 27.039166232s + 0s) 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] cancel_recovery 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] clear_recovery_state 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.3s4 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.602+0000 7f8f0297e700 1 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] start_peering_interval up [7,5,4,1,2,3] -> [7,5,4,1,2,3], acting [7,5,4,1,2,3] -> [2147483647,5,4,1,2,3], acting_primary 7(0) -> 5, up_primary 7(0) -> 7, role 0 -> -1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.602+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] clear_primary_state 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] release_backoffs [1:30000000::::0,1:40000000::::head) 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] agent_stop 2019-12-07T19:34:03.602+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] on_change 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] cancel_copy_ops 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] cancel_flush_ops 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] cancel_proxy_ops 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] cancel_manifest_ops 2019-12-07T19:34:03.602+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:26 sent:26 1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4)))) e26/26) v1 -- 0x558c57453100 con 0x558c54c5b400 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] on_change_cleanup 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] on_change 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] exit NotTrimming 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped mbc={}] enter NotTrimming 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] plpg_on_role_change 2019-12-07T19:34:03.602+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:26 sent:26 1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4)))) e26/26) v1 -- 0x558c576b8000 con 0x558c54c5b400 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.602+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:03.602+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740ac00 seq 23 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] handle_activate_map: 26 2019-12-07T19:34:03.602+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c55141000 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] exit Reset 2019-12-07T19:34:03.603829+0000 1 0.000185 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] enter Started 2019-12-07T19:34:03.602+0000 7f8f0918b700 20 osd.7 26 agent_entry empty queue 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] enter Start 2019-12-07T19:34:03.602+0000 7f8f0297e700 1 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] state: transitioning to Stray 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 25 -> 26 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] exit Start 2019-12-07T19:34:03.603856+0000 0 0.000000 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 osd.7 pg_epoch: 25 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:03.602+0000 7f8f0297e700 5 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] enter Started/Stray 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 26 send_incremental_map 25 -> 26 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:03.602+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(26..26 src has 1..26) v4 -- 0x558c575c78c0 con 0x558c54c19c00 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 26 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.602+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_notify2(1.cs1 1.cs1 (query:26 sent:26 1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 0->1 ([18,25] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e26/26) v1 -- 0x558c576b9c00 con 0x558c54c19c00 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c5740bb00 seq 2 2019-12-07T19:34:03.602+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5440 (1 ios * 4000 + 1440 bytes) 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c548ab000 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 25 -> 26 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 25 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active advmap 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1284 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:03.602+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] Active: kicking snap trim 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1307 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1307 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7f8f0297e700 7 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:03.602+0000 7f8f0297e700 15 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] publish_stats_to_osd 24:24 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 26 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.602+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 26 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.602+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:03.602+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:03.602+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:03.602+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:03.602+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740ac00 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:03.603+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740ac00 onodes shared_blobs 2019-12-07T19:34:03.603+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:03.603+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 prepare 2019-12-07T19:34:03.603+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740ac00 2019-12-07T19:34:03.603+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 io_done 2019-12-07T19:34:03.603+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c5513e000 2019-12-07T19:34:03.603+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:03.602+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4e984~51b to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4e984 0x4e984~51b to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1e984 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x984 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe9f and padding block with 0x161 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4ee9f 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000242558s (0.000000107s flush + 0.000242451s kv commit) 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4ee9f~517 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4ee9f 0x4ee9f~517 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1ee9f 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe9f 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3b6 and padding block with 0xc4a 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4f3b6 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:03.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000133400s (0.000000092s flush + 0.000133308s kv commit) 2019-12-07T19:34:03.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:03.603+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740bb00,0x558c5740ac00> 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 kv_submitted 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740ac00 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 finishing 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740ac00 onodes 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740ac00 done 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740ac00 [] 2019-12-07T19:34:03.603+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:03.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:03.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c54d5c000 2019-12-07T19:34:03.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.603+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 25 -> 26 2019-12-07T19:34:03.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 25 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5] -- 3/3 2019-12-07T19:34:03.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [3,2,7,1,4,5] newacting [3,2147483647,7,1,2147483647,5] 2019-12-07T19:34:03.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.603+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.604823+0000 3 0.000069 2019-12-07T19:34:03.603+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.604834+0000 0 0.000000 2019-12-07T19:34:03.603+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=25 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558c54d5f000 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 25 -> 26 2019-12-07T19:34:03.603+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 25 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2] -- 1/1 2019-12-07T19:34:03.603+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.603+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,2147483647,2147483647,0,7,2] 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.603+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.604923+0000 3 0.000081 2019-12-07T19:34:03.603+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.604932+0000 0 0.000000 2019-12-07T19:34:03.603+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,2147483647,2147483647,0,7,2] 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7f8f0117b700 15 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [1,4,3,0,7,2](1) acting [1,4,3,0,7,2](1)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [1,4,3,0,7,2](1) acting [1,4,3,0,7,2](1)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:03.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.040507450s + 0s) 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.604+0000 7f8f0117b700 1 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [1,4,3,0,7,2] -> [1,4,3,0,7,2], acting [1,4,3,0,7,2] -> [1,2147483647,2147483647,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:03.604+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.604+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.604+0000 7f8f0117b700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.604+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.605530+0000 1 0.000605 2019-12-07T19:34:03.604+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.604+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=26 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=26 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=26 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.604+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 13 ==== pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (crc 0 0 0) 0x558c574a0a00 con 0x558c54c64400 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=26 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [3,2,7,1,4,5] newacting [3,2147483647,7,1,2147483647,5] 2019-12-07T19:34:03.604+0000 7f8f23c56700 15 osd.7 26 enqueue_peering_evt 1.2s4 epoch_sent: 26 epoch_requested: 26 MQuery 1.2s4 from 0(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.604+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.2s4 from 0(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.604+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 14 ==== pg_query2(1.cs0 1.cs0 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c5756f180 con 0x558c54c19c00 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=26 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]p3(0) r=2 lpr=26 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.604+0000 7f8f24457700 15 osd.7 26 enqueue_peering_evt 1.cs0 epoch_sent: 26 epoch_requested: 26 MQuery 1.cs0 from 5(1) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.604+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.cs0 from 5(1) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.604+0000 7f8f0197c700 15 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,25)/1 crt=23'13 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [3,2,7,1,4,5](3) acting [3,2,7,1,4,5](3)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [3,2,7,1,4,5](3) acting [3,2,7,1,4,5](3)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2))) 2019-12-07T19:34:03.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.2s4 from 0(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.2s4 from 0(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.604+0000 7f8f03980700 10 osd.7 26 handle_pg_query_nopg 1.2s4 2019-12-07T19:34:03.604+0000 7f8f03980700 10 osd.7 26 pg 1.2s4 dne 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558c54ea3ce0] 2019-12-07T19:34:03.604+0000 7f8f03980700 10 osd.7 26 send_incremental_map 25 -> 26 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.041116463s + 0s) 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.604+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(26..26 src has 1..26) v4 -- 0x558c576c0000 con 0x558c54c64400 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.604+0000 7f8f03980700 10 osd.7 26 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.604+0000 7f8f0197c700 1 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [3,2,7,1,4,5] -> [3,2,7,1,4,5], acting [3,2,7,1,4,5] -> [3,2147483647,7,1,2147483647,5], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 2 -> 2, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.604+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_notify((query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 26) v7 -- 0x558c575cde00 con 0x558c54c64400 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:60000000::::0,1:70000000::::head) 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.604+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.605836+0000 1 0.001013 2019-12-07T19:34:03.604+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.604+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.604+0000 7f8f0197c700 1 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.604+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.605871+0000 0 0.000000 2019-12-07T19:34:03.604+0000 7f8f0197c700 5 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 26 send_incremental_map 25 -> 26 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:03.604+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(26..26 src has 1..26) v4 -- 0x558c5742c240 con 0x558c54c5bc00 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 osd.7 26 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.604+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:26 sent:26 1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e26/26) v1 -- 0x558c54e98000 con 0x558c54c5bc00 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740b500 seq 30 2019-12-07T19:34:03.604+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes shared_blobs 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[] released 0x[] 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:03.604+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:03.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4f3b6~522 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4f3b6 0x4f3b6~522 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1f3b6 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3b6 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8d8 and padding block with 0x728 2019-12-07T19:34:03.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.cs0 from 5(1) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.cs0 from 5(1) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558c57676000 2019-12-07T19:34:03.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.cs0 from 5(1) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] sending info 2019-12-07T19:34:03.604+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 14 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (crc 0 0 0) 0x558c57580480 con 0x558c54c64400 2019-12-07T19:34:03.604+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7f8f1199c700 20 osd.7 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.604+0000 7f8f1199c700 10 osd.7 26 do_waiters -- start 2019-12-07T19:34:03.604+0000 7f8f1199c700 10 osd.7 26 do_waiters -- finish 2019-12-07T19:34:03.604+0000 7f8f1199c700 20 osd.7 26 _dispatch 0x558c57580480 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.604+0000 7f8f1199c700 3 osd.7 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.604+0000 7f8f1199c700 10 osd.7 26 no new maps here, dropping 2019-12-07T19:34:03.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_notify2(1.cs1 1.cs1 (query:26 sent:26 1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 0->1 ([18,25] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e26/26) v1 -- 0x558c576ba300 con 0x558c54c19c00 2019-12-07T19:34:03.604+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 15 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558c575c78c0 con 0x558c54c19c00 2019-12-07T19:34:03.604+0000 7f8f1199c700 20 osd.7 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.604+0000 7f8f1199c700 10 osd.7 26 do_waiters -- start 2019-12-07T19:34:03.604+0000 7f8f1199c700 10 osd.7 26 do_waiters -- finish 2019-12-07T19:34:03.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:03.604+0000 7f8f1199c700 20 osd.7 26 _dispatch 0x558c575c78c0 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.604+0000 7f8f1199c700 3 osd.7 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.604+0000 7f8f1199c700 10 osd.7 26 no new maps here, dropping 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4f8d8 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:03.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000159524s (0.000000091s flush + 0.000159433s kv commit) 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:03.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:03.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:03.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:03.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:03.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:03.604+0000 7f8f0117b700 1 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.605+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.606247+0000 0 0.000000 2019-12-07T19:34:03.605+0000 7f8f0117b700 5 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:26 sent:26 1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e26/26) v1 -- 0x558c54e9aa00 con 0x558c54c64000 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d58f00 seq 25 2019-12-07T19:34:03.605+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes shared_blobs 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[] released 0x[] 2019-12-07T19:34:03.605+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:34:03.605+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4f8d8~522 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4f8d8 0x4f8d8~522 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1f8d8 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8d8 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdfa and padding block with 0x206 2019-12-07T19:34:03.605+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 74 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (crc 0 0 0) 0x558c5756f180 con 0x558c54c64000 2019-12-07T19:34:03.605+0000 7f8f24457700 15 osd.7 26 enqueue_peering_evt 1.9s4 epoch_sent: 26 epoch_requested: 26 MQuery 1.9s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.605+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.9s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.605+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 75 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (crc 0 0 0) 0x558c55195200 con 0x558c54c64000 2019-12-07T19:34:03.605+0000 7f8f1199c700 20 osd.7 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7f8f1199c700 10 osd.7 26 do_waiters -- start 2019-12-07T19:34:03.605+0000 7f8f1199c700 10 osd.7 26 do_waiters -- finish 2019-12-07T19:34:03.605+0000 7f8f1199c700 20 osd.7 26 _dispatch 0x558c55195200 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 46 ==== pg_query2(1.6s2 1.6s2 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c574a0a00 con 0x558c54c5bc00 2019-12-07T19:34:03.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7f8f1199c700 3 osd.7 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.605+0000 7f8f1199c700 10 osd.7 26 no new maps here, dropping 2019-12-07T19:34:03.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.9s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7f8f23c56700 15 osd.7 26 enqueue_peering_evt 1.6s2 epoch_sent: 26 epoch_requested: 26 MQuery 1.6s2 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.9s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558c5500a000 2019-12-07T19:34:03.605+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.6s2 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.605+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.9s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.605+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 47 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558c5742c240 con 0x558c54c5bc00 2019-12-07T19:34:03.605+0000 7f8f04181700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.605+0000 7f8f1199c700 20 osd.7 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7f8f1199c700 10 osd.7 26 do_waiters -- start 2019-12-07T19:34:03.605+0000 7f8f1199c700 10 osd.7 26 do_waiters -- finish 2019-12-07T19:34:03.605+0000 7f8f1199c700 20 osd.7 26 _dispatch 0x558c5742c240 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7f8f1199c700 3 osd.7 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.605+0000 7f8f1199c700 10 osd.7 26 no new maps here, dropping 2019-12-07T19:34:03.605+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4fdfa 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:03.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000320346s (0.000000090s flush + 0.000320256s kv commit) 2019-12-07T19:34:03.605+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:26 sent:26 1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e26/26) v1 -- 0x558c54e9a300 con 0x558c54c64000 2019-12-07T19:34:03.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00> 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:34:03.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:34:03.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:03.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:34:03.605+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:03.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:03.605+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.6s2 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.605+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.6s2 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558c54d5c000 2019-12-07T19:34:03.605+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.6s2 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.605+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.605+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:26 sent:26 1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e26/26) v1 -- 0x558c574e4700 con 0x558c54c5bc00 2019-12-07T19:34:03.605+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 76 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (crc 0 0 0) 0x558c5756f180 con 0x558c54c64000 2019-12-07T19:34:03.610+0000 7f8f24457700 15 osd.7 26 enqueue_peering_evt 1.7s4 epoch_sent: 26 epoch_requested: 26 MQuery 1.7s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.610+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.7s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.7s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.7s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558c54d5f000 2019-12-07T19:34:03.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.7s4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.610+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:03.610+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:26 sent:26 1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e26/26) v1 -- 0x558c54e98e00 con 0x558c54c64000 2019-12-07T19:34:03.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 10 osd.7 26 tick_without_osd_lock 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fd94000/0x40000000/0x1680000000, data 0x26c1c2/0x26a000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 25 osd.7 26 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 25 osd.7 26 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:03.373892+0000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 25 osd.7 26 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 25 osd.7 26 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373810+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 25 osd.7 26 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 25 osd.7 26 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 20 osd.7 26 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 20 osd.7 26 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 20 osd.7 26 scrub_load_below_threshold loadavg per cpu 0.0625 < max 5 = yes 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 20 osd.7 26 sched_scrub load_is_low=1 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 10 osd.7 26 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:03.623677+0000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 20 osd.7 26 sched_scrub done 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 10 osd.7 26 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 20 osd.7 26 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:03.622+0000 7f8f1fbbe700 10 osd.7 26 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:03.624+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:34:03.624+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:03.624+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 15 0x2000~1000 2019-12-07T19:34:03.624+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 55839bb3 2019-12-07T19:34:03.624+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 15 0x1b000~1000 2019-12-07T19:34:03.624+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 9316858d 2019-12-07T19:34:03.624+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.624+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:03.624+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:03.674+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.724+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.774+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.824+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.874+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.925+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.942+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.409567154s delta_ub 0.031097589s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c54c5ac00 2019-12-07T19:34:03.942+0000 7f8f23c56700 20 osd.7 26 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030725849s,0.031097589s]) 2019-12-07T19:34:03.942+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.378841305s delta_ub 18446744073.678825767s) v5 -- 0x558c5768b200 con 0x558c54c5ac00 2019-12-07T19:34:03.942+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.409567154s delta_ub 0.031097589s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c54c5a400 2019-12-07T19:34:03.942+0000 7f8f24c58700 20 osd.7 26 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030706919s,0.031097589s]) 2019-12-07T19:34:03.942+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.378860235s delta_ub 18446744073.678844697s) v5 -- 0x558c54c58600 con 0x558c54c5a400 2019-12-07T19:34:03.975+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.025+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.075+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:04.075+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.125+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.175+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.225+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.275+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.325+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.376+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.388+0000 7f8f21451700 10 osd.7 26 tick 2019-12-07T19:34:04.388+0000 7f8f21451700 10 osd.7 26 do_waiters -- start 2019-12-07T19:34:04.388+0000 7f8f21451700 10 osd.7 26 do_waiters -- finish 2019-12-07T19:34:04.388+0000 7f8f21451700 20 osd.7 26 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:04.425+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.448+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:04.476+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.526+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.576+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 232109 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.602+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 17 ==== osd_map(26..27 src has 1..27) v4 ==== 1205+0+0 (crc 0 0 0) 0x558c576c0000 con 0x558c5476d000 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 osd.7 26 OSD::ms_dispatch: osd_map(26..27 src has 1..27) v4 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 osd.7 26 do_waiters -- start 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 osd.7 26 do_waiters -- finish 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 osd.7 26 _dispatch 0x558c576c0000 osd_map(26..27 src has 1..27) v4 2019-12-07T19:34:04.602+0000 7f8f164bb700 3 osd.7 26 handle_osd_map epochs [26,27], i have 26, src has [1,27] 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 osd.7 26 handle_osd_map got inc map for epoch 27 2019-12-07T19:34:04.602+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:35a0e826:::osdmap.26:0# 0x0~0 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1671 size 0x1671 (5745) 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265c00 blob([0x26a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575bdb90 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55264700 blob([!~1000,0x26b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558c54677030 sbid 0x0)) need 0x1000~671 cache has 0x[1000~671] 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:35a0e826:::osdmap.26:0# 0x0~1671 = 5745 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 osd.7 26 add_map_bl 26 5745 bytes 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 osd.7 26 got_full_map 27, nothing requested 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 osd.7 26 handle_osd_map pg_num_history pg_num_history(e27 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {27={}} 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 osd.7 26 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e27 [1,27] lci=[13,27]) 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c574fe000 seq 18 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c63b972f:::inc_osdmap.27:0# key 0x7f7fffffffffffffffc63b97'/!inc_osdmap.27!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:04.602+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1113 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~11c 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xee4 on front/back, now 0x0~1000 2019-12-07T19:34:04.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574fe000 1 blobs 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x26c000~1000/1000,1000,0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x26c000~1000] 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75110 blob([0x26c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3d50 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54d75110) get_ref 0x0~11c Blob(0x558c54d75110 blob([0x26c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3d50 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54d75110) get_ref init 0x1000, 1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~11c: 0x0~11c Blob(0x558c54d75110 blob([0x26c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x11c) SharedBlob(0x558c54ea3d50 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea3d68 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x11c 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756f508) dirty_range mark inline shard dirty 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c = 0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3ea0e826:::osdmap.27:0# key 0x7f7fffffffffffffff3ea0e8'&!osdmap.27!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:04.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ea0e826:::osdmap.27:0# 0x0~1671 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1114 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3ea0e826:::osdmap.27:0# 0x0~1671 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~671 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x98f on front/back, now 0x0~1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574fe000 2 blobs 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x26d000~2000/1000,2000,0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x26d000~2000] 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57567420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575673b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57567420 blob([0x26d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575673b0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57567420) get_ref 0x0~1000 Blob(0x558c57567420 blob([0x26d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575673b0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57567420) get_ref init 0x1000, 1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57567420 blob([0x26d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575673b0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c575673c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57567340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575672d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57567340 blob([!~1000,0x26e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575672d0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57567340) get_ref 0x1000~671 Blob(0x558c57567340 blob([!~1000,0x26e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c575672d0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57567340) get_ref init 0x2000, 1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~671: 0x1000~671 Blob(0x558c57567340 blob([!~1000,0x26e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558c575672d0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c575672e8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1671 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756fa08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ea0e826:::osdmap.27:0# 0x0~1671 = 0 2019-12-07T19:34:04.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574fe000 0 blobs 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:04.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:04.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:04.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574fe000 0 blobs 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe000 cost 23510 (4 ios * 4000 + 7510 bytes) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe000 onodes 0x558c53a91900,0x558c55276c80,0x558c5756f400,0x558c5756f900 shared_blobs 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756f508) update #-1:c63b972f:::inc_osdmap.27:0# 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756f508) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c63b972f:::inc_osdmap.27:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756fa08) update #-1:3ea0e826:::osdmap.27:0# 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756fa08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3ea0e826:::osdmap.27:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe000 allocated 0x[26c000~3000] released 0x[] 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 freelist allocate 0x26c000~3000 2019-12-07T19:34:04.603+0000 7f8f164bb700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 prepare 2019-12-07T19:34:04.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c574fe000 2019-12-07T19:34:04.603+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 aio_wait 2019-12-07T19:34:04.603+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe000 2019-12-07T19:34:04.603+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 io_done 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 518 < min_flush_size 524288 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f400 had 1 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f900 had 1 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860529152 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998711) bluestore 86 GiB free (0.959973), bluefs_ratio 0.039952 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8871 < min_flush_size 524288 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8871 < min_flush_size 524288 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4fdfa~22a7 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4fdfa 0x4fdfa~22a7 to file(ino 16 size 0x50000 mtime 2019-12-07T19:34:00.665348+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _allocate len 0x20a1 from 0 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _allocate len 0x20a1 from 1 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd180000 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd180000~10000/10000,10000,acd180000 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x60000 to include allocated 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 27 (was clean) 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x1fdfa 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdfa 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa1 and padding block with 0xf5f 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x60000 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x520a1 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (27) on file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]), flushing log 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 27 len 0x49 crc 0x7702669) 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x1a000~1000 to file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x1a000 0x1a000~1000 to file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1a000 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x1b000 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:04.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000017 2019-12-07T19:34:04.603+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 48 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c574e4700 con 0x558c54c5bc00 2019-12-07T19:34:04.606+0000 7f8f23c56700 15 osd.7 26 enqueue_peering_evt 1.6s2 epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s) 2019-12-07T19:34:04.606+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s)) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 27, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 49 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558c5742db00 con 0x558c54c5bc00 2019-12-07T19:34:04.606+0000 7f8f1199c700 20 osd.7 26 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.606+0000 7f8f1199c700 10 osd.7 26 do_waiters -- start 2019-12-07T19:34:04.606+0000 7f8f1199c700 10 osd.7 26 do_waiters -- finish 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f1199c700 20 osd.7 26 _dispatch 0x558c5742db00 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.606+0000 7f8f1199c700 3 osd.7 26 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.606+0000 7f8f1199c700 10 osd.7 26 no new maps here, dropping 2019-12-07T19:34:04.606+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 27 2019-12-07T19:34:04.606+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.002801632s (0.000038128s flush + 0.002763504s kv commit) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe000> 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c5525f680> 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 kv_submitted 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe000 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c574fe000 osr 0x558c54ea4b40 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7 26 _committed_osd_maps 27..27 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7 26 advance to epoch 27 (<= last 27 <= newest_map 27) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5754d7c0) prepare_write seq 16 0x1b000~1000 crc 89c92970 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5754d7c0) prepare_write seq 16 0x2000~1000 crc eeb88f8e 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 deferred_cleanup 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6900 onodes 0x558c53a91900,0x558c55149680,0x558c55149900,0x558c55276c80 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552642b8 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8cf0 space 0x558c552642b8 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 7 osd.7 27 consume_map version 27 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552647f8 in 0x558c53a59340) _finish_write discard buffer(0x558c548deab0 space 0x558c552647f8 0x1000~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.0s3 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0e88 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8b40 space 0x558c574e0e88 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.9s4 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:2.identify_splits_and_merges identify_splits_and_merges slot 1.2s4 has no pg and waiting_for_split 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs3 has no pg and waiting_for_split 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.3s4 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.cs0 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 done 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.4s0 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20900 deferred_cleanup 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.ds2 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6900 [] 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.6s2 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 identify_splits_and_merges 1.7s4 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 27 prune_pg_created 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:0.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:1.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 deferred_cleanup 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:2.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20900 onodes 0x558c53a91900,0x558c54d2d400,0x558c54d2d680,0x558c55276c80 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677a58 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8a20 space 0x558c54677a58 0x1000~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 empty, pruning 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c248 in 0x558c53a59340) _finish_write discard buffer(0x558c551a87e0 space 0x558c54d3c248 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:3.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55265358 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8b40 space 0x558c55265358 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 empty, pruning 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20900 done 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 deferred_cleanup 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20900 [] 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:4.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:5.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:6.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 deferred_cleanup 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 0x558c53a91900,0x558c55149b80,0x558c55276c80,0x558c5756e000 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ee88 in 0x558c53a59340) _finish_write discard buffer(0x558c548decf0 space 0x558c5757ee88 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f438 in 0x558c53a59340) _finish_write discard buffer(0x558c548ded80 space 0x558c5757f438 0x1000~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bd828 in 0x558c53a59340) _finish_write discard buffer(0x558c548dec60 space 0x558c575bd828 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7:7.consume_map 27 (was 26) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 deferred_cleanup 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.606+0000 7f8f141a1700 10 osd.7 27 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 deferred_cleanup 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff200 onodes 0x558c53a91900,0x558c54d2d900,0x558c54d2db80,0x558c55276c80 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677048 in 0x558c53a59340) _finish_write discard buffer(0x558c548df050 space 0x558c54677048 0x1000~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.0s3 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1a58 in 0x558c53a59340) _finish_write discard buffer(0x558c548de1b0 space 0x558c574f1a58 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bdba8 in 0x558c53a59340) _finish_write discard buffer(0x558c548de2d0 space 0x558c575bdba8 0x0~1000 writing nocache) 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s)) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 done 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.9s4 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 deferred_queued 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff200 [] 2019-12-07T19:34:04.606+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.4s0 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.ds2 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.6s2 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f141a1700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f141a1700 7 osd.7 27 activate_map version 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c5500d000 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 26 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5] -- 3/3 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c5500a000 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f04982700 10 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f04982700 10 osd.7 pg_epoch: 26 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.606+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 pg_epoch: 26 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,2147483647] -- 1/1 2019-12-07T19:34:04.606+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062329548s int 16s) 2019-12-07T19:34:04.606+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 27 2019-12-07T19:34:04.606+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.3(0) 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.062329548s int 16s) 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.022013251s -> ruub 44.040316297s 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f0918b700 20 osd.7 27 agent_entry empty queue 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.040316297s 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c55141000 2019-12-07T19:34:04.606+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f0197c700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.607715+0000 5 0.000100 2019-12-07T19:34:04.606+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 27 send_incremental_map 26 -> 27 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:04.606+0000 7f8f06185700 10 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~0 2019-12-07T19:34:04.606+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~11c size 0x11c (284) 2019-12-07T19:34:04.606+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:04.606+0000 7f8f06185700 10 osd.7 pg_epoch: 26 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:04.606+0000 7f8f0197c700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.606+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75110 blob([0x26c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x11c) SharedBlob(0x558c54ea3d50 sbid 0x0)) need 0x0~11c cache has 0x[0~11c] 2019-12-07T19:34:04.606+0000 7f8f06185700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.606+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c = 284 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 27 add_map_inc_bl 27 284 bytes 2019-12-07T19:34:04.606+0000 7f8f06185700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(27..27 src has 1..27) v4 -- 0x558c5524c000 con 0x558c54c64000 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 23'13 -> 23'13 2019-12-07T19:34:04.606+0000 7f8f06185700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.606+0000 7f8f08189700 10 osd.7 27 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.606+0000 7f8f06185700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 27 2019-12-07T19:34:04.606+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f06185700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:27 sent:27 1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e27/27) v1 -- 0x558c54e99c00 con 0x558c54c64000 2019-12-07T19:34:04.606+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c5513e000 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 27 send_incremental_map 26 -> 27 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:04.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(27..27 src has 1..27) v4 -- 0x558c5742c000 con 0x558c54c5bc00 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 osd.7 27 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 pg_epoch: 26 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,2147483647] -- 4/4 2019-12-07T19:34:04.606+0000 7f8f05183700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:27 sent:27 1.6s2( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e27/27) v1 -- 0x558c574e4000 con 0x558c54c5bc00 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f05183700 10 osd.7 pg_epoch: 26 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2] -- 1/1 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0c00 seq 31 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:04.606+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.606+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes shared_blobs 2019-12-07T19:34:04.606+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:04.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.606+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 27 send_incremental_map 26 -> 27 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:04.606+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(27..27 src has 1..27) v4 -- 0x558c54d2fb00 con 0x558c54c5b400 2019-12-07T19:34:04.606+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:27 sent:27 1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e27/27) v1 -- 0x558c57452a00 con 0x558c54c64000 2019-12-07T19:34:04.606+0000 7f8f07187700 10 osd.7 27 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.606+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:27 sent:27 1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4)))) e27/27) v1 -- 0x558c576b8700 con 0x558c54c5b400 2019-12-07T19:34:04.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c57676000 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 26 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] handle_advance_map [7,5,4,1,2,3]/[2147483647,5,4,1,2,3] -- 7/5 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 27 send_incremental_map 26 -> 27 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:04.606+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(27..27 src has 1..27) v4 -- 0x558c54e6e480 con 0x558c54c19c00 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 27 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.606+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_notify2(1.cs1 1.cs1 (query:27 sent:27 1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 0->1 ([18,25] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([18,23] acting 1(3),2(4),3(5),4(2),5(1),6(0)))) e27/27) v1 -- 0x558c57450e00 con 0x558c54c19c00 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558c548ab000 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 26 -> 27 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 26 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active advmap 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] Active: kicking snap trim 2019-12-07T19:34:04.606+0000 7f8f06986700 7 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.606+0000 7f8f06986700 15 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] publish_stats_to_osd 25:25 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 27 2019-12-07T19:34:04.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x520a1~522 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x520a1 0x520a1~522 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x220a1 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa1 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5c3 and padding block with 0xa3d 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x525c3 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.607+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000117526s (0.000000059s flush + 0.000117467s kv commit) 2019-12-07T19:34:04.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:04.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:04.607+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 77 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c64000 2019-12-07T19:34:04.607+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.9s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s) 2019-12-07T19:34:04.607+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 78 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57452a00 con 0x558c54c64000 2019-12-07T19:34:04.607+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s) 2019-12-07T19:34:04.607+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) pg 0x558c5500a000 2019-12-07T19:34:04.607+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s) 2019-12-07T19:34:04.607+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.1(0) 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.607+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.062707990s int 16s) 2019-12-07T19:34:04.607+0000 7f8f04181700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.021357758s -> ruub 44.041350232s 2019-12-07T19:34:04.607+0000 7f8f04181700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.041350232s 2019-12-07T19:34:04.607+0000 7f8f04181700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.608884+0000 6 0.000076 2019-12-07T19:34:04.607+0000 7f8f04181700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.607+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7f8f04181700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.607+0000 7f8f05984700 15 osd.7 27 enqueue_peering_evt 1.6s2 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.607+0000 7f8f05984700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.607+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'6 -> 21'6 2019-12-07T19:34:04.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 43 ==== PGlog(1.bs3 log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) e27/26) v6 ==== 7462+0+0 (unknown 0 0 0) 0x558c575bf000 con 0x558c54c5b400 2019-12-07T19:34:04.607+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.607+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.607+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:04.607+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.607+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info 2019-12-07T19:34:04.607+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:04.607+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 crt=23'13 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.607+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d20600 seq 17 2019-12-07T19:34:04.607+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:04.607+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:04.607+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.6s0 1.6s0 1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558c576b8000 con 0x558c54c5bc00 2019-12-07T19:34:04.607+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:04.607+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:04.607+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:04.607+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:04.607+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:04.607+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes shared_blobs 2019-12-07T19:34:04.607+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 44 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c57452a00 con 0x558c54c5b400 2019-12-07T19:34:04.607+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:04.607+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:04.607+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:04.607+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.057306578s int 16s) 2019-12-07T19:34:04.607+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.057306578s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.607+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 79 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (crc 0 0 0) 0x558c57377680 con 0x558c54c64000 2019-12-07T19:34:04.607+0000 7f8f1199c700 20 osd.7 27 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.607+0000 7f8f1199c700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:04.607+0000 7f8f1199c700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:04.607+0000 7f8f1199c700 20 osd.7 27 _dispatch 0x558c57377680 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 no pg, should create on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 osd.7 27 _make_pg 1.bs3 2019-12-07T19:34:04.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.b(unlocked)] enter Initial 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.607+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.607+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s) 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 26 pg[1.b(unlocked)] enter NotTrimming 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.bs3_head fresh osr 0x558c576a28c0 2019-12-07T19:34:04.607+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.1(0) 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.607+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.063255566s int 16s) 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [4,0,2,7,1,5] acting [4,2147483647,2,2147483647,1,5] history ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26 past_intervals ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:04.607+0000 7f8f0117b700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.021357758s -> ruub 44.041897808s 2019-12-07T19:34:04.607+0000 7f8f0117b700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.041897808s 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:04.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.607+0000 7f8f0117b700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.609180+0000 6 0.000785 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:04.607+0000 7f8f0117b700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 28.044633190s + 0s) 2019-12-07T19:34:04.608+0000 7f8f0117b700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.608+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'9 -> 21'9 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:04.609243+0000 0 0.000000 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740af00 seq 26 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:04.608+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x525c3~517 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x525c3 0x525c3~517 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x225c3 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5c3 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 26 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xada and padding block with 0x526 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:04.608+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.608+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:04.609295+0000 1 0.000039 2019-12-07T19:34:04.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:04.608+0000 7f8f0317f700 1 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:04.609320+0000 0 0.000000 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 26 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5740a000 seq 1 2019-12-07T19:34:04.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.bs3_head bits 4 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.bs3_head bits 4 = 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:d0000000::::head# key 0x838000000000000001d000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x52ada 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000126989s (0.000000139s flush + 0.000126850s kv commit) 2019-12-07T19:34:04.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1115 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20600> 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._infover' <- _infover 2019-12-07T19:34:04.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:04.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:04.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7f8f08189700 15 osd.7 27 enqueue_peering_evt 1.9s4 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7f8f08189700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:04.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:04.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x52ada~522 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558c5500a000 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:04.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x52ada 0x52ada~522 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x22ada 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xada 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 5945 (1 ios * 4000 + 1945 bytes) 2019-12-07T19:34:04.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xffc and padding block with 0x4 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes 0x558c54d2c280 shared_blobs 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54d2c388) update 3#1:d0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54d2c388) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.608+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:d0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[] released 0x[] 2019-12-07T19:34:04.608+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:34:04.608+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 27 handle_pg_create_info new pg pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7:3._attach_pg 1.bs3 0x558c54d9b000 2019-12-07T19:34:04.608+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558c576de000 con 0x558c54c64000 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 osd.7 27 identify_splits_and_merges 1.bs3 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7:3._prime_splits 2019-12-07T19:34:04.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7f8f1199c700 3 osd.7 27 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7f8f1199c700 10 osd.7 27 no new maps here, dropping 2019-12-07T19:34:04.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.057306578s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.057306578s int 16s)) prio 255 cost 10 e27) pg 0x558c5513e000 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map: 27 2019-12-07T19:34:04.608+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 45 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558c5742d680 con 0x558c54c5b400 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 26 -> 27 2019-12-07T19:34:04.608+0000 7f8f1199c700 20 osd.7 27 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.608+0000 7f8f1199c700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:04.608+0000 7f8f1199c700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:04.608+0000 7f8f1199c700 20 osd.7 27 _dispatch 0x558c5742d680 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.608+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.057306578s int 16s) 2019-12-07T19:34:04.608+0000 7f8f1199c700 3 osd.7 27 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.608+0000 7f8f1199c700 10 osd.7 27 no new maps here, dropping 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 26 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5] -- 4/4 2019-12-07T19:34:04.608+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.4(0) 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 27 2019-12-07T19:34:04.608+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.057306578s int 16s) 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 osd.7 pg_epoch: 27 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:04.608+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.013703655s -> ruub 44.043602923s 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info 2019-12-07T19:34:04.608+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.043602923s 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,21'12], crt=21'12) 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 merge_log log((0'0,21'12], crt=21'12) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 merge_log extending head to 21'12 2019-12-07T19:34:04.608+0000 7f8f07187700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.609530+0000 6 0.000091 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 23'10 -> 23'10 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 ? 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 merge_log original_crt = 0'0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:04.608+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740b800 seq 24 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:04.608+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b800 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b800 onodes shared_blobs 2019-12-07T19:34:04.608+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b800 allocated 0x[] released 0x[] 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 prepare 2019-12-07T19:34:04.608+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b800 2019-12-07T19:34:04.608+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 io_done 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 update missing, append 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 merge_log result log((0'0,21'12], crt=21'12) missing(4 may_include_deletes = 1) changed=1 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.609670+0000 3 0.000051 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] activate - not complete, missing(4 may_include_deletes = 1) 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.608+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730a000 seq 2 2019-12-07T19:34:04.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.608+0000 7f8f0317f700 20 bluestore.onode(0x558c54d2c280).flush flush cnt:1 2019-12-07T19:34:04.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 16 ==== PGlog(1.cs0 log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 44.076045274s int 16s) e27/26) v6 ==== 6522+0+0 (unknown 0 0 0) 0x558c551dd800 con 0x558c54c19c00 2019-12-07T19:34:04.608+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 27 epoch_requested: 26 MLogRec from 5(1) log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 44.076045274s int 16s) +create_info 2019-12-07T19:34:04.608+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 5(1) log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 44.076045274s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 5(1) log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 44.076045274s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 5(1) log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 44.076045274s int 16s) +create_info) prio 255 cost 10 e27) pg 0x558c57676000 2019-12-07T19:34:04.608+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 26 MLogRec from 5(1) log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 44.076045274s int 16s) +create_info 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 remapped NOTIFY mbc={}] state: got info+log from osd.5(1) 1.cs1( v 21'12 (0'0,21'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,21'12], crt=21'12) 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 merge_log log((0'0,21'12], crt=21'12) from osd.5(1) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 merge_log extending head to 21'12 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'12 (21'6) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:166 0.000000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'11 (21'10) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:161 2019-12-07T19:33:58.808061+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'10 (21'9) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:160 2019-12-07T19:33:58.808039+0000 0 ObjectCleanRegions clean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'9 (21'5) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:159 2019-12-07T19:33:58.805739+0000 0 ObjectCleanRegions clean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'8 (21'7) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:68 2019-12-07T19:33:58.673986+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'7 (0'0) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:67 2019-12-07T19:33:58.673753+0000 0 ObjectCleanRegions clean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'6 (21'3) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:26 2019-12-07T19:33:58.630747+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'5 (21'2) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:5 2019-12-07T19:33:58.625925+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'4 (21'1) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:2 2019-12-07T19:33:58.625650+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'3 (0'0) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:25 2019-12-07T19:33:58.630729+0000 0 ObjectCleanRegions clean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'2 (0'0) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:4 2019-12-07T19:33:58.625910+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 ? 21'1 (0'0) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:1 2019-12-07T19:33:58.625551+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 merge_log original_crt = 0'0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'1 (0'0) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:1 2019-12-07T19:33:58.625551+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'2 (0'0) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:4 2019-12-07T19:33:58.625910+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'3 (0'0) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:25 2019-12-07T19:33:58.630729+0000 0 ObjectCleanRegions clean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'4 (21'1) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:2 2019-12-07T19:33:58.625650+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'5 (21'2) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:5 2019-12-07T19:33:58.625925+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'6 (21'3) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:26 2019-12-07T19:33:58.630747+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'7 (0'0) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:67 2019-12-07T19:33:58.673753+0000 0 ObjectCleanRegions clean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'8 (21'7) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:68 2019-12-07T19:33:58.673986+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'9 (21'5) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:159 2019-12-07T19:33:58.805739+0000 0 ObjectCleanRegions clean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'10 (21'9) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:160 2019-12-07T19:33:58.808039+0000 0 ObjectCleanRegions clean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'11 (21'10) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:161 2019-12-07T19:33:58.808061+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 update missing, append 21'12 (21'6) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:166 0.000000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 merge_log result log((0'0,21'12], crt=21'12) missing(4 may_include_deletes = 1) changed=1 2019-12-07T19:34:04.609+0000 7f8f0297e700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY m=4 mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.610354+0000 6 0.000080 2019-12-07T19:34:04.609+0000 7f8f0297e700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.609+0000 7f8f0297e700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY m=4 mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped m=4 mbc={} ps=[2~1]] activate - not complete, missing(4 may_include_deletes = 1) 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 remapped m=4 mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.609+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d59b00 seq 3 2019-12-07T19:34:04.609+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore.onode(0x558c5756f680).flush flush done 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x0000000000000456'.0000000000.00000000000000000000' end: 0x0000000000000456'.4294967295.18446744073709551615' 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.missing/0000000000000001.C311C407.head.smithi13140796-1..' <- missing/0000000000000001.C311C407.head.smithi13140796-1.. 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.missing/0000000000000001.C5A5CFAD.head.smithi13140796-23..' <- missing/0000000000000001.C5A5CFAD.head.smithi13140796-23.. 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.missing/0000000000000001.CCFADF08.head.smithi13140796-2..' <- missing/0000000000000001.CCFADF08.head.smithi13140796-2.. 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.missing/0000000000000001.CFE5E24B.head.smithi13140796-9..' <- missing/0000000000000001.CFE5E24B.head.smithi13140796-9.. 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 12234 (1 ios * 4000 + 8234 bytes) 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:04.609+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:04.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.609+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 15 ==== PGlog(1.2s4 log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) e27/26) v6 ==== 5347+0+0 (crc 0 0 0) 0x558c575bf800 con 0x558c54c64400 2019-12-07T19:34:04.610+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.2s4 epoch_sent: 27 epoch_requested: 26 MLogRec from 0(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) +create_info 2019-12-07T19:34:04.610+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 0(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 0(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 0(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 no pg, should create on OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 0(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 27 _make_pg 1.2s4 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2(unlocked)] enter Initial 2019-12-07T19:34:04.610+0000 7f8f07988700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.610+0000 7f8f07988700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.610+0000 7f8f07988700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 pg_epoch: 26 pg[1.2(unlocked)] enter NotTrimming 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.2s4_head fresh osr 0x558c54e8fe00 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [0,4,1,5,7,3] acting [0,4,1,2147483647,2147483647,3] history ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26 past_intervals ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 28.046996850s + 0s) 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:04.610+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:04.611599+0000 0 0.000000 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 26 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:04.611651+0000 1 0.000041 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:04.610+0000 7f8f07988700 1 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:04.611676+0000 0 0.000000 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 26 2019-12-07T19:34:04.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5730b200 seq 1 2019-12-07T19:34:04.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.2s4_head bits 4 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.2s4_head bits 4 = 0 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:40000000::::head# key 0x8480000000000000014000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1116 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._infover' <- _infover 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 5945 (1 ios * 4000 + 1945 bytes) 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes 0x558c574a0c80 shared_blobs 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0d88) update 4#1:40000000::::head# 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0d88) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:40000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[] released 0x[] 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:04.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 27 handle_pg_create_info new pg pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7:2._attach_pg 1.2s4 0x558c5731a000 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 27 identify_splits_and_merges 1.2s4 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7:2._prime_splits 2019-12-07T19:34:04.610+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map: 27 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s4 26 -> 27 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 26 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3] -- 0/0 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x52ffc 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002480615s (0.000000098s flush + 0.002480517s kv commit) 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00> 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 27 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1539 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [] 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.610+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 26 MLogRec from 0(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) +create_info 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2819 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.0(0) 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,23'10], crt=23'10) 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:04.610+0000 7f8f07988700 10 merge_log log((0'0,23'10], crt=23'10) from osd.0(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore.onode(0x558c54d2c280).flush flush done 2019-12-07T19:34:04.610+0000 7f8f07988700 10 merge_log extending head to 23'10 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10875 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12414 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12437 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12437 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000045b'.0000000000.00000000000000000000' end: 0x000000000000045b'.4294967295.18446744073709551615' 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x52ffc~3095 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x52ffc 0x52ffc~3095 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x22ffc 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xffc 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x91 and padding block with 0xf6f 2019-12-07T19:34:04.610+0000 7f8f07988700 20 ? 21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f07988700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:04.610+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.610+0000 7f8f07988700 20 merge_log original_crt = 0'0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:34:04.610+0000 7f8f05183700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:04.610+0000 7f8f05183700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:04.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.missing/0000000000000001.B0E469FE.head.smithi13140796-28..' <- missing/0000000000000001.B0E469FE.head.smithi13140796-28.. 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.missing/0000000000000001.B376C33C.head.smithi13140796-50..' <- missing/0000000000000001.B376C33C.head.smithi13140796-50.. 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.missing/0000000000000001.BC8CBBB8.head.smithi13140796-49..' <- missing/0000000000000001.BC8CBBB8.head.smithi13140796-49.. 2019-12-07T19:34:04.610+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.missing/0000000000000001.BD1A381A.head.smithi13140796-18..' <- missing/0000000000000001.BD1A381A.head.smithi13140796-18.. 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a000 cost 13019 (1 ios * 4000 + 9019 bytes) 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a000 onodes shared_blobs 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a000 allocated 0x[] released 0x[] 2019-12-07T19:34:04.610+0000 7f8f07988700 20 update missing, append 23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.610+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558c57452300 con 0x558c54c64000 2019-12-07T19:34:04.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 prepare 2019-12-07T19:34:04.610+0000 7f8f07988700 10 merge_log result log((0'0,23'10], crt=23'10) missing(4 may_include_deletes = 1) changed=1 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a000 2019-12-07T19:34:04.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 io_done 2019-12-07T19:34:04.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=4 mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.612029+0000 3 0.000051 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x56091 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000190072s (0.000000081s flush + 0.000189991s kv commit) 2019-12-07T19:34:04.610+0000 7f8f07988700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=4 mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8047 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000,0x558c5740b800,0x558c54d59b00,0x558c5730b200> 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8852 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=4 mbc={} ps=[2~1]] activate - not complete, missing(4 may_include_deletes = 1) 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 0x558c54d2c280 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 kv_queued 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 kv_submitted 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b800 2019-12-07T19:34:04.610+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 finishing 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b800 onodes 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b800 done 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b800 [] 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8875 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8875 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x56091~22ab to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x56091 0x56091~22ab to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 0x558c574a0c80 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x26091 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x91 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [] 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x33c and padding block with 0xcc4 2019-12-07T19:34:04.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.610+0000 7f8f07187700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.610+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558c5513e000 2019-12-07T19:34:04.610+0000 7f8f06986700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.610+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558c57676000 2019-12-07T19:34:04.610+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 crt=23'10 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 remapped m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.610+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558c576b8e00 con 0x558c54c5b400 2019-12-07T19:34:04.610+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 remapped m=4 mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.610+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5833c 2019-12-07T19:34:04.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558c57451500 con 0x558c54c19c00 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000140599s (0.000000074s flush + 0.000140525s kv commit) 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5730b500 seq 2 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a000> 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 kv_submitted 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a000 2019-12-07T19:34:04.611+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore.onode(0x558c574a0c80).flush flush done 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 finishing 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a000 onodes 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 done 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a000 [] 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.611+0000 7f8f07187700 15 osd.7 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.611+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.611+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000045c'.0000000000.00000000000000000000' end: 0x000000000000045c'.4294967295.18446744073709551615' 2019-12-07T19:34:04.611+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.611+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:04.611+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:04.611+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:04.611+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558c576b9500 con 0x558c54c5b400 2019-12-07T19:34:04.611+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000023.00000000000000000010' <- 0000000023.00000000000000000010 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.missing/0000000000000001.23972223.head.smithi13140796-48..' <- missing/0000000000000001.23972223.head.smithi13140796-48.. 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.missing/0000000000000001.28BF48E5.head.smithi13140796-39..' <- missing/0000000000000001.28BF48E5.head.smithi13140796-39.. 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.missing/0000000000000001.28F71588.head.smithi13140796-45..' <- missing/0000000000000001.28F71588.head.smithi13140796-45.. 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.missing/0000000000000001.2916DE48.head.smithi13140796-43..' <- missing/0000000000000001.2916DE48.head.smithi13140796-43.. 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.611+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.611+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b500 cost 10810 (1 ios * 4000 + 6810 bytes) 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b500 onodes shared_blobs 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.611+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 prepare 2019-12-07T19:34:04.611+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b500 2019-12-07T19:34:04.611+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 io_done 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6618 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6641 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6641 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5833c~19f1 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5833c 0x5833c~19f1 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2833c 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x33c 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd2d and padding block with 0x2d3 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x59d2d 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000142433s (0.000000130s flush + 0.000142303s kv commit) 2019-12-07T19:34:04.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b500> 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 kv_submitted 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b500 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 finishing 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b500 onodes 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 done 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b500 [] 2019-12-07T19:34:04.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7f8f07988700 15 osd.7 27 enqueue_peering_evt 1.2s4 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558c5731a000 2019-12-07T19:34:04.611+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.611+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7f8f07988700 10 osd.7 27 send_incremental_map 26 -> 27 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:34:04.611+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(27..27 src has 1..27) v4 -- 0x558c57302d80 con 0x558c54c64400 2019-12-07T19:34:04.611+0000 7f8f07988700 10 osd.7 27 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.611+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.2s0 1.2s0 1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558c57450000 con 0x558c54c64400 2019-12-07T19:34:04.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 50 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c57450000 con 0x558c54c5bc00 2019-12-07T19:34:04.615+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.6s2 epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062329548s ub 44.062329548s int 16s) 2019-12-07T19:34:04.615+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062329548s ub 44.062329548s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062329548s ub 44.062329548s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062329548s ub 44.062329548s int 16s)) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.615+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.6s0( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062329548s ub 44.062329548s int 16s) 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.615+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:04.615+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0300 seq 32 2019-12-07T19:34:04.615+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.615+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.615+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes shared_blobs 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.615+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:34:04.615+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x59d2d~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x59d2d 0x59d2d~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x29d2d 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd2d 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x143 and padding block with 0xebd 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5a143 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000072755s (0.000000098s flush + 0.000072657s kv commit) 2019-12-07T19:34:04.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0300> 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:34:04.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:34:04.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7f8f0998c700 15 osd.7 27 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:04.615+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) queued 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:34:04.615+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.0 ruub 36.052429711s 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.5 ruub 36.052429711s 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.2 ruub 36.052429711s 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.4 ruub 36.052429711s 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until readable_until[_ub] 36.052429711s (sent 44.052553893s) 2019-12-07T19:34:04.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_lease(1.4s1 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) e27/27) v1 -- 0x558c57569800 con 0x558c54c64400 2019-12-07T19:34:04.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.4s2 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) e27/27) v1 -- 0x558c57569a00 con 0x558c54c64000 2019-12-07T19:34:04.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.4s4 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) e27/27) v1 -- 0x558c57569c00 con 0x558c54c19800 2019-12-07T19:34:04.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.4s5 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) e27/27) v1 -- 0x558c57569e00 con 0x558c54c5b400 2019-12-07T19:34:04.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.4s3 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) e27/27) v1 -- 0x558c576b5200 con 0x558c54c19c00 2019-12-07T19:34:04.615+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 46 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 44.052553893s) e27/27) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c54d390e0 con 0x558c54c5b400 2019-12-07T19:34:04.616+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.4s0 epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.4 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.4 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.4 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.4 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) pg 0x558c548ab000 2019-12-07T19:34:04.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.4 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.0 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.5 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.2 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until readable_until[_ub] 36.052429711s (sent 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 17 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 44.052553893s) e27/27) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c54d390e0 con 0x558c54c19c00 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:04.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:04.616+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.4s0 epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.5 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.5 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.5 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.5 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) pg 0x558c548ab000 2019-12-07T19:34:04.616+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.5 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.0 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.5 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.2 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until readable_until[_ub] 36.052429711s (sent 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:04.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 16 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 44.052553893s) e27/27) v1 ==== 42+0+0 (crc 0 0 0) 0x558c575cde00 con 0x558c54c64400 2019-12-07T19:34:04.616+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.4s0 epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.0 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.0 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.616+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 36 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 44.052553893s) e27/27) v1 ==== 42+0+0 (crc 0 0 0) 0x558c576dcd20 con 0x558c54c19800 2019-12-07T19:34:04.616+0000 7f8f24c58700 15 osd.7 27 enqueue_peering_evt 1.4s0 epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.2 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.2 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.0 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.0 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) pg 0x558c548ab000 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.2 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.0 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.5 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.2 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until readable_until[_ub] 36.052429711s (sent 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:04.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:04.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.2 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) pg 0x558c548ab000 2019-12-07T19:34:04.616+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.2 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.1 ruub 36.052429711s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.5 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.2 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until readable_until[_ub] 36.052429711s (sent 44.052553893s) 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:04.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.617+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 37 ==== pg_lease(1.ds2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s) e27/27) v1 ==== 58+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c19800 2019-12-07T19:34:04.617+0000 7f8f24c58700 15 osd.7 27 enqueue_peering_evt 1.ds2 epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s) 2019-12-07T19:34:04.617+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s)) prio 255 cost 10 e27) pg 0x558c55141000 2019-12-07T19:34:04.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s) 2019-12-07T19:34:04.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_lease pg_lease(ru 36.088857038s ub 44.090670001s int 16s) 2019-12-07T19:34:04.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] peer_clock_delta_ub 0.036744339s -> ru 36.052112699s 2019-12-07T19:34:04.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_lease readable_until now 36.052112699s 2019-12-07T19:34:04.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] peer_clock_delta_lb 0.036085548s -> ruub 44.054584453s 2019-12-07T19:34:04.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.054584453s 2019-12-07T19:34:04.618+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.ds0 pg_lease_ack(ruub 44.090670001s) e27/27) v1 -- 0x558c54d9eb40 con 0x558c54c19800 2019-12-07T19:34:04.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:04.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 80 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c57452300 con 0x558c54c64000 2019-12-07T19:34:04.618+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s) 2019-12-07T19:34:04.618+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s)) prio 255 cost 10 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 81 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c57452300 con 0x558c54c64000 2019-12-07T19:34:04.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s) 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.618+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.9s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s) 2019-12-07T19:34:04.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:04.618+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740a600 seq 27 2019-12-07T19:34:04.618+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:04.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:04.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes shared_blobs 2019-12-07T19:34:04.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 82 ==== pg_lease(1.0s3 pg_lease(ru 36.074625527s ub 44.075403950s int 16s) e27/27) v1 ==== 58+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c54c64000 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.618+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.0s3 epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.1 pg_lease(ru 36.074625527s ub 44.075403950s int 16s) 2019-12-07T19:34:04.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:04.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:04.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s)) prio 255 cost 10 e27) pg 0x558c5500a000 2019-12-07T19:34:04.618+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.1 pg_lease(ru 36.074625527s ub 44.075403950s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.618+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s) 2019-12-07T19:34:04.618+0000 7f8f08189700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:04.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c548f7b00 seq 18 2019-12-07T19:34:04.619+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:04.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:04.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:04.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes shared_blobs 2019-12-07T19:34:04.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:04.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:04.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:04.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2046 < min_flush_size 524288 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5a143~815 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5a143 0x5a143~815 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2a143 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x143 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x958 and padding block with 0x6a8 2019-12-07T19:34:04.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 83 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 44.052553893s) e27/27) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54d390e0 con 0x558c54c64000 2019-12-07T19:34:04.619+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.4s0 epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.1 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.1 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5a958 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000074330s (0.000000072s flush + 0.000074258s kv commit) 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a600,0x558c548f7b00> 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.1 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.1 pg_lease_ack(ruub 44.052553893s)) prio 255 cost 10 e27) pg 0x558c548ab000 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:04.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLeaseAck epoch 27 from osd.1 pg_lease_ack(ruub 44.052553893s) 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:04.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.1 ruub 44.052553893s 2019-12-07T19:34:04.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.5 ruub 44.052553893s 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.2 ruub 44.052553893s 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recalc_readable_until readable_until[_ub] 44.052553893s (sent 44.052553893s) 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.1 pg_lease(ru 36.074625527s ub 44.075403950s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.1 pg_lease(ru 36.074625527s ub 44.075403950s int 16s)) prio 255 cost 10 e27) pg 0x558c5500d000 2019-12-07T19:34:04.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.1 pg_lease(ru 36.074625527s ub 44.075403950s int 16s) 2019-12-07T19:34:04.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] proc_lease pg_lease(ru 36.074625527s ub 44.075403950s int 16s) 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] peer_clock_delta_ub 0.021951027s -> ru 36.052674500s 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] proc_lease readable_until now 36.052674500s 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] peer_clock_delta_lb 0.021357758s -> ruub 44.054046192s 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.054046192s 2019-12-07T19:34:04.619+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.0s0 pg_lease_ack(ruub 44.075403950s) e27/27) v1 -- 0x558c576dcf00 con 0x558c54c64000 2019-12-07T19:34:04.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 47 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c57452300 con 0x558c54c5b400 2019-12-07T19:34:04.619+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s) 2019-12-07T19:34:04.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s)) prio 255 cost 10 e27) pg 0x558c5513e000 2019-12-07T19:34:04.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s) 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5730a300 seq 25 2019-12-07T19:34:04.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:04.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:04.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a300 onodes shared_blobs 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 prepare 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a300 2019-12-07T19:34:04.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 io_done 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5a958~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5a958 0x5a958~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2a958 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x958 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd6e and padding block with 0x292 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5ad6e 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000073813s (0.000000086s flush + 0.000073727s kv commit) 2019-12-07T19:34:04.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a300> 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 kv_submitted 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a300 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 finishing 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a300 onodes 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a300 done 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a300 [] 2019-12-07T19:34:04.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 48 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c57452300 con 0x558c54c5b400 2019-12-07T19:34:04.620+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s) 2019-12-07T19:34:04.620+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s)) prio 255 cost 10 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.620+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s) 2019-12-07T19:34:04.620+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c574ff500 seq 3 2019-12-07T19:34:04.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:04.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff500 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff500 onodes shared_blobs 2019-12-07T19:34:04.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 prepare 2019-12-07T19:34:04.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff500 2019-12-07T19:34:04.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 io_done 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5ad6e~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5ad6e 0x5ad6e~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2ad6e 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd6e 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x184 and padding block with 0xe7c 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5b184 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000076835s (0.000000110s flush + 0.000076725s kv commit) 2019-12-07T19:34:04.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff500> 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 kv_submitted 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff500 2019-12-07T19:34:04.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 finishing 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff500 onodes 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff500 done 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:04.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff500 [] 2019-12-07T19:34:04.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 51 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=51, to_read={1:6336c6f2:::smithi13140796-16:head=0,65536,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c574a0a00 con 0x558c54c5bc00 2019-12-07T19:34:04.620+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000022 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=51, to_read={1:6336c6f2:::smithi13140796-16:head=0,65536,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.620+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=51, to_read={1:6336c6f2:::smithi13140796-16:head=0,65536,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=51, to_read={1:6336c6f2:::smithi13140796-16:head=0,65536,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=51, to_read={1:6336c6f2:::smithi13140796-16:head=0,65536,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.620+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000071 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=51, to_read={1:6336c6f2:::smithi13140796-16:head=0,65536,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.620+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=51, to_read={1:6336c6f2:::smithi13140796-16:head=0,65536,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.620+0000 7f8f05984700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.620+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~10000 size 0x10000 (65536) 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8b08 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c53a69b00 space 0x558c54cf8b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8ef8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a93b0 space 0x558c54cf8ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8fd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a9170 space 0x558c54cf8fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf90b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a9320 space 0x558c54cf90b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf9198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a90e0 space 0x558c54cf9198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d75eb8 in 0x558c53a58fc0) _discard 0x1000~1000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c573d87e0 space 0x558c54d75eb8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c573d8630 space 0x558c54cf8da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d3cc58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.621+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54c9ab40 space 0x558c54d3cc58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 = 65536 2019-12-07T19:34:04.621+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=65536 r=65536 len=65536 2019-12-07T19:34:04.621+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:04.621+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:04.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:04.621+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 65536 2019-12-07T19:34:04.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key 2019-12-07T19:34:04.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key = 0 2019-12-07T19:34:04.621+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:04.621+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=51, attrs_read=0)) v2 -- 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:04.621+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:34:04.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 84 ==== MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c5756f180 con 0x558c54c64000 2019-12-07T19:34:04.622+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334160 prio 127 cost 0 latency 0.000022 epoch 27 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.622+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 85 ==== MOSDECSubOpRead(1.0s3 27/18 ECSubRead(tid=100, to_read={1:0e6be130:::smithi13140796-36:head=0,147456,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c54e45900 con 0x558c54c64000 2019-12-07T19:34:04.622+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334420 prio 127 cost 0 latency 0.000014 epoch 27 MOSDECSubOpRead(1.0s3 27/18 ECSubRead(tid=100, to_read={1:0e6be130:::smithi13140796-36:head=0,147456,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.622+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 27/18 ECSubRead(tid=100, to_read={1:0e6be130:::smithi13140796-36:head=0,147456,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.622+0000 7f8f05183700 10 osd.7 27 dequeue_op 0x558c57334160 prio 127 cost 0 latency 0.000268 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.622+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.622+0000 7f8f05183700 25 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 2019-12-07T19:34:04.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~13000 size 0x13000 (77824) 2019-12-07T19:34:04.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 sbid 0x0)) need {<0x0, 0x1000> : [0x12000:0~1000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.623+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 27/18 ECSubRead(tid=100, to_read={1:0e6be130:::smithi13140796-36:head=0,147456,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.623+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 27/18 ECSubRead(tid=100, to_read={1:0e6be130:::smithi13140796-36:head=0,147456,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558c5500d000 2019-12-07T19:34:04.623+0000 7f8f04982700 10 osd.7 27 dequeue_op 0x558c57334420 prio 127 cost 0 latency 0.000524 MOSDECSubOpRead(1.0s3 27/18 ECSubRead(tid=100, to_read={1:0e6be130:::smithi13140796-36:head=0,147456,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:04.623+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.0s3 27/18 ECSubRead(tid=100, to_read={1:0e6be130:::smithi13140796-36:head=0,147456,0}, subchunks={1:0e6be130:::smithi13140796-36:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.623+0000 7f8f04982700 25 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 0x0~24000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24000 size 0x24000 (147456) 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c5b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c700 blob([0x15f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265d50 blob([0x161000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265c70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e567e0 blob([0x163000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54677d50 blob([0x165000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c547400e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278e70 blob([0x167000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552793b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43f10 blob([0x169000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582070 blob([0x16b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575820e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582150 blob([0x16d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575821c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582230 blob([0x16f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575822a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582310 blob([0x171000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575823f0 blob([0x173000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575824d0 blob([0x175000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582540 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575825b0 blob([0x177000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582690 blob([0x179000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582770 blob([0x17b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575827e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582850 blob([0x17d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575828c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57582930 blob([0x17f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575829a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54677d50 blob([0x165000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c547400e0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c700 blob([0x15f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c690 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c5b0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43f10 blob([0x169000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582000 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e567e0 blob([0x163000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56380 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55265d50 blob([0x161000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265c70 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55278e70 blob([0x167000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552793b0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582070 blob([0x16b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575820e0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582150 blob([0x16d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575821c0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582230 blob([0x16f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575822a0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582310 blob([0x171000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582380 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575823f0 blob([0x173000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582460 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575824d0 blob([0x175000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582540 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575825b0 blob([0x177000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582620 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582690 blob([0x179000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582700 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582770 blob([0x17b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575827e0 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582850 blob([0x17d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575828c0 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57582930 blob([0x17f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575829a0 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c55265978 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9bdd0 space 0x558c55265978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54d3d128 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9be60 space 0x558c54d3d128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54cf9668 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9bef0 space 0x558c54cf9668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e430b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9bd40 space 0x558c54e430b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e43198 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9a6c0 space 0x558c54e43198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e43278 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9a630 space 0x558c54e43278 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e43358 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9a510 space 0x558c54e43358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e43438 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9a5a0 space 0x558c54e43438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12000:0~1000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e43518 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9abd0 space 0x558c54e43518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.623+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54e42fd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.623+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9bb00 space 0x558c54e42fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 = 77824 2019-12-07T19:34:04.623+0000 7f8f05183700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=77824 r=77824 len=77824 2019-12-07T19:34:04.623+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.623+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:04.623+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:04.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:34:04.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:34:04.623+0000 7f8f05183700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.623+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=99, attrs_read=0)) v2 -- 0x558c54d97400 con 0x558c54c64000 2019-12-07T19:34:04.623+0000 7f8f05183700 10 osd.7 27 dequeue_op 0x558c57334160 finish 2019-12-07T19:34:04.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54677d50 blob([0x165000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c547400e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c547400f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c551a8090 space 0x558c547400f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c700 blob([0x15f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54d3c6a8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c551a9950 space 0x558c54d3c6a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c770 blob([0x15d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c5b0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54d3c5c8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c548de120 space 0x558c54d3c5c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43f10 blob([0x169000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57582018 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c548de090 space 0x558c57582018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e567e0 blob([0x163000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54e56398 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c54c9ba70 space 0x558c54e56398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55265d50 blob([0x161000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265c70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c55265c88 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c54c9b8c0 space 0x558c55265c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55278e70 blob([0x167000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552793b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c552793c8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c54c9b560 space 0x558c552793c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582070 blob([0x16b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575820e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c575820f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c54c9b9e0 space 0x558c575820f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582150 blob([0x16d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575821c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c575821d8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c573d9320 space 0x558c575821d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582230 blob([0x16f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575822a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c575822b8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c573d93b0 space 0x558c575822b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582310 blob([0x171000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57582398 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c573d9290 space 0x558c57582398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575823f0 blob([0x173000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57582478 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c573d90e0 space 0x558c57582478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575824d0 blob([0x175000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582540 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57582558 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c573d8cf0 space 0x558c57582558 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575825b0 blob([0x177000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57582638 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c548de000 space 0x558c57582638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582690 blob([0x179000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57582700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57582718 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c548df560 space 0x558c57582718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582770 blob([0x17b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575827e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c575827f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c548deab0 space 0x558c575827f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582850 blob([0x17d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575828c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c575828d8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c548de6c0 space 0x558c575828d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57582930 blob([0x17f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575829a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.624+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c575829b8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7f8f04982700 20 _add level 1 near 0 on buffer(0x558c548de630 space 0x558c575829b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 0x0~24000 = 147456 2019-12-07T19:34:04.624+0000 7f8f04982700 20 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_sub_read read request=147456 r=147456 len=147456 2019-12-07T19:34:04.624+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:0e6be130:::smithi13140796-36:head 2019-12-07T19:34:04.624+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] get_hash_info: not in cache 1:0e6be130:::smithi13140796-36:head 2019-12-07T19:34:04.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# 2019-12-07T19:34:04.624+0000 7f8f04982700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] get_hash_info: found on disk, size 147456 2019-12-07T19:34:04.624+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key 2019-12-07T19:34:04.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0e6be130:::smithi13140796-36:head# hinfo_key = 0 2019-12-07T19:34:04.624+0000 7f8f04982700 20 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:0e6be130:::smithi13140796-36:head 2019-12-07T19:34:04.624+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.0s0 27/18 ECSubReadReply(tid=100, attrs_read=0)) v2 -- 0x558c575bb680 con 0x558c54c64000 2019-12-07T19:34:04.624+0000 7f8f04982700 10 osd.7 27 dequeue_op 0x558c57334420 finish 2019-12-07T19:34:04.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.626+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 241973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.629+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 17 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c57450000 con 0x558c54c64400 2019-12-07T19:34:04.629+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.2s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.065221680s ub 44.065221680s int 16s) 2019-12-07T19:34:04.629+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.065221680s ub 44.065221680s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.629+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.629+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.065221680s ub 44.065221680s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.629+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.629+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.065221680s ub 44.065221680s int 16s)) prio 255 cost 10 e27) pg 0x558c5731a000 2019-12-07T19:34:04.629+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.065221680s ub 44.065221680s int 16s) 2019-12-07T19:34:04.629+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.629+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:04.629+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.629+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54ea1200 seq 3 2019-12-07T19:34:04.629+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.629+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.629+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.629+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.629+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:34:04.629+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:04.629+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:04.629+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:04.629+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.629+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5b184~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5b184 0x5b184~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2b184 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x184 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x59a and padding block with 0xa66 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5b59a 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000103013s (0.000000198s flush + 0.000102815s kv commit) 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200> 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:04.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:04.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.629+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 18 ==== pg_info2(1.cs0 1.cs0 1.cs1( v 21'12 (0'0,21'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c54e99c00 con 0x558c54c19c00 2019-12-07T19:34:04.634+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(1) info: 1.cs1( v 21'12 (0'0,21'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.076045274s ub 44.076045274s int 16s) 2019-12-07T19:34:04.634+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(1) info: 1.cs1( v 21'12 (0'0,21'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.076045274s ub 44.076045274s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(1) info: 1.cs1( v 21'12 (0'0,21'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.076045274s ub 44.076045274s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(1) info: 1.cs1( v 21'12 (0'0,21'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.076045274s ub 44.076045274s int 16s)) prio 255 cost 10 e27) pg 0x558c57676000 2019-12-07T19:34:04.634+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(1) info: 1.cs1( v 21'12 (0'0,21'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.076045274s ub 44.076045274s int 16s) 2019-12-07T19:34:04.634+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.634+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d58300 seq 4 2019-12-07T19:34:04.634+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes shared_blobs 2019-12-07T19:34:04.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:34:04.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:34:04.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:34:04.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5b59a~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5b59a 0x5b59a~416 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2b59a 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x59a 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9b0 and padding block with 0x650 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5b9b0 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000074876s (0.000000098s flush + 0.000074778s kv commit) 2019-12-07T19:34:04.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300> 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:34:04.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:34:04.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.637+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 52 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=52, reqid=client.4540.0:187, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 81156+0+0 (unknown 0 0 0) 0x558c575c0000 con 0x558c54c5bc00 2019-12-07T19:34:04.637+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000040 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=52, reqid=client.4540.0:187, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.637+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=52, reqid=client.4540.0:187, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=52, reqid=client.4540.0:187, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=52, reqid=client.4540.0:187, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c54e78c60 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=52, reqid=client.4540.0:187, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=52, reqid=client.4540.0:187, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 snap_mapper.add_oid 1:6c205177:::smithi13140796-8:4 4 2019-12-07T19:34:04.637+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:4 got.empty() 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6c205177:::smithi13140796-8:4 4 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:04.637+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 23'13 (0'0,23'13] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] append_log log((0'0,23'13], crt=23'13) [27'14 (21'5) clone 1:6c205177:::smithi13140796-8:4 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'15 (21'5) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:187 0.000000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'14 (0'0,27'14] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'13 crt=23'13 lcod 23'13 active+remapped mbc={} ps=[2~1]] add_log_entry 27'14 (21'5) clone 1:6c205177:::smithi13140796-8:4 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'13 crt=23'13 lcod 23'13 active+remapped mbc={} ps=[2~1]] add_log_entry 27'15 (21'5) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:187 0.000000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'13 crt=23'13 lcod 23'13 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 15 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'13 crt=23'13 lcod 23'13 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.637+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.638+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740bb00 seq 33 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:4# key 0x8280000000000000016c2051'w!smithi13140796-8!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:4# 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1117 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.onode(0x558c54d96500).flush flush done 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# 0x0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:4# 0x0~8000 -> 0x0~8000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10264 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d75340 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d75340 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup new Blob(0x558c54d75260 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75260) get_ref 0x0~2000 Blob(0x558c54d75260 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup dst 0x0~2000: 0x0~2000 Blob(0x558c54d75260 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d3df80 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10265 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3df80 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3df80 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup new Blob(0x558c552656c0 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c552656c0) get_ref 0x0~2000 Blob(0x558c552656c0 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c552656c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c552656c0 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d75420 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10266 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d75420 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d75420 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup new Blob(0x558c57314620 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314620) get_ref 0x0~2000 Blob(0x558c57314620 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57314620 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54cf8070 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10267 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54cf8070 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54cf8070 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup new Blob(0x558c575bd6c0 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) get_ref 0x0~2000 Blob(0x558c575bd6c0 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=2)))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dirty_range mark inline shard dirty 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:4# = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# snapset 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# snapset = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# 1 keys 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# 1 keys = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:head#f key 0x8280000000000000016c2051'w!smithi13140796-8!='0xfffffffffffffffe000000000000000f'o' 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:head#f 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:head#f = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1118 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 3 keys 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 3 keys = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~13000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6c205177:::smithi13140796-8:head# 0x0~13000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~13000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740bb00 10 blobs 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x13000/1000,13000,0 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x26f000~13000/1000,13000,0 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x26f000~13000] 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd260) get_ref 0x0~2000 Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bd3c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd420) get_ref 0x0~2000 Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573142b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57314230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314230) get_ref 0x0~2000 Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573141d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57314150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314150) get_ref 0x0~2000 Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573140f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57314070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314070) get_ref 0x0~2000 Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314558 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573145b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573145b0) get_ref 0x0~2000 Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573145b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573146a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57314700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314700) get_ref 0x0~2000 Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57314700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314788 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573147e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573147e0) get_ref 0x0~2000 Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573147e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314868 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573148c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573148c0) get_ref 0x0~2000 Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573148c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314948 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573149a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573149a0) get_ref 0x0~1000 Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.blob(0x558c573149a0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~1000: 0x0~1000 Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 sbid 0x0)) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314a28 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x13000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark inline shard dirty 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~13000 = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.4.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.2_0000000000000001.6340A8EE.4.smithi13140796-8..' <- SNA_1_0000000000000004_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000027.00000000000000000014' <- 0000000027.00000000000000000014 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000027.00000000000000000015' <- 0000000027.00000000000000000015 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 126180 (11 ios * 4000 + 82180 bytes) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes 0x558c54ce3680,0x558c54d96500,0x558c575ba280 shared_blobs 0x558c54cf8000,0x558c54cf80e0,0x558c54d753b0,0x558c54d75ab0 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3788) update 2#1:6c205177:::smithi13140796-8:4# 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3788) update inline shard 111 bytes from 4 extents 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:4# is 527 (410 bytes onode + 2 bytes spanning blobs + 115 bytes inline extents) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) update 2#1:6c205177:::smithi13140796-8:head#f 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) update inline shard 111 bytes from 4 extents 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head#f is 600 (483 bytes onode + 2 bytes spanning blobs + 115 bytes inline extents) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update inline shard 190 bytes from 10 extents 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 736 (540 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281a is 11 SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=2))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281b is 11 SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=2))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2819 is 11 SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=2))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2818 is 11 SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=2))) 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[26f000~13000] released 0x[] 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 freelist allocate 0x26f000~13000 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 freelist _xor first_key 0x200000 last_key 0x280000 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740bb00 2019-12-07T19:34:04.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'13 crt=23'13 lcod 23'13 active+remapped mbc={} ps=[2~1]] op_applied version 27'15 2019-12-07T19:34:04.638+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:34:04.638+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.638+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 aio_wait 2019-12-07T19:34:04.638+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:04.638+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4423 < min_flush_size 524288 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3680 had 1 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96500 had 1 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4446 < min_flush_size 524288 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4446 < min_flush_size 524288 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5b9b0~115e to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5b9b0 0x5b9b0~115e to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2b9b0 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9b0 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb0e and padding block with 0x4f2 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5cb0e 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132781s (0.000039633s flush + 0.000093148s kv commit) 2019-12-07T19:34:04.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740bb00> 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:04.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 0x558c54ce3680,0x558c54d96500,0x558c575ba280 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573140f8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a7e0 space 0x558c573140f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573141d8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a9cb0 space 0x558c573141d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573142b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b200 space 0x558c573142b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314558 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b3b0 space 0x558c57314558 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573146a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9afc0 space 0x558c573146a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314788 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b050 space 0x558c57314788 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314868 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b7a0 space 0x558c57314868 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314948 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548df4d0 space 0x558c57314948 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314a28 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548df8c0 space 0x558c57314a28 0x0~1000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bd3c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a9a70 space 0x558c575bd3c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:34:04.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.639+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=52, last_complete=27'15, committed=1, applied=1)) v2 -- 0x558c575b9800 con 0x558c54c5bc00 2019-12-07T19:34:04.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.639+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 86 ==== MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 150788+0+0 (crc 0 0 0) 0x558c54e3e800 con 0x558c54c64000 2019-12-07T19:34:04.640+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334c60 prio 127 cost 0 latency 0.000027 epoch 27 MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.640+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558c5500a000 2019-12-07T19:34:04.640+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c57334c60 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.640+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.640+0000 7f8f04181700 20 snap_mapper.add_oid 1:9694a0e5:::smithi13140796-3:4 4 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:4 got.empty() 2019-12-07T19:34:04.640+0000 7f8f04181700 20 snap_mapper.set_snaps 1:9694a0e5:::smithi13140796-3:4 4 2019-12-07T19:34:04.640+0000 7f8f04181700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.640+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.640+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 21'6 (0'0,21'6] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,21'6], crt=21'6) [27'7 (21'3) clone 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'8 (21'3) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:209 0.000000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.640+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'7 (0'0,27'7] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] add_log_entry 27'7 (21'3) clone 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.640+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] add_log_entry 27'8 (21'3) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:209 0.000000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.640+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:04.640+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.640+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.640+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d21b00 seq 19 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9694a0e5:::smithi13140796-3:4# key 0x8480000000000000019694a0e521736d'ithi13140796-3!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:4# 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1119 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.onode(0x558c54d96a00).flush flush done 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:4# 0x0~3000 -> 0x0~3000 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d3c7e0 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 sbid 0x0)) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10268 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54d3c7e0 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 sbid 0x0)) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54d3c7e0 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dup new Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=2)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8770) get_ref 0x0~2000 Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=2)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=2)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dup src 0x2000~1000: 0x0~1000 Blob(0x558c54d74fc0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 sbid 0x0)) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10269 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54d74fc0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 sbid 0x0)) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54d74fc0 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dup new Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=2)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75d50) get_ref 0x0~1000 Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=2)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.blob(0x558c54d75d50) get_ref init 0x1000, 1000 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dup dst 0x2000~1000: 0x0~1000 Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=2)))) 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:4# = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# snapset 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# snapset = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 1 keys 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 1 keys = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9694a0e5:::smithi13140796-3:head#8 key 0x8480000000000000019694a0e521736d'ithi13140796-3!='0xfffffffffffffffe0000000000000008'o' 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:head#8 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:head#8 = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1120 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 3 keys 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 3 keys = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x0~24000 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9694a0e5:::smithi13140796-3:head# 0x0~24000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~24000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d21b00 18 blobs 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0x24000/1000,24000,0 2019-12-07T19:34:04.641+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x282000~24000/1000,24000,0 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x282000~24000] 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf88c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf88c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8850) get_ref 0x0~2000 Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf88c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54cf88d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf89a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a10 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c54cf89a0) get_ref 0x0~2000 Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a10 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c54cf89a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54cf8a28 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573158f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57315880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315880 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573158f0) get_ref 0x0~2000 Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315880 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573158f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57315898 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573157a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573157a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315810) get_ref 0x0~2000 Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573157a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315810) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c573157b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573156c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573156c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315730) get_ref 0x0~2000 Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573156c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c573156d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573155e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573155e0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315650) get_ref 0x0~2000 Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573155e0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c573155f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552795e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567570 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c552795e0) get_ref 0x0~2000 Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567570 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c552795e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57567588 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa230 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c575bd960) get_ref 0x0~2000 Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa230 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c575bd960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c574fa248 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57315500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315500 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315570) get_ref 0x0~2000 Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315500 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57315518 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57315420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315420 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315490) get_ref 0x0~2000 Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315420 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57315438 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573153b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57315340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315340 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573153b0) get_ref 0x0~2000 Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315340 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573153b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57315358 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573152d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57315260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315260 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573152d0) get_ref 0x0~2000 Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315260 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573152d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57315278 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573151f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57315180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315180 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573151f0) get_ref 0x0~2000 Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315180 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c573151f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57315198 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573150a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573150a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315110) get_ref 0x0~2000 Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573150a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c573150b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314fc0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315030) get_ref 0x0~2000 Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314fc0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57314fd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57314f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314ee0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57314f50) get_ref 0x0~2000 Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314ee0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57314f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57314ef8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573159d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573159d0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315960) get_ref 0x0~2000 Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573159d0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c573159e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57315a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57315ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315ab0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315a40) get_ref 0x0~2000 Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315ab0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.blob(0x558c57315a40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57315ac8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x0~24000 = 0 2019-12-07T19:34:04.641+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.641+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9692507A.4.smithi13140796-3..' <- OBJ_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.641+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3..' <- SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.641+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000027.00000000000000000007' <- 0000000027.00000000000000000007 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000027.00000000000000000008' <- 0000000027.00000000000000000008 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 227812 (19 ios * 4000 + 151812 bytes) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes 0x558c54ce2a00,0x558c54d96a00,0x558c54d97680 shared_blobs 0x558c54d3c620,0x558c54d3de30 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update inline shard 345 bytes from 18 extents 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 891 (540 bytes onode + 2 bytes spanning blobs + 349 bytes inline extents) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) update 4#1:9694a0e5:::smithi13140796-3:head#8 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) update inline shard 54 bytes from 2 extents 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head#8 is 542 (482 bytes onode + 2 bytes spanning blobs + 58 bytes inline extents) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97788) update 4#1:9694a0e5:::smithi13140796-3:4# 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97788) update inline shard 54 bytes from 2 extents 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:4# is 469 (409 bytes onode + 2 bytes spanning blobs + 58 bytes inline extents) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281d is 11 SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=2))) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281c is 11 SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=2))) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[282000~24000] released 0x[] 2019-12-07T19:34:04.641+0000 7f8f04181700 10 freelist allocate 0x282000~24000 2019-12-07T19:34:04.641+0000 7f8f04181700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:04.641+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d21b00 2019-12-07T19:34:04.641+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] op_applied version 27'8 2019-12-07T19:34:04.641+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c57334c60 finish 2019-12-07T19:34:04.641+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 aio_wait 2019-12-07T19:34:04.641+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:04.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.641+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4385 < min_flush_size 524288 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96a00 had 1 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97680 had 1 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4408 < min_flush_size 524288 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4408 < min_flush_size 524288 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5cb0e~1138 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5cb0e 0x5cb0e~1138 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2cb0e 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb0e 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc46 and padding block with 0x3ba 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5dc46 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000202948s (0.000056317s flush + 0.000146631s kv commit) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:04.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 0x558c54ce2a00,0x558c54d96a00,0x558c54d97680 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf88d8 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a9560 space 0x558c54cf88d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8a28 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde000 space 0x558c54cf8a28 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314ef8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde7e0 space 0x558c57314ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314fd8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde750 space 0x558c57314fd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573150b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde6c0 space 0x558c573150b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57315198 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde630 space 0x558c57315198 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57315278 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde5a0 space 0x558c57315278 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57315358 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde510 space 0x558c57315358 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57315438 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde480 space 0x558c57315438 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57315518 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde3f0 space 0x558c57315518 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573155f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde240 space 0x558c573155f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573156d8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde1b0 space 0x558c573156d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573157b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde120 space 0x558c573157b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57315898 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde090 space 0x558c57315898 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573159e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde870 space 0x558c573159e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57315ac8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde900 space 0x558c57315ac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa248 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde360 space 0x558c574fa248 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567588 in 0x558c53a59a40) _finish_write discard buffer(0x558c54cde2d0 space 0x558c57567588 0x0~2000 writing nocache) 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:04.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:04.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.641+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 27/26 ECSubWriteReply(tid=104, last_complete=27'8, committed=1, applied=1)) v2 -- 0x558c5768a000 con 0x558c54c64000 2019-12-07T19:34:04.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 10 osd.7 27 tick_without_osd_lock 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fd5a000/0x40000000/0x1680000000, data 0x2af94f/0x2a4000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 25 osd.7 27 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 25 osd.7 27 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:03.373892+0000 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 25 osd.7 27 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 25 osd.7 27 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373810+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 25 osd.7 27 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 25 osd.7 27 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 20 osd.7 27 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 10 osd.7 27 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 20 osd.7 27 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:04.642+0000 7f8f1fbbe700 10 osd.7 27 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:04.651+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:04.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 87 ==== MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 ==== 1027+0+0 (crc 0 0 0) 0x558c54897000 con 0x558c54c64000 2019-12-07T19:34:04.651+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000023 epoch 27 MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 2019-12-07T19:34:04.651+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) pg 0x558c5500a000 2019-12-07T19:34:04.651+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 pg pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.651+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s4 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 2019-12-07T19:34:04.651+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'8], crt=21'6) [] 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'7 (21'3) clone 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'8 (21'3) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:209 0.000000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.651+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:04.651+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.651+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.651+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c574ff800 seq 20 2019-12-07T19:34:04.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head#8 onode 0x558c54d96a00 txc 0x558c574ff800 2019-12-07T19:34:04.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head#8 0x0 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore.blob(0x558c54d3c7e0) put_ref 0x0~2000 Blob(0x558c54d3c7e0 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=2)))) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore.blob(0x558c54d74fc0) put_ref 0x0~1000 Blob(0x558c54d74fc0 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=2)))) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d96b08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d3c7e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=2)))) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b000~2000] 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1))) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~1000: 0x0~1000 Blob(0x558c54d74fc0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=2)))) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3d000~1000] 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1))) 2019-12-07T19:34:04.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3c620,0x558c54d3de30 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c54ce2a00 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head#8 = 0 2019-12-07T19:34:04.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:04.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff800 cost 5562 (1 ios * 4000 + 1562 bytes) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff800 onodes shared_blobs 0x558c54d3c620,0x558c54d3de30 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281d is 11 SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1))) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281c is 11 SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1))) 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff800 allocated 0x[] released 0x[] 2019-12-07T19:34:04.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 prepare 2019-12-07T19:34:04.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff800 2019-12-07T19:34:04.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 io_done 2019-12-07T19:34:04.651+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c57334dc0 finish 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1386 < min_flush_size 524288 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96a00 had 1 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1409 < min_flush_size 524288 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1409 < min_flush_size 524288 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5dc46~581 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5dc46 0x5dc46~581 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2dc46 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc46 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1c7 and padding block with 0xe39 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5e1c7 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.651+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000094497s (0.000000113s flush + 0.000094384s kv commit) 2019-12-07T19:34:04.651+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff800> 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.651+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff800 2019-12-07T19:34:04.651+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 finishing 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff800 onodes 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 done 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:04.651+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff800 [] 2019-12-07T19:34:04.651+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.651+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 27/26 ECSubWriteReply(tid=105, last_complete=27'8, committed=1, applied=1)) v2 -- 0x558c5768a400 con 0x558c54c64000 2019-12-07T19:34:04.651+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.654+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 53 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=57, reqid=client.4540.0:187, at_version=0'0, trim_to=0'0, roll_forward_to=27'15)) v2 ==== 1079+0+0 (unknown 0 0 0) 0x558c575c0800 con 0x558c54c5bc00 2019-12-07T19:34:04.654+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57334b00 prio 127 cost 0 latency 0.000028 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=57, reqid=client.4540.0:187, at_version=0'0, trim_to=0'0, roll_forward_to=27'15)) v2 2019-12-07T19:34:04.654+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=57, reqid=client.4540.0:187, at_version=0'0, trim_to=0'0, roll_forward_to=27'15)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=57, reqid=client.4540.0:187, at_version=0'0, trim_to=0'0, roll_forward_to=27'15)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=57, reqid=client.4540.0:187, at_version=0'0, trim_to=0'0, roll_forward_to=27'15)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334b00 prio 127 cost 0 latency 0.000078 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=57, reqid=client.4540.0:187, at_version=0'0, trim_to=0'0, roll_forward_to=27'15)) v2 pg pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=57, reqid=client.4540.0:187, at_version=0'0, trim_to=0'0, roll_forward_to=27'15)) v2 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'13 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'15], crt=23'13) [] 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'14 (21'5) clone 1:6c205177:::smithi13140796-8:4 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'15 (21'5) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:187 0.000000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 15 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.654+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740ac00 seq 34 2019-12-07T19:34:04.654+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#f onode 0x558c54d96500 txc 0x558c5740ac00 2019-12-07T19:34:04.654+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#f 0x0 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75340) put_ref 0x0~2000 Blob(0x558c54d75340 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df80) put_ref 0x0~2000 Blob(0x558c54d3df80 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75420) put_ref 0x0~2000 Blob(0x558c54d75420 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf8070) put_ref 0x0~2000 Blob(0x558c54cf8070 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54d96608) dirty_range mark inline shard dirty 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d75340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4e000~2000] 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d3df80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x50000~2000] 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54d75420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x52000~2000] 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c54cf8070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=2)))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x54000~2000] 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf8000,0x558c54cf80e0,0x558c54d753b0,0x558c54d75ab0 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c575ba280 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#f = 0 2019-12-07T19:34:04.654+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740ac00 cost 5614 (1 ios * 4000 + 1614 bytes) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740ac00 onodes shared_blobs 0x558c54cf8000,0x558c54cf80e0,0x558c54d753b0,0x558c54d75ab0 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281a is 11 SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281b is 11 SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2819 is 11 SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2818 is 11 SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1))) 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 prepare 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740ac00 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 io_done 2019-12-07T19:34:04.654+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334b00 finish 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1486 < min_flush_size 524288 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96500 had 1 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1509 < min_flush_size 524288 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1509 < min_flush_size 524288 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5e1c7~5e5 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5e1c7 0x5e1c7~5e5 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2e1c7 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1c7 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7ac and padding block with 0x854 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5e7ac 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000097575s (0.000000121s flush + 0.000097454s kv commit) 2019-12-07T19:34:04.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740ac00> 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 kv_submitted 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740ac00 2019-12-07T19:34:04.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 finishing 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740ac00 onodes 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740ac00 done 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740ac00 [] 2019-12-07T19:34:04.654+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.654+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.654+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=57, last_complete=27'15, committed=1, applied=1)) v2 -- 0x558c54d77000 con 0x558c54c5bc00 2019-12-07T19:34:04.654+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.658+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 54 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=60, reqid=client.4540.0:193, at_version=27'17, trim_to=0'0, roll_forward_to=27'15)) v2 ==== 68928+0+0 (unknown 0 0 0) 0x558c54d87800 con 0x558c54c5bc00 2019-12-07T19:34:04.658+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c573349a0 prio 127 cost 0 latency 0.000033 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=60, reqid=client.4540.0:193, at_version=27'17, trim_to=0'0, roll_forward_to=27'15)) v2 2019-12-07T19:34:04.658+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=60, reqid=client.4540.0:193, at_version=27'17, trim_to=0'0, roll_forward_to=27'15)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=60, reqid=client.4540.0:193, at_version=27'17, trim_to=0'0, roll_forward_to=27'15)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=60, reqid=client.4540.0:193, at_version=27'17, trim_to=0'0, roll_forward_to=27'15)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c573349a0 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=60, reqid=client.4540.0:193, at_version=27'17, trim_to=0'0, roll_forward_to=27'15)) v2 pg pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=6 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=60, reqid=client.4540.0:193, at_version=27'17, trim_to=0'0, roll_forward_to=27'15)) v2 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 snap_mapper.add_oid 1:6ee4dfe7:::smithi13140796-47:4 4 2019-12-07T19:34:04.658+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6ee4dfe7:::smithi13140796-47:4 got.empty() 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6ee4dfe7:::smithi13140796-47:4 4 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'15 (0'0,27'15] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'15], crt=27'15) [27'16 (21'12) clone 1:6ee4dfe7:::smithi13140796-47:4 by unknown.0.0:0 2019-12-07T19:33:58.643819+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'17 (21'12) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:193 0.000000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=27'15 lcod 27'15 active+remapped mbc={} ps=[2~1]] add_log_entry 27'16 (21'12) clone 1:6ee4dfe7:::smithi13140796-47:4 by unknown.0.0:0 2019-12-07T19:33:58.643819+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=27'15 lcod 27'15 active+remapped mbc={} ps=[2~1]] add_log_entry 27'17 (21'12) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:193 0.000000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=27'15 lcod 27'15 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 17 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=27'15 lcod 27'15 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.658+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740b500 seq 35 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:4# key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:4# 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1121 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.onode(0x558c575ba000).flush flush done 2019-12-07T19:34:04.658+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# 0x0 2019-12-07T19:34:04.658+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:4# 0x0~30000 -> 0x0~30000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x0~2000: 0x0~2000 Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10270 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5742abd0 blob([0x22b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5742abd0 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) get_ref 0x0~2000 Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x2000~2000: 0x0~2000 Blob(0x558c5757e150 blob([0x22d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10271 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5757e150 blob([0x22d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5757e150 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) get_ref 0x0~2000 Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x4000~2000: 0x0~2000 Blob(0x558c5742acb0 blob([0x22f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10272 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5742acb0 blob([0x22f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5742acb0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) get_ref 0x0~2000 Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x6000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([0x231000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10273 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5742ad90 blob([0x231000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5742ad90 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2fc0) get_ref 0x0~2000 Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x8000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([0x233000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10274 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5742ae70 blob([0x233000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5742ae70 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2f50) get_ref 0x0~2000 Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0xa000~2000: 0x0~2000 Blob(0x558c5742af50 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10275 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5742af50 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5742af50 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2e70) get_ref 0x0~2000 Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0xc000~2000: 0x0~2000 Blob(0x558c5742b030 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10276 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5742b030 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5742b030 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2e00) get_ref 0x0~2000 Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2e00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0xe000~2000: 0x0~2000 Blob(0x558c5742b110 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10277 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5742b110 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 88 ==== MRecoveryReserve(1.7s4 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (crc 0 0 0) 0x558c575ccd20 con 0x558c54c64000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5742b110 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2d90) get_ref 0x0~2000 Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x10000~2000: 0x0~2000 Blob(0x558c574fde30 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10278 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c574fde30 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c574fde30 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2d20) get_ref 0x0~2000 Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2d20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574fdf10 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10279 2019-12-07T19:34:04.658+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c574fdf10 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 89 ==== pg_update_log_missing(1.7s4 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 ==== 258+0+0 (crc 0 0 0) 0x558c57303d40 con 0x558c54c64000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c574fdf10 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334f20 prio 127 cost 0 latency 0.000023 epoch 27 pg_update_log_missing(1.7s4 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2cb0) get_ref 0x0~2000 Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=pg_update_log_missing(1.7s4 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x14000~2000: 0x0~2000 Blob(0x558c54d3d8f0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10280 2019-12-07T19:34:04.658+0000 7f8f0117b700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.660076+0000 3 0.000102 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3d8f0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3d8f0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0117b700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2c40) get_ref 0x0~2000 Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2c40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:04.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=pg_update_log_missing(1.7s4 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x16000~2000: 0x0~2000 Blob(0x558c54d3d810 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10281 2019-12-07T19:34:04.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3d810 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=pg_update_log_missing(1.7s4 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3d810 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2bd0) get_ref 0x0~2000 Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 10 osd.7 27 dequeue_op 0x558c57334f20 prio 127 cost 0 latency 0.000065 pg_update_log_missing(1.7s4 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 pg pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x18000~2000: 0x0~2000 Blob(0x558c54d3d5e0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10282 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3d5e0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3d5e0 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: pg_update_log_missing(1.7s4 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2b60) get_ref 0x0~2000 Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f05183700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_update_log_missing op_trim_to = -- op_roll_forward_to = -- 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 20 update missing, append 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c54d3d7a0 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10283 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3d7a0 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f05183700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active+remapped mbc={} ps=[2~1]] append_log_entries_update_missing trim_to bool = 0 trim_to = 0'0 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3d7a0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2af0) get_ref 0x0~2000 Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c5757ff80 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10284 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5757ff80 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5757ff80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1500 seq 28 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2a80) get_ref 0x0~2000 Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c575bc000 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10285 2019-12-07T19:34:04.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000027.00000000000000000010' <- 0000000027.00000000000000000010 2019-12-07T19:34:04.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc000 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc000 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2a10) get_ref 0x0~2000 Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 4717 (1 ios * 4000 + 717 bytes) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:34:04.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x20000~2000: 0x0~2000 Blob(0x558c575bc0e0 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 sbid 0x0)) 2019-12-07T19:34:04.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10286 2019-12-07T19:34:04.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:04.659+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:04.659+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc0e0 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc0e0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d29a0) get_ref 0x0~2000 Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d29a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active+remapped mbc={} ps=[2~1]] op_applied version 27'10 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f05183700 10 osd.7 27 dequeue_op 0x558c57334f20 finish 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x22000~2000: 0x0~2000 Blob(0x558c575bc1c0 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10287 2019-12-07T19:34:04.659+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc1c0 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0b990700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc1c0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0b990700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2930) get_ref 0x0~2000 Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x24000~2000: 0x0~2000 Blob(0x558c575bc2a0 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10288 2019-12-07T19:34:04.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc2a0 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc2a0 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 lcod 21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2700) get_ref 0x0~2000 Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.7s0 GRANT e27/27) v3 -- 0x558c575cc000 con 0x558c54c64000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x26000~2000: 0x0~2000 Blob(0x558c575bc380 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10289 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc380 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc380 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0117b700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 lcod 21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.660254+0000 1 0.000030 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0117b700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 lcod 21'9 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) get_ref 0x0~2000 Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:04.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x28000~2000: 0x0~2000 Blob(0x558c575bc460 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10290 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc460 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc460 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) get_ref 0x0~2000 Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10291 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc540 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc540 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) get_ref 0x0~2000 Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c575bc620 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10292 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc620 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc620 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75810) get_ref 0x0~2000 Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75810) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c575bc700 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10293 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bc700 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bc700 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup new Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75f10) get_ref 0x0~2000 Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75f10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dirty_range mark inline shard dirty 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba608) dirty_range mark inline shard dirty 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:4# = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# snapset 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# snapset = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# 1 keys 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# 1 keys = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:head#11 key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0xfffffffffffffffe0000000000000011'o' 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:head#11 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:head#11 = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1122 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 3 keys 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 3 keys = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~10000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~10000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~10000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740b500 8 blobs 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x10000/1000,10000,0 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2a6000~10000/1000,10000,0 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2a6000~10000] 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75340 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75420 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75340) get_ref 0x0~2000 Blob(0x558c54d75340 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75420 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75340) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d75340 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75420 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75438 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3df80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3df80 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8620 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df80) get_ref 0x0~2000 Blob(0x558c54d3df80 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8620 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3df80 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8620 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54cf8638 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8700 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf8700) get_ref 0x0~2000 Blob(0x558c54cf8700 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf8700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54cf8700 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3880 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3810 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3880) get_ref 0x0~2000 Blob(0x558c574d3880 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3810 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c574d3880 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3810 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d37a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d37a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d37a0) get_ref 0x0~2000 Blob(0x558c574d37a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d37a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c574d37a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d36c0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3650 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d36c0) get_ref 0x0~2000 Blob(0x558c574d36c0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3650 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d36c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c574d36c0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3650 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3668 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d35e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d35e0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3570 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d35e0) get_ref 0x0~2000 Blob(0x558c574d35e0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3570 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d35e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c574d35e0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3570 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3588 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3500 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3490 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3500) get_ref 0x0~2000 Blob(0x558c574d3500 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3490 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3500) get_ref init 0x2000, 1000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c574d3500 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3490 sbid 0x0)) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d34a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x10000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba888) dirty_range mark inline shard dirty 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~10000 = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6772BF7E.4.smithi13140796-47..' <- OBJ_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47..' <- SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000027.00000000000000000016' <- 0000000027.00000000000000000016 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000027.00000000000000000017' <- 0000000027.00000000000000000017 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 105955 (9 ios * 4000 + 69955 bytes) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes 0x558c575ba000,0x558c575ba500,0x558c575ba780 shared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) update 2#1:6ee4dfe7:::smithi13140796-47:head#11 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) update inline shard 651 bytes from 24 extents 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head#11 is 1142 (485 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba608) update 2#1:6ee4dfe7:::smithi13140796-47:4# 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba608) update inline shard 651 bytes from 24 extents 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:4# is 1069 (412 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba888) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba888) update inline shard 155 bytes from 8 extents 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 702 (541 bytes onode + 2 bytes spanning blobs + 159 bytes inline extents) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2))) 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[2a6000~10000] released 0x[] 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 freelist allocate 0x2a6000~10000 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740b500 2019-12-07T19:34:04.659+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=27'15 lcod 27'15 active+remapped mbc={} ps=[2~1]] op_applied version 27'17 2019-12-07T19:34:04.659+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c573349a0 finish 2019-12-07T19:34:04.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.659+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 aio_wait 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.659+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:04.659+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 567 < min_flush_size 524288 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5e7ac~24e to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5e7ac 0x5e7ac~24e to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2e7ac 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7ac 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9fa and padding block with 0x606 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5e9fa 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000135757s (0.000000118s flush + 0.000135639s kv commit) 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:04.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:04.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:04.659+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5638 < min_flush_size 524288 2019-12-07T19:34:04.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5997 < min_flush_size 524288 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba000 had 1 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba500 had 1 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba780 had 1 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6020 < min_flush_size 524288 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6020 < min_flush_size 524288 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5e9fa~1784 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5e9fa 0x5e9fa~1784 to file(ino 16 size 0x60000 mtime 2019-12-07T19:34:04.604798+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _allocate len 0x17e from 0 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _allocate len 0x17e from 1 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd190000 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd190000~10000/10000,10000,acd190000 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x70000 to include allocated 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 28 (was clean) 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x2e9fa 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9fa 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x17e and padding block with 0xe82 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x70000 2019-12-07T19:34:04.660+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.660+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_update_log_missing_reply(1.7s0 epoch 27/26 rep_tid 110 lcod 27'10) v3 -- 0x558c54d76a00 con 0x558c54c64000 2019-12-07T19:34:04.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6017e 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (28) on file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]), flushing log 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 28 len 0x49 crc 0x3af0a9ce) 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x1b000~1000 to file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x1b000 0x1b000~1000 to file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1b000 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x1c000 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:04.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000018 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 28 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001000253s (0.000003379s flush + 0.000996874s kv commit) 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:04.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 0x558c575ba000,0x558c575ba500,0x558c575ba780 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8638 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a900 space 0x558c54cf8638 0x0~2000 writing nocache) 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75438 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a990 space 0x558c54d75438 0x0~2000 writing nocache) 2019-12-07T19:34:04.660+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d34a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548dfd40 space 0x558c574d34a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3588 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548dfcb0 space 0x558c574d3588 0x0~2000 writing nocache) 2019-12-07T19:34:04.660+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=60, last_complete=27'17, committed=1, applied=1)) v2 -- 0x558c54d76c00 con 0x558c54c5bc00 2019-12-07T19:34:04.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3668 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548dfc20 space 0x558c574d3668 0x0~2000 writing nocache) 2019-12-07T19:34:04.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548dfb90 space 0x558c574d3748 0x0~2000 writing nocache) 2019-12-07T19:34:04.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3828 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548dfb00 space 0x558c574d3828 0x0~2000 writing nocache) 2019-12-07T19:34:04.661+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548dfa70 space 0x558c574d3908 0x0~2000 writing nocache) 2019-12-07T19:34:04.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:04.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:04.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.661+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.661+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 90 ==== MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c575bb680 con 0x558c54c64000 2019-12-07T19:34:04.661+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c573351e0 prio 3 cost 0 latency 0.000016 epoch 27 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.661+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.661+0000 7f8f0117b700 10 osd.7 27 dequeue_op 0x558c573351e0 prio 3 cost 0 latency 0.000057 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.661+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.661+0000 7f8f0117b700 25 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.661+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~40000 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0xa000 (40960) 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.661+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d74948 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c548df680 space 0x558c54d74948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d74b08 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c551a9830 space 0x558c54d74b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d74c58 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c573d9950 space 0x558c54d74c58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d74a28 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c54c9a000 space 0x558c54d74a28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d3ce88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c573d99e0 space 0x558c54d3ce88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~40000 = 40960 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=40960 len=40960 2019-12-07T19:34:04.662+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:04.662+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:04.662+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:04.662+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 40960 2019-12-07T19:34:04.662+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key 2019-12-07T19:34:04.662+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key = 0 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:04.662+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=111, attrs_read=0)) v2 -- 0x558c54ce3180 con 0x558c54c64000 2019-12-07T19:34:04.662+0000 7f8f0117b700 10 osd.7 27 dequeue_op 0x558c573351e0 finish 2019-12-07T19:34:04.662+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.667+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 55 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=63, reqid=client.4540.0:193, at_version=0'0, trim_to=0'0, roll_forward_to=27'17)) v2 ==== 1080+0+0 (unknown 0 0 0) 0x558c54d87000 con 0x558c54c5bc00 2019-12-07T19:34:04.667+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57334840 prio 127 cost 0 latency 0.000022 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=63, reqid=client.4540.0:193, at_version=0'0, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.667+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=63, reqid=client.4540.0:193, at_version=0'0, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=63, reqid=client.4540.0:193, at_version=0'0, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=63, reqid=client.4540.0:193, at_version=0'0, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334840 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=63, reqid=client.4540.0:193, at_version=0'0, trim_to=0'0, roll_forward_to=27'17)) v2 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=63, reqid=client.4540.0:193, at_version=0'0, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'15 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'17], crt=27'15) [] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'16 (21'12) clone 1:6ee4dfe7:::smithi13140796-47:4 by unknown.0.0:0 2019-12-07T19:33:58.643819+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'17 (21'12) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:193 0.000000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 17 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.667+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d20900 seq 36 2019-12-07T19:34:04.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#11 onode 0x558c575ba000 txc 0x558c54d20900 2019-12-07T19:34:04.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#11 0x0 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5742abd0) put_ref 0x0~2000 Blob(0x558c5742abd0 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5757e150) put_ref 0x0~2000 Blob(0x558c5757e150 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5742acb0) put_ref 0x0~2000 Blob(0x558c5742acb0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) put_ref 0x0~2000 Blob(0x558c5742ad90 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) put_ref 0x0~2000 Blob(0x558c5742ae70 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5742af50) put_ref 0x0~2000 Blob(0x558c5742af50 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5742b030) put_ref 0x0~2000 Blob(0x558c5742b030 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5742b110) put_ref 0x0~2000 Blob(0x558c5742b110 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c574fde30) put_ref 0x0~2000 Blob(0x558c574fde30 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c574fdf10) put_ref 0x0~2000 Blob(0x558c574fdf10 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d8f0) put_ref 0x0~2000 Blob(0x558c54d3d8f0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d810) put_ref 0x0~2000 Blob(0x558c54d3d810 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d5e0) put_ref 0x0~2000 Blob(0x558c54d3d5e0 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3d7a0) put_ref 0x0~2000 Blob(0x558c54d3d7a0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c5757ff80) put_ref 0x0~2000 Blob(0x558c5757ff80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc000) put_ref 0x0~2000 Blob(0x558c575bc000 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc0e0) put_ref 0x0~2000 Blob(0x558c575bc0e0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc1c0) put_ref 0x0~2000 Blob(0x558c575bc1c0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc2a0) put_ref 0x0~2000 Blob(0x558c575bc2a0 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc380) put_ref 0x0~2000 Blob(0x558c575bc380 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc460) put_ref 0x0~2000 Blob(0x558c575bc460 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc540) put_ref 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) put_ref 0x0~2000 Blob(0x558c575bc620 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) put_ref 0x0~2000 Blob(0x558c575bc700 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba108) dirty_range mark inline shard dirty 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c5742abd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22b000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c5757e150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22d000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c5742acb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22f000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x231000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x233000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c5742af50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x235000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c5742b030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x237000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c5742b110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x239000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c574fde30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23b000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c574fdf10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23d000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c54d3d8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23f000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c54d3d810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x241000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c54d3d5e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x243000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c54d3d7a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x245000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c5757ff80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x247000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c575bc000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x249000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c575bc0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24b000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c575bc1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24d000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c575bc2a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24f000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c575bc380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x251000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c575bc460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x253000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c575bc540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x255000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c575bc620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x257000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c575bc700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x259000~2000] 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c575ba780 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#11 = 0 2019-12-07T19:34:04.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20900 cost 5615 (1 ios * 4000 + 1615 bytes) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20900 onodes shared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1))) 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20900 allocated 0x[] released 0x[] 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 prepare 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20900 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 io_done 2019-12-07T19:34:04.667+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334840 finish 2019-12-07T19:34:04.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.667+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.667+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.667+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1967 < min_flush_size 524288 2019-12-07T19:34:04.667+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba000 had 1 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1990 < min_flush_size 524288 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1990 < min_flush_size 524288 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6017e~7c6 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6017e 0x6017e~7c6 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3017e 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x17e 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x944 and padding block with 0x6bc 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x60944 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000128272s (0.000000101s flush + 0.000128171s kv commit) 2019-12-07T19:34:04.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20900> 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 kv_submitted 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20900 2019-12-07T19:34:04.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 finishing 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20900 onodes 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20900 done 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20900 [] 2019-12-07T19:34:04.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.668+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=63, last_complete=27'17, committed=1, applied=1)) v2 -- 0x558c54d76000 con 0x558c54c5bc00 2019-12-07T19:34:04.668+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.670+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 91 ==== MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c54ce3180 con 0x558c54c64000 2019-12-07T19:34:04.670+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c573349a0 prio 3 cost 0 latency 0.000017 epoch 27 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.670+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.670+0000 7f8f05183700 10 osd.7 27 dequeue_op 0x558c573349a0 prio 3 cost 0 latency 0.000045 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.670+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.670+0000 7f8f05183700 25 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.670+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~40000 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x13000 (77824) 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:04.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~40000 = 77824 2019-12-07T19:34:04.670+0000 7f8f05183700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=77824 len=77824 2019-12-07T19:34:04.670+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.670+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:04.670+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:04.670+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:34:04.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:34:04.670+0000 7f8f05183700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.670+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=113, attrs_read=0)) v2 -- 0x558c5756f180 con 0x558c54c64000 2019-12-07T19:34:04.670+0000 7f8f05183700 10 osd.7 27 dequeue_op 0x558c573349a0 finish 2019-12-07T19:34:04.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.676+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 265440 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.681+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 92 ==== MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c5756f180 con 0x558c54c64000 2019-12-07T19:34:04.681+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334160 prio 3 cost 0 latency 0.000025 epoch 27 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.681+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.681+0000 7f8f0117b700 10 osd.7 27 dequeue_op 0x558c57334160 prio 3 cost 0 latency 0.000072 MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.681+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s4 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.681+0000 7f8f0117b700 25 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.681+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x0~40000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x19000 (102400) 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e57f10 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e57f10 sbid 0x0)) need {<0x0, 0x1000> : [0x18000:0~1000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d3d3c8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c573d9a70 space 0x558c54d3d3c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d3d4a8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c573d9b00 space 0x558c54d3d4a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e56638 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c551a98c0 space 0x558c54e56638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d3d588 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c551a9b00 space 0x558c54d3d588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e56088 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c548deb40 space 0x558c54e56088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57ac8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c54c9a7e0 space 0x558c54e57ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57908 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c54c9b200 space 0x558c54e57908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57518 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c54c9b3b0 space 0x558c54e57518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57ba8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c54c9afc0 space 0x558c54e57ba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57c88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c54c9b050 space 0x558c54e57c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57d68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c54c9b7a0 space 0x558c54e57d68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.681+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57e48 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c573d8900 space 0x558c54e57e48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e57f10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18000:0~1000]} 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54e57f28 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c573d8c60 space 0x558c54e57f28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.682+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x0~40000 = 102400 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=102400 len=102400 2019-12-07T19:34:04.682+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:04.682+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:04.682+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:34:04.682+0000 7f8f0117b700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 102400 2019-12-07T19:34:04.682+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:34:04.682+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:04.682+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=114, attrs_read=0)) v2 -- 0x558c575bb680 con 0x558c54c64000 2019-12-07T19:34:04.682+0000 7f8f0117b700 10 osd.7 27 dequeue_op 0x558c57334160 finish 2019-12-07T19:34:04.682+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 56 ==== MRecoveryReserve(1.6s2 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c575cde00 con 0x558c54c5bc00 2019-12-07T19:34:04.682+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.6s2 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.682+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.682+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.682+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.682+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.682+0000 7f8f0197c700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.683759+0000 4 0.000106 2019-12-07T19:34:04.682+0000 7f8f0197c700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.682+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:04.682+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7f8f0b990700 15 osd.7 27 enqueue_peering_evt 1.6s2 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.682+0000 7f8f0b990700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.682+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.682+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.682+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.682+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.6s0 GRANT e27/27) v3 -- 0x558c54d9e1e0 con 0x558c54c5bc00 2019-12-07T19:34:04.682+0000 7f8f05984700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.683827+0000 1 0.000029 2019-12-07T19:34:04.682+0000 7f8f05984700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.682+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:04.682+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.684+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 57 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=64, to_read={1:6336c6f2:::smithi13140796-16:head=0,262144,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:04.684+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c5740cdc0 prio 3 cost 0 latency 0.000018 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=64, to_read={1:6336c6f2:::smithi13140796-16:head=0,262144,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.684+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=64, to_read={1:6336c6f2:::smithi13140796-16:head=0,262144,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=64, to_read={1:6336c6f2:::smithi13140796-16:head=0,262144,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=64, to_read={1:6336c6f2:::smithi13140796-16:head=0,262144,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c5740cdc0 prio 3 cost 0 latency 0.000054 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=64, to_read={1:6336c6f2:::smithi13140796-16:head=0,262144,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=64, to_read={1:6336c6f2:::smithi13140796-16:head=0,262144,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.684+0000 7f8f0197c700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.684+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~40000 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x10000 (65536) 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~40000 = 65536 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=65536 len=65536 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 65536 2019-12-07T19:34:04.684+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key = 0 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:04.684+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=64, attrs_read=0)) v2 -- 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:34:04.684+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c5740cdc0 finish 2019-12-07T19:34:04.684+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.686+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 93 ==== MRecoveryReserve(1.7s4 RELEASE e27/27) v3 ==== 30+0+0 (crc 0 0 0) 0x558c575cc000 con 0x558c54c64000 2019-12-07T19:34:04.686+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.686+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.686+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.686+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.686+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.686+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558c54d5f000 2019-12-07T19:34:04.686+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.686+0000 7f8f05183700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.687766+0000 1 0.000022 2019-12-07T19:34:04.686+0000 7f8f05183700 5 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.686+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:04.686+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 18 ==== MRecoveryReserve(1.2s4 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54d9e1e0 con 0x558c54c64400 2019-12-07T19:34:04.687+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.2s4 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.687+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558c5731a000 2019-12-07T19:34:04.687+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.687+0000 7f8f03980700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.688473+0000 3 0.000199 2019-12-07T19:34:04.687+0000 7f8f03980700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.687+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:04.687+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7f8f0b990700 15 osd.7 27 enqueue_peering_evt 1.2s4 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558c5731a000 2019-12-07T19:34:04.687+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MRecoveryReserve(1.2s0 GRANT e27/27) v3 -- 0x558c576dc960 con 0x558c54c64400 2019-12-07T19:34:04.687+0000 7f8f07988700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.688538+0000 1 0.000026 2019-12-07T19:34:04.687+0000 7f8f07988700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.687+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:04.687+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.688+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 58 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=65, to_read={1:6ad15c4c:::smithi13140796-31:head=0,262144,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:34:04.688+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c54e79600 prio 3 cost 0 latency 0.000018 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=65, to_read={1:6ad15c4c:::smithi13140796-31:head=0,262144,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.688+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=65, to_read={1:6ad15c4c:::smithi13140796-31:head=0,262144,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.688+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=65, to_read={1:6ad15c4c:::smithi13140796-31:head=0,262144,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.688+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=65, to_read={1:6ad15c4c:::smithi13140796-31:head=0,262144,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.688+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c54e79600 prio 3 cost 0 latency 0.000055 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=65, to_read={1:6ad15c4c:::smithi13140796-31:head=0,262144,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.688+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=65, to_read={1:6ad15c4c:::smithi13140796-31:head=0,262144,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.688+0000 7f8f05984700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.688+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~40000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x1f000 (126976) 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 sbid 0x0)) need {<0x0, 0x1000> : [0x1e000:0~1000]} 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.688+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54e43eb8 in 0x558c53a58fc0) _discard 0x1000~1000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54c9a360 space 0x558c54e43eb8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57538c58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54c9a3f0 space 0x558c57538c58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57538da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54c9a480 space 0x558c57538da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57538e88 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54c9a750 space 0x558c57538e88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57538f68 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a8870 space 0x558c57538f68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539048 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a9440 space 0x558c57539048 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539128 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548debd0 space 0x558c57539128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539208 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548decf0 space 0x558c57539208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575392e8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548ded80 space 0x558c575392e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575393c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548dec60 space 0x558c575393c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575394a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548df050 space 0x558c575394a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539588 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548de1b0 space 0x558c57539588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539668 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548de2d0 space 0x558c57539668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539748 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548dee10 space 0x558c57539748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a8a20 space 0x558c57539828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e000:0~1000]} 2019-12-07T19:34:04.689+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57539908 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:34:04.689+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a87e0 space 0x558c57539908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~40000 = 126976 2019-12-07T19:34:04.689+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=126976 len=126976 2019-12-07T19:34:04.689+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:04.689+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:04.689+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:04.689+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:04.689+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key 2019-12-07T19:34:04.689+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key = 0 2019-12-07T19:34:04.689+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:04.689+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=65, attrs_read=0)) v2 -- 0x558c574a0a00 con 0x558c54c5bc00 2019-12-07T19:34:04.689+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:04.689+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.691+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 19 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (crc 0 0 0) 0x558c5730dd40 con 0x558c54c64400 2019-12-07T19:34:04.691+0000 7f8f1199c700 20 osd.7 27 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.691+0000 7f8f1199c700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:04.691+0000 7f8f1199c700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:04.691+0000 7f8f1199c700 20 osd.7 27 _dispatch 0x558c5730dd40 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.691+0000 7f8f1199c700 3 osd.7 27 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.691+0000 7f8f1199c700 10 osd.7 27 no new maps here, dropping 2019-12-07T19:34:04.691+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 20 ==== MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 181286+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c64400 2019-12-07T19:34:04.691+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57335080 prio 127 cost 181224 latency 0.000033 epoch 27 MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.692+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 181224 e27) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.692+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 181224 e27) queued 2019-12-07T19:34:04.692+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.692+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 181224 e27) pg 0x558c5731a000 2019-12-07T19:34:04.692+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c57335080 prio 127 cost 181224 latency 0.000076 MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] 2019-12-07T19:34:04.692+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.692+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] on_local_recover: 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.692+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=4 mbc={} ps=[2~1]] got missing 1:4986b721:::smithi13140796-43:head v 21'4 2019-12-07T19:34:04.692+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'4 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] last_complete now 21'4 log.complete_to 21'5 2019-12-07T19:34:04.692+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54d59b00 seq 4 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:4986b721:::smithi13140796-43:head# key 0x8480000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:4986b721:::smithi13140796-43:head# key 0x8480000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.692+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1123 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# = 0 2019-12-07T19:34:04.692+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 0x0~2c000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:4986b721:::smithi13140796-43:head# 0x0~2c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d59b00 22 blobs 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.692+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2c000/1000,2c000,0 2019-12-07T19:34:04.692+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2b6000~2c000/1000,2c000,0 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2b6000~2c000] 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578182a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578182a0 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818230 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578182a0) get_ref 0x0~2000 Blob(0x558c578182a0 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818230 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578182a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c578182a0 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818230 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578181c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578180e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578181c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578180e0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578181c0) get_ref 0x0~2000 Blob(0x558c578181c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578180e0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578181c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c578181c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578180e0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578180f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818070 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818000 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818070) get_ref 0x0~2000 Blob(0x558c57818070 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818000 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57818070 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818000 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818018 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818310 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818380 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818310) get_ref 0x0~2000 Blob(0x558c57818310 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818380 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57818310 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818380 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818398 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578183f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578183f0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818460 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578183f0) get_ref 0x0~2000 Blob(0x558c578183f0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818460 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578183f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c578183f0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818460 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818478 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578184d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578184d0 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818540 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578184d0) get_ref 0x0~2000 Blob(0x558c578184d0 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818540 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578184d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c578184d0 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818540 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818558 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578185b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578185b0 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578185b0) get_ref 0x0~2000 Blob(0x558c578185b0 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578185b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c578185b0 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818638 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818690 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818700 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818690) get_ref 0x0~2000 Blob(0x558c57818690 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818700 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57818690 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818700 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818718 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578187e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818770 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818770) get_ref 0x0~2000 Blob(0x558c57818770 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57818770 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578187f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578188c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818850 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818850) get_ref 0x0~2000 Blob(0x558c57818850 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57818850 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578188d8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578189a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818930 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818930) get_ref 0x0~2000 Blob(0x558c57818930 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57818930 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578189b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818a10 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818a10) get_ref 0x0~2000 Blob(0x558c57818a10 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57818a10 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818a98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818af0 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818af0) get_ref 0x0~2000 Blob(0x558c57818af0 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c57818af0 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818b78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818bd0 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818bd0) get_ref 0x0~2000 Blob(0x558c57818bd0 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57818bd0 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818c58 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818cb0 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818d20 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818cb0) get_ref 0x0~2000 Blob(0x558c57818cb0 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818d20 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57818cb0 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818d20 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818d38 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818d90 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818e00 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818d90) get_ref 0x0~2000 Blob(0x558c57818d90 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818e00 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57818d90 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818e00 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818e18 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818e70 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818ee0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818e70) get_ref 0x0~2000 Blob(0x558c57818e70 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818ee0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57818e70 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818ee0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818ef8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818f50 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818fc0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818f50) get_ref 0x0~2000 Blob(0x558c57818f50 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818fc0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57818f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57818f50 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818fc0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57818fd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578190a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819030 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57819030) get_ref 0x0~2000 Blob(0x558c57819030 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57819030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57819030 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578190b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819180 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57819110) get_ref 0x0~2000 Blob(0x558c57819110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819180 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c57819110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57819110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819180 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57819198 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578191f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578191f0 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819260 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578191f0) get_ref 0x0~2000 Blob(0x558c578191f0 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819260 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578191f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c578191f0 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819260 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57819278 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578192d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578192d0 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578192d0) get_ref 0x0~2000 Blob(0x558c578192d0 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.blob(0x558c578192d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c578192d0 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57819358 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2c000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.extentmap(0x558c54d96d88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 0x0~2c000 = 0 2019-12-07T19:34:04.692+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 4 keys 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 4 keys = 0 2019-12-07T19:34:04.692+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045c'.missing/0000000000000001.2916DE48.head.smithi13140796-43..' <- missing/0000000000000001.2916DE48.head.smithi13140796-43.. 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.692+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 274461 (23 ios * 4000 + 182461 bytes) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes 0x558c54d96c80 shared_blobs 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.extentmap(0x558c54d96d88) update 4#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore.extentmap(0x558c54d96d88) update inline shard 421 bytes from 22 extents 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:4986b721:::smithi13140796-43:head# is 912 (485 bytes onode + 2 bytes spanning blobs + 425 bytes inline extents) 2019-12-07T19:34:04.692+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[2b6000~2c000] released 0x[] 2019-12-07T19:34:04.692+0000 7f8f03980700 10 freelist allocate 0x2b6000~2c000 2019-12-07T19:34:04.692+0000 7f8f03980700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:04.692+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d59b00 2019-12-07T19:34:04.692+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c57335080 finish 2019-12-07T19:34:04.692+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.692+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'4 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.692+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.692+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 aio_wait 2019-12-07T19:34:04.692+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:04.692+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2235 < min_flush_size 524288 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2258 < min_flush_size 524288 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2258 < min_flush_size 524288 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x60944~8d2 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x60944 0x60944~8d2 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x30944 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x944 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x216 and padding block with 0xdea 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x61216 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.692+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.692+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000149342s (0.000040843s flush + 0.000108499s kv commit) 2019-12-07T19:34:04.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00> 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:04.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 0x558c54d96c80 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818018 in 0x558c53a595e0) _finish_write discard buffer(0x558c573d85a0 space 0x558c57818018 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578180f8 in 0x558c53a595e0) _finish_write discard buffer(0x558c551a9c20 space 0x558c578180f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818248 in 0x558c53a595e0) _finish_write discard buffer(0x558c573d86c0 space 0x558c57818248 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818398 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdebd0 space 0x558c57818398 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818478 in 0x558c53a595e0) _finish_write discard buffer(0x558c551a9a70 space 0x558c57818478 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818558 in 0x558c53a595e0) _finish_write discard buffer(0x558c551a9560 space 0x558c57818558 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818638 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdec60 space 0x558c57818638 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818718 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdecf0 space 0x558c57818718 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578187f8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cded80 space 0x558c578187f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578188d8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdee10 space 0x558c578188d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578189b8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdeea0 space 0x558c578189b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818a98 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdef30 space 0x558c57818a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'4 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] recovery_committed_to version 21'4 now ondisk 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818b78 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdefc0 space 0x558c57818b78 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818c58 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf050 space 0x558c57818c58 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818d38 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf0e0 space 0x558c57818d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818e18 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf170 space 0x558c57818e18 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818ef8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf200 space 0x558c57818ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818fd8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf290 space 0x558c57818fd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578190b8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf320 space 0x558c578190b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819198 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf3b0 space 0x558c57819198 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819278 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf440 space 0x558c57819278 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819358 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf4d0 space 0x558c57819358 0x0~2000 writing nocache) 2019-12-07T19:34:04.693+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 -- 0x558c5768b000 con 0x558c54c64400 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:04.693+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.693+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.694+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 59 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=66, to_read={1:639abb38:::smithi13140796-40:head=0,262144,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c54ce2000 con 0x558c54c5bc00 2019-12-07T19:34:04.694+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57334840 prio 3 cost 0 latency 0.000019 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=66, to_read={1:639abb38:::smithi13140796-40:head=0,262144,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.694+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=66, to_read={1:639abb38:::smithi13140796-40:head=0,262144,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=66, to_read={1:639abb38:::smithi13140796-40:head=0,262144,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=66, to_read={1:639abb38:::smithi13140796-40:head=0,262144,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334840 prio 3 cost 0 latency 0.000047 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=66, to_read={1:639abb38:::smithi13140796-40:head=0,262144,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=66, to_read={1:639abb38:::smithi13140796-40:head=0,262144,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.694+0000 7f8f0197c700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.694+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~40000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x29000 (167936) 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e310 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e4d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e5b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e770 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0cb0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0bd0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0a10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0930 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f07e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a0e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5742a1c0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e070 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a000 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279490 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a0e0 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5742a1c0 sbid 0x0)) need {<0x0, 0x1000> : [0x28000:0~1000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3810 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f07e0 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0930 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0a10 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0af0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0bd0 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0cb0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0d90 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e230 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e310 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e3f0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e4d0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e5b0 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e690 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e770 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e088 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfe60 space 0x558c5757e088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742a018 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c573d8e10 space 0x558c5742a018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c552794a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54c9b830 space 0x558c552794a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a0e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742a0f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfef0 space 0x558c5742a0f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5742a1c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x28000:0~1000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5742a1d8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c551a8cf0 space 0x558c5742a1d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54ea3828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde000 space 0x558c54ea3828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f07e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f07f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde7e0 space 0x558c574f07f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0930 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0948 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde750 space 0x558c574f0948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0a10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0a28 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde6c0 space 0x558c574f0a28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0b08 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c573d85a0 space 0x558c574f0b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0bd0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0be8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c573d86c0 space 0x558c574f0be8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0cb0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0cc8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c551a9c20 space 0x558c574f0cc8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574f0da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfd40 space 0x558c574f0da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54cf9588 in 0x558c53a58fc0) _discard 0x1000~1000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfcb0 space 0x558c54cf9588 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e248 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfc20 space 0x558c5757e248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e310 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e328 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfb90 space 0x558c5757e328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e3f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e408 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfb00 space 0x558c5757e408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e4d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e4e8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c548dfa70 space 0x558c5757e4e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e5b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e5c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54c9a900 space 0x558c5757e5c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e6a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54c9a990 space 0x558c5757e6a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e770 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5757e788 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde630 space 0x558c5757e788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~40000 = 167936 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=167936 len=167936 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 167936 2019-12-07T19:34:04.694+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key = 0 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:04.694+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=66, attrs_read=0)) v2 -- 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:04.694+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334840 finish 2019-12-07T19:34:04.694+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.698+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 21 ==== MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 160806+0+0 (crc 0 0 0) 0x558c5768b000 con 0x558c54c64400 2019-12-07T19:34:04.698+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57334160 prio 127 cost 160744 latency 0.000026 epoch 27 MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.698+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 160744 e27) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.698+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 160744 e27) queued 2019-12-07T19:34:04.698+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.698+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 160744 e27) pg 0x558c5731a000 2019-12-07T19:34:04.698+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c57334160 prio 127 cost 160744 latency 0.000065 MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.2s4( v 23'10 lc 21'4 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=3 mbc={} ps=[2~1]] 2019-12-07T19:34:04.698+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'4 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=3 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.698+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'4 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=3 mbc={} ps=[2~1]] on_local_recover: 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.698+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'4 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=3 mbc={} ps=[2~1]] got missing 1:41df217a:::smithi13140796-39:head v 21'5 2019-12-07T19:34:04.698+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'4 active m=2 mbc={} ps=[2~1]] last_complete now 21'5 log.complete_to 21'6 2019-12-07T19:34:04.698+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.698+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c574ff500 seq 5 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41df217a:::smithi13140796-39:head# key 0x84800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.698+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41df217a:::smithi13140796-39:head# key 0x84800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.698+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1124 2019-12-07T19:34:04.698+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# = 0 2019-12-07T19:34:04.698+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 0x0~27000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:41df217a:::smithi13140796-39:head# 0x0~27000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.698+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~27000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574ff500 20 blobs 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.698+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate 0x27000/1000,27000,0 2019-12-07T19:34:04.698+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2e2000~27000/1000,27000,0 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2e2000~27000] 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd8f0 blob([0x2e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f0a0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c575bd8f0) get_ref 0x0~2000 Blob(0x558c575bd8f0 blob([0x2e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f0a0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c575bd8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c575bd8f0 blob([0x2e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f0a0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5757f0b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75180 blob([0x2e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3f80 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c54d75180) get_ref 0x0~2000 Blob(0x558c54d75180 blob([0x2e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3f80 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c54d75180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d75180 blob([0x2e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3f80 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574d3f98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819d50 blob([0x2e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c57819d50) get_ref 0x0~2000 Blob(0x558c57819d50 blob([0x2e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c57819d50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57819d50 blob([0x2e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57819dd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819e30 blob([0x2e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819ea0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c57819e30) get_ref 0x0~2000 Blob(0x558c57819e30 blob([0x2e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819ea0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c57819e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57819e30 blob([0x2e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ea0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57819eb8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819f10 blob([0x2ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f80 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c57819f10) get_ref 0x0~2000 Blob(0x558c57819f10 blob([0x2ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f80 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c57819f10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57819f10 blob([0x2ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f80 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57819f98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de000 blob([0x2ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de070 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de000) get_ref 0x0~2000 Blob(0x558c578de000 blob([0x2ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de070 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c578de000 blob([0x2ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de070 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de088 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de0e0 blob([0x2ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de150 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de0e0) get_ref 0x0~2000 Blob(0x558c578de0e0 blob([0x2ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de150 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c578de0e0 blob([0x2ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de150 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de168 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de1c0 blob([0x2f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de230 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de1c0) get_ref 0x0~2000 Blob(0x558c578de1c0 blob([0x2f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de230 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c578de1c0 blob([0x2f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de230 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de2a0 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de310 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de2a0) get_ref 0x0~2000 Blob(0x558c578de2a0 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de310 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c578de2a0 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de310 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de328 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de380 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de380) get_ref 0x0~2000 Blob(0x558c578de380 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c578de380 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de408 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de460) get_ref 0x0~2000 Blob(0x558c578de460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c578de460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de4e8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de5b0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de540) get_ref 0x0~2000 Blob(0x558c578de540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de5b0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c578de540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de5b0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de5c8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de620) get_ref 0x0~2000 Blob(0x558c578de620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c578de620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de6a8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de770 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de700) get_ref 0x0~2000 Blob(0x558c578de700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de770 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c578de700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de770 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de7e0 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de850 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de7e0) get_ref 0x0~2000 Blob(0x558c578de7e0 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de850 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c578de7e0 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de850 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de868 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de8c0 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de8c0) get_ref 0x0~2000 Blob(0x558c578de8c0 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c578de8c0 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de948 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578dea10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de9a0 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dea10 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de9a0) get_ref 0x0~2000 Blob(0x558c578de9a0 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dea10 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578de9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c578de9a0 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea10 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578dea28 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578dea80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578dea80 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578dea80) get_ref 0x0~2000 Blob(0x558c578dea80 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578dea80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c578dea80 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578deb08 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578deb60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578debd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578deb60 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578debd0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578deb60) get_ref 0x0~2000 Blob(0x558c578deb60 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578debd0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578deb60) get_ref init 0x2000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c578deb60 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578debd0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578debe8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578dec40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578decb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578dec40 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578decb0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578dec40) get_ref 0x0~1000 Blob(0x558c578dec40 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578decb0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.blob(0x558c578dec40) get_ref init 0x1000, 1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~1000: 0x0~1000 Blob(0x558c578dec40 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578decb0 sbid 0x0)) 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578decc8 in 0x558c53a595e0) _discard 0x0~1000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x27000 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.698+0000 7f8f03980700 20 bluestore.extentmap(0x558c574a0b08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.698+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 0x0~27000 = 0 2019-12-07T19:34:04.698+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 4 keys 2019-12-07T19:34:04.699+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 4 keys = 0 2019-12-07T19:34:04.699+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045c'.missing/0000000000000001.28BF48E5.head.smithi13140796-39..' <- missing/0000000000000001.28BF48E5.head.smithi13140796-39.. 2019-12-07T19:34:04.699+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.699+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.699+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.699+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff500 cost 245981 (21 ios * 4000 + 161981 bytes) 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff500 onodes 0x558c574a0a00 shared_blobs 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore.extentmap(0x558c574a0b08) update 4#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore.extentmap(0x558c574a0b08) update inline shard 380 bytes from 20 extents 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:41df217a:::smithi13140796-39:head# is 871 (485 bytes onode + 2 bytes spanning blobs + 384 bytes inline extents) 2019-12-07T19:34:04.699+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff500 allocated 0x[2e2000~27000] released 0x[] 2019-12-07T19:34:04.699+0000 7f8f03980700 10 freelist allocate 0x2e2000~27000 2019-12-07T19:34:04.699+0000 7f8f03980700 20 freelist _xor first_key 0x280000 last_key 0x300000 2019-12-07T19:34:04.699+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 prepare 2019-12-07T19:34:04.699+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c574ff500 2019-12-07T19:34:04.699+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c57334160 finish 2019-12-07T19:34:04.699+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.699+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'4 active m=2 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.699+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.699+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 aio_wait 2019-12-07T19:34:04.699+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff500 2019-12-07T19:34:04.699+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 io_done 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2223 < min_flush_size 524288 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2246 < min_flush_size 524288 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2246 < min_flush_size 524288 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x61216~8c6 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x61216 0x61216~8c6 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x31216 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x216 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xadc and padding block with 0x524 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x61adc 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.699+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000120912s (0.000037599s flush + 0.000083313s kv commit) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff500> 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.699+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 kv_submitted 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff500 2019-12-07T19:34:04.699+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 finishing 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff500 onodes 0x558c574a0a00 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3f98 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf710 space 0x558c574d3f98 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f0b8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf680 space 0x558c5757f0b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819dd8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf7a0 space 0x558c57819dd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819eb8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf830 space 0x558c57819eb8 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819f98 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf9e0 space 0x558c57819f98 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de088 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfa70 space 0x558c578de088 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de168 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfb00 space 0x558c578de168 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de248 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfb90 space 0x558c578de248 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de328 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfc20 space 0x558c578de328 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de408 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfcb0 space 0x558c578de408 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de4e8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfd40 space 0x558c578de4e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de5c8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfdd0 space 0x558c578de5c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de6a8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfe60 space 0x558c578de6a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de788 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdfef0 space 0x558c578de788 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de868 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee000 space 0x558c578de868 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de948 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee090 space 0x558c578de948 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dea28 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee120 space 0x558c578dea28 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578deb08 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee1b0 space 0x558c578deb08 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578debe8 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee240 space 0x558c578debe8 0x0~2000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578decc8 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee2d0 space 0x558c578decc8 0x0~1000 writing nocache) 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff500 done 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.699+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff500 [] 2019-12-07T19:34:04.699+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.699+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'4 active m=2 mbc={} ps=[2~1]] recovery_committed_to version 21'5 now ondisk 2019-12-07T19:34:04.699+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 -- 0x558c551aec00 con 0x558c54c64400 2019-12-07T19:34:04.699+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.699+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 60 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=67, to_read={1:6c205177:::smithi13140796-8:4=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:4=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (unknown 0 0 0) 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:04.699+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57335ce0 prio 3 cost 0 latency 0.000016 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=67, to_read={1:6c205177:::smithi13140796-8:4=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:4=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.699+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=67, to_read={1:6c205177:::smithi13140796-8:4=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:4=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.699+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.699+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=67, to_read={1:6c205177:::smithi13140796-8:4=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:4=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.699+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.699+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=67, to_read={1:6c205177:::smithi13140796-8:4=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:4=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.699+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c57335ce0 prio 3 cost 0 latency 0.000048 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=67, to_read={1:6c205177:::smithi13140796-8:4=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:4=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.699+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=67, to_read={1:6c205177:::smithi13140796-8:4=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:4=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.699+0000 7f8f05984700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.699+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# 0x0~40000 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x8000 (32768) 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75260 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1)))) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552656c0 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314620 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd6c0 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75260 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1)))) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c552656c0 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1)))) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314620 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1)))) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575bd6c0 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1)))) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.699+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75260 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d75ac8 in 0x558c53a58fc0) _discard 0x1000~1000 2019-12-07T19:34:04.700+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a8b40 space 0x558c54d75ac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c552656c0 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1)))) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d753c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.700+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a8990 space 0x558c54d753c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314620 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1)))) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8018 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.700+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a8ab0 space 0x558c54cf8018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575bd6c0 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1)))) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.700+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf80f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.700+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c551a9cb0 space 0x558c54cf80f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.700+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# 0x0~40000 = 32768 2019-12-07T19:34:04.700+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=32768 len=32768 2019-12-07T19:34:04.700+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6c205177:::smithi13140796-8:4 2019-12-07T19:34:04.700+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6c205177:::smithi13140796-8:4 2019-12-07T19:34:04.700+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# 2019-12-07T19:34:04.700+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 32768 2019-12-07T19:34:04.700+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# hinfo_key 2019-12-07T19:34:04.700+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# hinfo_key = 0 2019-12-07T19:34:04.700+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6c205177:::smithi13140796-8:4 2019-12-07T19:34:04.700+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=67, attrs_read=0)) v2 -- 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:34:04.700+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c57335ce0 finish 2019-12-07T19:34:04.700+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.704+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 61 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=68, to_read={1:6c205177:::smithi13140796-8:head=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (unknown 0 0 0) 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:34:04.704+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c54e79080 prio 3 cost 0 latency 0.000018 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=68, to_read={1:6c205177:::smithi13140796-8:head=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.704+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=68, to_read={1:6c205177:::smithi13140796-8:head=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=68, to_read={1:6c205177:::smithi13140796-8:head=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=68, to_read={1:6c205177:::smithi13140796-8:head=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.704+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c54e79080 prio 3 cost 0 latency 0.000047 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=68, to_read={1:6c205177:::smithi13140796-8:head=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.704+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=68, to_read={1:6c205177:::smithi13140796-8:head=0,262144,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.704+0000 7f8f0197c700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.704+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~40000 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x13000 (77824) 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.704+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 sbid 0x0)) need {<0x0, 0x1000> : [0x12000:0~1000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314558 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde5a0 space 0x558c57314558 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573140f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde510 space 0x558c573140f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573141d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde480 space 0x558c573141d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573146a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde3f0 space 0x558c573146a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314788 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde240 space 0x558c57314788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314868 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde1b0 space 0x558c57314868 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314948 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde120 space 0x558c57314948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12000:0~1000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57314a28 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde090 space 0x558c57314a28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575bd3c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde870 space 0x558c575bd3c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573142b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde900 space 0x558c573142b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~40000 = 77824 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=77824 len=77824 2019-12-07T19:34:04.705+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:04.705+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:04.705+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:04.705+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:04.705+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key 2019-12-07T19:34:04.705+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key = 0 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:04.705+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=68, attrs_read=0)) v2 -- 0x558c54ce2000 con 0x558c54c5bc00 2019-12-07T19:34:04.705+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c54e79080 finish 2019-12-07T19:34:04.705+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.707+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 22 ==== MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 189478+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c64400 2019-12-07T19:34:04.707+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57334000 prio 127 cost 189416 latency 0.000025 epoch 27 MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.707+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 189416 e27) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.707+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 189416 e27) queued 2019-12-07T19:34:04.707+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.707+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 189416 e27) pg 0x558c5731a000 2019-12-07T19:34:04.707+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c57334000 prio 127 cost 189416 latency 0.000072 MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.2s4( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.707+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.707+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] on_local_recover: 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.707+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] got missing 1:41fe8a11:::smithi13140796-45:head v 21'6 2019-12-07T19:34:04.707+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'5 active m=1 mbc={} ps=[2~1]] last_complete now 21'9 log.complete_to 23'10 2019-12-07T19:34:04.707+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54d58300 seq 6 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41fe8a11:::smithi13140796-45:head# key 0x84800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41fe8a11:::smithi13140796-45:head# key 0x84800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.707+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1125 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# = 0 2019-12-07T19:34:04.707+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2e000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58300 23 blobs 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.707+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2e000/1000,2e000,0 2019-12-07T19:34:04.707+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x309000~2e000/1000,2e000,0 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x309000~2e000] 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa4d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8b60 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c574fa4d0) get_ref 0x0~2000 Blob(0x558c574fa4d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8b60 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c574fa4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c574fa4d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8b60 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c54cf8b78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bd650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9730 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd650 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c54cf9730) get_ref 0x0~2000 Blob(0x558c54cf9730 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd650 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c54cf9730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54cf9730 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd650 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575bd668 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df5e0 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df5e0) get_ref 0x0~2000 Blob(0x558c578df5e0 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c578df5e0 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df668 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df6c0 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df730 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df6c0) get_ref 0x0~2000 Blob(0x558c578df6c0 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df730 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c578df6c0 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df730 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df748 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df7a0 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df810 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df7a0) get_ref 0x0~2000 Blob(0x558c578df7a0 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df810 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c578df7a0 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df810 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df828 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df880 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df8f0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df880) get_ref 0x0~2000 Blob(0x558c578df880 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df8f0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c578df880 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df8f0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df908 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df960 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df9d0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df960) get_ref 0x0~2000 Blob(0x558c578df960 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df9d0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578df960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c578df960 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df9d0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df9e8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578dfa40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578dfab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578dfa40 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfab0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578dfa40) get_ref 0x0~2000 Blob(0x558c578dfa40 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfab0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578dfa40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c578dfa40 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dfab0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578dfac8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0000 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0070 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0000) get_ref 0x0~2000 Blob(0x558c578f0000 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0070 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c578f0000 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0070 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0088 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f00e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f00e0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f00e0) get_ref 0x0~2000 Blob(0x558c578f00e0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f00e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c578f00e0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0168 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f01c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f01c0 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0230 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f01c0) get_ref 0x0~2000 Blob(0x558c578f01c0 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0230 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f01c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c578f01c0 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0230 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f02a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f02a0 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0310 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f02a0) get_ref 0x0~2000 Blob(0x558c578f02a0 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0310 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f02a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c578f02a0 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0310 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0328 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f03f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0380 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f03f0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0380) get_ref 0x0~2000 Blob(0x558c578f0380 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f03f0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c578f0380 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f03f0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0408 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f04d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0460 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f04d0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0460) get_ref 0x0~2000 Blob(0x558c578f0460 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f04d0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c578f0460 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f04d0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f04e8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f05b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0540 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f05b0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0540) get_ref 0x0~2000 Blob(0x558c578f0540 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f05b0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c578f0540 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f05b0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f05c8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0620 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0690 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0620) get_ref 0x0~2000 Blob(0x558c578f0620 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0690 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c578f0620 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0690 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f06a8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0700 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0770 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0700) get_ref 0x0~2000 Blob(0x558c578f0700 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0770 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c578f0700 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0770 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f07e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f07e0 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0850 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f07e0) get_ref 0x0~2000 Blob(0x558c578f07e0 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0850 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f07e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c578f07e0 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0850 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0868 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f08c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f08c0 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0930 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f08c0) get_ref 0x0~2000 Blob(0x558c578f08c0 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0930 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f08c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c578f08c0 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0930 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0948 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f09a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f09a0 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f09a0) get_ref 0x0~2000 Blob(0x558c578f09a0 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f09a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c578f09a0 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0a28 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0a80 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0af0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0a80) get_ref 0x0~2000 Blob(0x558c578f0a80 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0af0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c578f0a80 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0af0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0b08 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0b60 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0bd0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0b60) get_ref 0x0~2000 Blob(0x558c578f0b60 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0bd0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0b60) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c578f0b60 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0bd0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0be8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0c40 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0cb0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0c40) get_ref 0x0~2000 Blob(0x558c578f0c40 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0cb0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.blob(0x558c578f0c40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c578f0c40 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0cb0 sbid 0x0)) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0cc8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2e000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.extentmap(0x558c54d96888) dirty_range mark inline shard dirty 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 = 0 2019-12-07T19:34:04.707+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 4 keys 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 4 keys = 0 2019-12-07T19:34:04.707+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045c'.missing/0000000000000001.28F71588.head.smithi13140796-45..' <- missing/0000000000000001.28F71588.head.smithi13140796-45.. 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.707+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 286653 (24 ios * 4000 + 190653 bytes) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes 0x558c54d96780 shared_blobs 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.extentmap(0x558c54d96888) update 4#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore.extentmap(0x558c54d96888) update inline shard 440 bytes from 23 extents 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:41fe8a11:::smithi13140796-45:head# is 931 (485 bytes onode + 2 bytes spanning blobs + 444 bytes inline extents) 2019-12-07T19:34:04.707+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[309000~2e000] released 0x[] 2019-12-07T19:34:04.707+0000 7f8f03980700 10 freelist allocate 0x309000~2e000 2019-12-07T19:34:04.707+0000 7f8f03980700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:34:04.707+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58300 2019-12-07T19:34:04.707+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c57334000 finish 2019-12-07T19:34:04.707+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.707+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'5 active m=1 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.707+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.707+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 aio_wait 2019-12-07T19:34:04.707+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:34:04.707+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:34:04.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.707+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.707+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2254 < min_flush_size 524288 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96780 had 1 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2277 < min_flush_size 524288 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2277 < min_flush_size 524288 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x61adc~8e5 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x61adc 0x61adc~8e5 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x31adc 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xadc 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3c1 and padding block with 0xc3f 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x623c1 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.708+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000149777s (0.000042522s flush + 0.000107255s kv commit) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.708+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300> 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:34:04.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 0x558c54d96780 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8b78 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee480 space 0x558c54cf8b78 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bd668 in 0x558c53a595e0) _finish_write discard buffer(0x558c578ee510 space 0x558c575bd668 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df668 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdec60 space 0x558c578df668 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df748 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdecf0 space 0x558c578df748 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df828 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cded80 space 0x558c578df828 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df908 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdee10 space 0x558c578df908 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df9e8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdeea0 space 0x558c578df9e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dfac8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdef30 space 0x558c578dfac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0088 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdefc0 space 0x558c578f0088 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0168 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf050 space 0x558c578f0168 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0248 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf0e0 space 0x558c578f0248 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0328 in 0x558c53a595e0) _finish_write discard buffer(0x558c551a9a70 space 0x558c578f0328 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0408 in 0x558c53a595e0) _finish_write discard buffer(0x558c551a9560 space 0x558c578f0408 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f04e8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf170 space 0x558c578f04e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f05c8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf200 space 0x558c578f05c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f06a8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf290 space 0x558c578f06a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0788 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf320 space 0x558c578f0788 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0868 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf3b0 space 0x558c578f0868 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0948 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf440 space 0x558c578f0948 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0a28 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf4d0 space 0x558c578f0a28 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0b08 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf710 space 0x558c578f0b08 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0be8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf680 space 0x558c578f0be8 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'5 active m=1 mbc={} ps=[2~1]] recovery_committed_to version 21'9 now ondisk 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0cc8 in 0x558c53a595e0) _finish_write discard buffer(0x558c54cdf7a0 space 0x558c578f0cc8 0x0~2000 writing nocache) 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.708+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:34:04.708+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.708+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.708+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 -- 0x558c5768b000 con 0x558c54c64400 2019-12-07T19:34:04.708+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 62 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=69, to_read={1:6ee4dfe7:::smithi13140796-47:4=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:4=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c54ce2000 con 0x558c54c5bc00 2019-12-07T19:34:04.709+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c573351e0 prio 3 cost 0 latency 0.000021 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=69, to_read={1:6ee4dfe7:::smithi13140796-47:4=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:4=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.709+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=69, to_read={1:6ee4dfe7:::smithi13140796-47:4=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:4=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.709+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=69, to_read={1:6ee4dfe7:::smithi13140796-47:4=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:4=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.709+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=69, to_read={1:6ee4dfe7:::smithi13140796-47:4=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:4=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.709+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c573351e0 prio 3 cost 0 latency 0.000060 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=69, to_read={1:6ee4dfe7:::smithi13140796-47:4=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:4=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.709+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=69, to_read={1:6ee4dfe7:::smithi13140796-47:4=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:4=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.709+0000 7f8f05984700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.709+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# 0x0~40000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x30000 (196608) 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1)))) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1)))) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1)))) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1)))) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1)))) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1)))) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1)))) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1)))) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1)))) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1)))) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1)))) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1)))) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1)))) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1)))) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1)))) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1)))) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1)))) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1)))) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1)))) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1)))) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1)))) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1)))) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1)))) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1)))) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1)))) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.709+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.709+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 23 ==== MOSDPGRecoveryDelete(1.2s4 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 ==== 123+0+0 (crc 0 0 0) 0x558c54c59200 con 0x558c54c64400 2019-12-07T19:34:04.709+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c5740c2c0 prio 3 cost 1000 latency 0.000013 epoch 27 MOSDPGRecoveryDelete(1.2s4 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 2019-12-07T19:34:04.709+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGRecoveryDelete(1.2s4 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 1000 e27) 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGRecoveryDelete(1.2s4 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 1000 e27) queued 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDPGRecoveryDelete(1.2s4 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 1000 e27) pg 0x558c5731a000 2019-12-07T19:34:04.709+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c5740c2c0 prio 3 cost 1000 latency 0.000043 MOSDPGRecoveryDelete(1.2s4 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 pg pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] handle_recovery_delete 0x558c5740c2c0 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] remove_missing_object 1:4c9e444c:::smithi13140796-48:head 23'10 2019-12-07T19:34:04.709+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.709+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54ea0300 seq 7 2019-12-07T19:34:04.709+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:4c9e444c:::smithi13140796-48:head# key 0x8480000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.709+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.709+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:04.709+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:04.709+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes shared_blobs 2019-12-07T19:34:04.709+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.709+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:34:04.709+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:34:04.709+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.709+0000 7f8f03980700 10 osd.7 27 dequeue_op 0x558c5740c2c0 finish 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x623c1~2e to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x623c1 0x623c1~2e to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x323c1 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3c1 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3ef and padding block with 0xc11 2019-12-07T19:34:04.709+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x623ef 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0300> 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000081240s (0.000000116s flush + 0.000081124s kv commit) 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.709+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] on_local_recover: 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:04.709+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 lc 21'9 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] got missing 1:4c9e444c:::smithi13140796-48:head v 23'10 2019-12-07T19:34:04.709+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'9 active mbc={} ps=[2~1]] last_complete now 23'10 log.complete_to at end 2019-12-07T19:34:04.709+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.709+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c574ff200 seq 8 2019-12-07T19:34:04.709+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045c'.missing/0000000000000001.23972223.head.smithi13140796-48..' <- missing/0000000000000001.23972223.head.smithi13140796-48.. 2019-12-07T19:34:04.709+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.709+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.709+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.709+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff200 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff200 onodes shared_blobs 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff200 allocated 0x[] released 0x[] 2019-12-07T19:34:04.709+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 prepare 2019-12-07T19:34:04.709+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff200 2019-12-07T19:34:04.709+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 io_done 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:04.709+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'9 active mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.709+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x623ef~4b8 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x623ef 0x623ef~4b8 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x323ef 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3ef 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8a7 and padding block with 0x759 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x628a7 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.709+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000120296s (0.000000076s flush + 0.000120220s kv commit) 2019-12-07T19:34:04.709+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff200> 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 kv_submitted 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff200 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 finishing 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff200 onodes 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 done 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff200 [] 2019-12-07T19:34:04.709+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.709+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.709+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'9 active mbc={} ps=[2~1]] recovery_committed_to version 23'10 now ondisk 2019-12-07T19:34:04.710+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_trim(1.2s0 23'10 e27/27) v2 -- 0x558c576dcb40 con 0x558c54c64400 2019-12-07T19:34:04.710+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 -- 0x558c551aec00 con 0x558c54c64400 2019-12-07T19:34:04.710+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1)))) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc6a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548deea0 space 0x558c575bc6a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1)))) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc788 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548df4d0 space 0x558c575bc788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1)))) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc328 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c548df8c0 space 0x558c575bc328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1)))) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc408 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee120 space 0x558c575bc408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1)))) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc4e8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee1b0 space 0x558c575bc4e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1)))) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc5c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee240 space 0x558c575bc5c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1)))) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc248 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54cdfa70 space 0x558c575bc248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1)))) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc168 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54cdfb00 space 0x558c575bc168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1)))) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575bc088 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54cdfb90 space 0x558c575bc088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1)))) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574e0638 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54cdfc20 space 0x558c574e0638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1)))) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574f1828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54cdfcb0 space 0x558c574f1828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1)))) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d3d668 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54cdfd40 space 0x558c54d3d668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1)))) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d3d6d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c54cdfdd0 space 0x558c54d3d6d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1)))) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d3d898 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee5a0 space 0x558c54d3d898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1)))) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574fdf98 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee630 space 0x558c574fdf98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1)))) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574fdeb8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee6c0 space 0x558c574fdeb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1)))) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5742b198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee750 space 0x558c5742b198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1)))) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5742b0b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee7e0 space 0x558c5742b0b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1)))) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5742afd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee870 space 0x558c5742afd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1)))) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5742aef8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee900 space 0x558c5742aef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1)))) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5742ae18 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578ee990 space 0x558c5742ae18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d3cb08 in 0x558c53a58fc0) _discard 0x1000~1000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578eea20 space 0x558c54d3cb08 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1)))) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54e56718 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578eeab0 space 0x558c54e56718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1)))) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.710+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5742ad38 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.710+0000 7f8f05984700 20 _add level 1 near 0 on buffer(0x558c578eeb40 space 0x558c5742ad38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.710+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# 0x0~40000 = 196608 2019-12-07T19:34:04.710+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=196608 len=196608 2019-12-07T19:34:04.710+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6ee4dfe7:::smithi13140796-47:4 2019-12-07T19:34:04.710+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6ee4dfe7:::smithi13140796-47:4 2019-12-07T19:34:04.710+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# 2019-12-07T19:34:04.710+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 196608 2019-12-07T19:34:04.710+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# hinfo_key 2019-12-07T19:34:04.710+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# hinfo_key = 0 2019-12-07T19:34:04.710+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6ee4dfe7:::smithi13140796-47:4 2019-12-07T19:34:04.710+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=69, attrs_read=0)) v2 -- 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:04.710+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c573351e0 finish 2019-12-07T19:34:04.710+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.711+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 24 ==== MRecoveryReserve(1.2s4 RELEASE e27/27) v3 ==== 30+0+0 (crc 0 0 0) 0x558c576dcb40 con 0x558c54c64400 2019-12-07T19:34:04.711+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.2s4 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.711+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.711+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.711+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.711+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.711+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558c5731a000 2019-12-07T19:34:04.711+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.711+0000 7f8f03980700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.712362+0000 1 0.000025 2019-12-07T19:34:04.711+0000 7f8f03980700 5 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.711+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:04.711+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.712+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 19 ==== MRecoveryReserve(1.cs0 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c575cc000 con 0x558c54c19c00 2019-12-07T19:34:04.712+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.712+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.712+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.712+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.712+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.712+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558c57676000 2019-12-07T19:34:04.712+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.712+0000 7f8f0297e700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.713473+0000 3 0.000232 2019-12-07T19:34:04.712+0000 7f8f0297e700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.712+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:04.712+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.712+0000 7f8f0b990700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.712+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558c57676000 2019-12-07T19:34:04.712+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.712+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MRecoveryReserve(1.cs1 GRANT e27/27) v3 -- 0x558c54d9e000 con 0x558c54c19c00 2019-12-07T19:34:04.712+0000 7f8f06986700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.713540+0000 1 0.000022 2019-12-07T19:34:04.712+0000 7f8f06986700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.712+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:04.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.714+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 20 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558c57325b00 con 0x558c54c19c00 2019-12-07T19:34:04.714+0000 7f8f1199c700 20 osd.7 27 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.714+0000 7f8f1199c700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:04.714+0000 7f8f1199c700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:04.714+0000 7f8f1199c700 20 osd.7 27 _dispatch 0x558c57325b00 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.714+0000 7f8f1199c700 3 osd.7 27 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.714+0000 7f8f1199c700 10 osd.7 27 no new maps here, dropping 2019-12-07T19:34:04.714+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 21 ==== MOSDPGPush(1.cs0 27/26 [PushOp(1:3c88320e:::smithi13140796-1:head, version: 21'4, data_included: [0~8192], data_size: 8192, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3c88320e:::smithi13140796-1:head@21'4, size: 32768, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:32768, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 9249+0+0 (unknown 0 0 0) 0x558c5768a400 con 0x558c54c19c00 2019-12-07T19:34:04.714+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335340 prio 127 cost 9192 latency 0.000024 epoch 27 MOSDPGPush(1.cs0 27/26 [PushOp(1:3c88320e:::smithi13140796-1:head, version: 21'4, data_included: [0~8192], data_size: 8192, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3c88320e:::smithi13140796-1:head@21'4, size: 32768, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:32768, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.714+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3c88320e:::smithi13140796-1:head, version: 21'4, data_included: [0~8192], data_size: 8192, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3c88320e:::smithi13140796-1:head@21'4, size: 32768, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:32768, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 9192 e27) 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3c88320e:::smithi13140796-1:head, version: 21'4, data_included: [0~8192], data_size: 8192, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3c88320e:::smithi13140796-1:head@21'4, size: 32768, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:32768, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 9192 e27) queued 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3c88320e:::smithi13140796-1:head, version: 21'4, data_included: [0~8192], data_size: 8192, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3c88320e:::smithi13140796-1:head@21'4, size: 32768, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:32768, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 9192 e27) pg 0x558c57676000 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c57335340 prio 127 cost 9192 latency 0.000082 MOSDPGPush(1.cs0 27/26 [PushOp(1:3c88320e:::smithi13140796-1:head, version: 21'4, data_included: [0~8192], data_size: 8192, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3c88320e:::smithi13140796-1:head@21'4, size: 32768, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:32768, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.cs0 27/26 [PushOp(1:3c88320e:::smithi13140796-1:head, version: 21'4, data_included: [0~8192], data_size: 8192, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3c88320e:::smithi13140796-1:head@21'4, size: 32768, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:32768, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] on_local_recover: 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'3 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=4 mbc={} ps=[2~1]] got missing 1:3c88320e:::smithi13140796-1:head v 21'4 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'7 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=3 mbc={} ps=[2~1]] last_complete now 21'7 log.complete_to 21'8 2019-12-07T19:34:04.714+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d58600 seq 5 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3c88320e:::smithi13140796-1:head# key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3c88320e:::smithi13140796-1:head# key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1126 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# = 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~2000 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3c88320e:::smithi13140796-1:head# 0x0~2000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58600 1 blobs 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x337000~2000/1000,2000,0 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x337000~2000] 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf87e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf87e0 blob([0x337000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a80 sbid 0x0)) 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf87e0) get_ref 0x0~2000 Blob(0x558c54cf87e0 blob([0x337000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a80 sbid 0x0)) 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf87e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54cf87e0 blob([0x337000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 sbid 0x0)) 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54cf8a98 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2000 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1008) dirty_range mark inline shard dirty 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~2000 = 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 4 keys 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 4 keys = 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000456'.missing/0000000000000001.C311C407.head.smithi13140796-1..' <- missing/0000000000000001.C311C407.head.smithi13140796-1.. 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58600 cost 18425 (2 ios * 4000 + 10425 bytes) 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58600 onodes 0x558c574a0f00 shared_blobs 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1008) update 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1008) update inline shard 22 bytes from 1 extents 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3c88320e:::smithi13140796-1:head# is 510 (482 bytes onode + 2 bytes spanning blobs + 26 bytes inline extents) 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58600 allocated 0x[337000~2000] released 0x[] 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 freelist allocate 0x337000~2000 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 prepare 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58600 2019-12-07T19:34:04.714+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c57335340 finish 2019-12-07T19:34:04.714+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.714+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'7 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=3 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.714+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.714+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 aio_wait 2019-12-07T19:34:04.714+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58600 2019-12-07T19:34:04.714+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 io_done 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.714+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1831 < min_flush_size 524288 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0f00 had 1 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.714+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1854 < min_flush_size 524288 2019-12-07T19:34:04.714+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1854 < min_flush_size 524288 2019-12-07T19:34:04.714+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.714+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x628a7~73e to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.714+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x628a7 0x628a7~73e to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x328a7 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8a7 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.714+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfe5 and padding block with 0x1b 2019-12-07T19:34:04.715+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x62fe5 2019-12-07T19:34:04.715+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.715+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.715+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.715+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000162387s (0.000036917s flush + 0.000125470s kv commit) 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.715+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58600> 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.715+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 kv_submitted 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58600 2019-12-07T19:34:04.715+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 finishing 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58600 onodes 0x558c574a0f00 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8a98 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578eec60 space 0x558c54cf8a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 done 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.715+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58600 [] 2019-12-07T19:34:04.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.715+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.715+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.715+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'7 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active+remapped m=3 mbc={} ps=[2~1]] recovery_committed_to version 21'7 now ondisk 2019-12-07T19:34:04.715+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGPushReply(1.cs1 27/26 [PushReplyOp(1:3c88320e:::smithi13140796-1:head)]) v3 -- 0x558c548cc000 con 0x558c54c19c00 2019-12-07T19:34:04.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.716+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 63 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=70, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:04.716+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57334160 prio 3 cost 0 latency 0.000021 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=70, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.716+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=70, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=70, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=70, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.716+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334160 prio 3 cost 0 latency 0.000078 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=70, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.716+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=70, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,262144,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.716+0000 7f8f0197c700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.716+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~40000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x10000 (65536) 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75340 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3df80 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8700 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3880 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d37a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d36c0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d35e0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3500 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8700 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3df80 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8620 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75340 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75420 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d3500 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3490 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d35e0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3570 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d36c0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3650 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d37a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3730 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d3880 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3810 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.716+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8700 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde360 space 0x558c574d3908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3df80 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54cf8638 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cde2d0 space 0x558c54cf8638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75340 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d75438 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cdebd0 space 0x558c54d75438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d3500 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d34a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cdf8c0 space 0x558c574d34a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d35e0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3588 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c54cdf950 space 0x558c574d3588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d36c0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3668 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c578eed80 space 0x558c574d3668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d37a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c578eee10 space 0x558c574d3748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d3880 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c574d3828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 _add level 1 near 0 on buffer(0x558c578eeea0 space 0x558c574d3828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~40000 = 65536 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=65536 len=65536 2019-12-07T19:34:04.717+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6ee4dfe7:::smithi13140796-47:head 2019-12-07T19:34:04.717+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6ee4dfe7:::smithi13140796-47:head 2019-12-07T19:34:04.717+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.717+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 65536 2019-12-07T19:34:04.717+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key 2019-12-07T19:34:04.717+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key = 0 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6ee4dfe7:::smithi13140796-47:head 2019-12-07T19:34:04.717+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=70, attrs_read=0)) v2 -- 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:34:04.717+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334160 finish 2019-12-07T19:34:04.717+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.720+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 22 ==== MOSDPGPush(1.cs0 27/26 [PushOp(1:3a5a3f5b:::smithi13140796-23:head, version: 21'8, data_included: [0~94208], data_size: 94208, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3a5a3f5b:::smithi13140796-23:head@21'8, size: 376832, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:376832, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 95270+0+0 (unknown 0 0 0) 0x558c548cc000 con 0x558c54c19c00 2019-12-07T19:34:04.720+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c573354a0 prio 127 cost 95208 latency 0.000024 epoch 27 MOSDPGPush(1.cs0 27/26 [PushOp(1:3a5a3f5b:::smithi13140796-23:head, version: 21'8, data_included: [0~94208], data_size: 94208, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3a5a3f5b:::smithi13140796-23:head@21'8, size: 376832, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:376832, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.720+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3a5a3f5b:::smithi13140796-23:head, version: 21'8, data_included: [0~94208], data_size: 94208, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3a5a3f5b:::smithi13140796-23:head@21'8, size: 376832, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:376832, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 95208 e27) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3a5a3f5b:::smithi13140796-23:head, version: 21'8, data_included: [0~94208], data_size: 94208, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3a5a3f5b:::smithi13140796-23:head@21'8, size: 376832, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:376832, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 95208 e27) queued 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3a5a3f5b:::smithi13140796-23:head, version: 21'8, data_included: [0~94208], data_size: 94208, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3a5a3f5b:::smithi13140796-23:head@21'8, size: 376832, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:376832, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 95208 e27) pg 0x558c57676000 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c573354a0 prio 127 cost 95208 latency 0.000076 MOSDPGPush(1.cs0 27/26 [PushOp(1:3a5a3f5b:::smithi13140796-23:head, version: 21'8, data_included: [0~94208], data_size: 94208, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3a5a3f5b:::smithi13140796-23:head@21'8, size: 376832, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:376832, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.cs0( v 21'12 lc 21'7 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=3 mbc={} ps=[2~1]] 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'7 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=3 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.cs0 27/26 [PushOp(1:3a5a3f5b:::smithi13140796-23:head, version: 21'8, data_included: [0~94208], data_size: 94208, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3a5a3f5b:::smithi13140796-23:head@21'8, size: 376832, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:376832, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'7 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=3 mbc={} ps=[2~1]] on_local_recover: 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'7 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=3 mbc={} ps=[2~1]] got missing 1:3a5a3f5b:::smithi13140796-23:head v 21'8 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'10 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'7 active+remapped m=2 mbc={} ps=[2~1]] last_complete now 21'10 log.complete_to 21'11 2019-12-07T19:34:04.720+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c548f6300 seq 6 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:head# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:head# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1127 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# = 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~17000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~17000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6300 12 blobs 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate 0x17000/1000,17000,0 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x339000~17000/1000,17000,0 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x339000~17000] 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573961c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57396230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573961c0 blob([0x339000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396230 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c573961c0) get_ref 0x0~2000 Blob(0x558c573961c0 blob([0x339000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396230 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c573961c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c573961c0 blob([0x339000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57396248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573962a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573962a0 blob([0x33b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 64 ==== MRecoveryReserve(1.6s2 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54d9e3c0 con 0x558c54c5bc00 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c573962a0) get_ref 0x0~2000 Blob(0x558c573962a0 blob([0x33b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c573962a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c573962a0 blob([0x33b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.6s2 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.720+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f18f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1f80 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f18f0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c574f1f80) get_ref 0x0~2000 Blob(0x558c574f1f80 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f18f0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c574f1f80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c574f1f80 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1908 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f19d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1960 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f19d0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.720+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1960) get_ref 0x0~2000 Blob(0x558c578f1960 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f19d0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.720+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c578f1960 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f19e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1a40 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1ab0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1a40) get_ref 0x0~2000 Blob(0x558c578f1a40 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1ab0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1a40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c578f1a40 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1ac8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1b20 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1b90 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f05984700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.721742+0000 1 0.000036 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1b20) get_ref 0x0~2000 Blob(0x558c578f1b20 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1b90 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1b20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c578f1b20 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1ba8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1c00 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c70 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f05984700 5 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1c00) get_ref 0x0~2000 Blob(0x558c578f1c00 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c70 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1c00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c578f1c00 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1c88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1ce0 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1ce0) get_ref 0x0~2000 Blob(0x558c578f1ce0 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c578f1ce0 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54d75e48 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57314000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57314000 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c57314000) get_ref 0x0~2000 Blob(0x558c57314000 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c57314000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57314000 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d2868 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa310 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc00 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c574fa310) get_ref 0x0~2000 Blob(0x558c574fa310 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc00 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c574fa310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c574fa310 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5742bc18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf96c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bd9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf96c0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd9d0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf96c0) get_ref 0x0~2000 Blob(0x558c54cf96c0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd9d0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf96c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c54cf96c0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c575bd9e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e05b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e05b0 blob([0x34f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c574e05b0) get_ref 0x0~1000 Blob(0x558c574e05b0 blob([0x34f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.blob(0x558c574e05b0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~1000: 0x0~1000 Blob(0x558c574e05b0 blob([0x34f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578dfeb8 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x17000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark inline shard dirty 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~17000 = 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 4 keys 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 4 keys = 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000456'.missing/0000000000000001.C5A5CFAD.head.smithi13140796-23..' <- missing/0000000000000001.C5A5CFAD.head.smithi13140796-23.. 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 148445 (13 ios * 4000 + 96445 bytes) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes 0x558c574a1180 shared_blobs 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) update inline shard 228 bytes from 12 extents 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 719 (485 bytes onode + 2 bytes spanning blobs + 232 bytes inline extents) 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[339000~17000] released 0x[] 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 freelist allocate 0x339000~17000 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6300 2019-12-07T19:34:04.720+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c573354a0 finish 2019-12-07T19:34:04.720+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.720+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'10 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'7 active+remapped m=2 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.720+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.720+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 aio_wait 2019-12-07T19:34:04.720+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:04.720+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:04.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2042 < min_flush_size 524288 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1180 had 1 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2065 < min_flush_size 524288 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2065 < min_flush_size 524288 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x62fe5~811 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x62fe5 0x62fe5~811 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x32fe5 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfe5 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7f6 and padding block with 0x80a 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x637f6 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.721+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000142574s (0.000044884s flush + 0.000097690s kv commit) 2019-12-07T19:34:04.721+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300> 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.721+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:04.721+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 0x558c574a1180 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75e48 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efa70 space 0x558c54d75e48 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396248 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54c9a120 space 0x558c57396248 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bc18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efb90 space 0x558c5742bc18 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d2868 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efb00 space 0x558c574d2868 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578ef710 space 0x558c5757e8d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bd9e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efc20 space 0x558c575bd9e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dfeb8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efcb0 space 0x558c578dfeb8 0x0~1000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1908 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578ef7a0 space 0x558c578f1908 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f19e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578ef830 space 0x558c578f19e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1ac8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578ef8c0 space 0x558c578f1ac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1ba8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578ef950 space 0x558c578f1ba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1c88 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578ef9e0 space 0x558c578f1c88 0x0~2000 writing nocache) 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.721+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:04.721+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.721+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.721+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'10 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'7 active+remapped m=2 mbc={} ps=[2~1]] recovery_committed_to version 21'10 now ondisk 2019-12-07T19:34:04.721+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGPushReply(1.cs1 27/26 [PushReplyOp(1:3a5a3f5b:::smithi13140796-23:head)]) v3 -- 0x558c5768a400 con 0x558c54c19c00 2019-12-07T19:34:04.721+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.721+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 49 ==== MRecoveryReserve(1.3s4 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54d9e000 con 0x558c54c5b400 2019-12-07T19:34:04.721+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.721+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.721+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.721+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.721+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.721+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558c5513e000 2019-12-07T19:34:04.721+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.721+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.722794+0000 3 0.000116 2019-12-07T19:34:04.721+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.721+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:04.721+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.721+0000 7f8f0b990700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.721+0000 7f8f0b990700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.721+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.721+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.721+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.721+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558c5513e000 2019-12-07T19:34:04.721+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.721+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.3s0 GRANT e27/27) v3 -- 0x558c54d38d20 con 0x558c54c5b400 2019-12-07T19:34:04.721+0000 7f8f07187700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.722881+0000 1 0.000034 2019-12-07T19:34:04.721+0000 7f8f07187700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.721+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.721+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.722+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 50 ==== MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=44, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558c575bb680 con 0x558c54c5b400 2019-12-07T19:34:04.722+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335600 prio 3 cost 0 latency 0.000022 epoch 27 MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=44, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.722+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=44, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=44, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=44, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c5513e000 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c57335600 prio 3 cost 0 latency 0.000054 MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=44, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=44, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.722+0000 7f8f0317f700 25 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x200~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e0c58 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c548de480 space 0x558c574e0c58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x200~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x600~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x600~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa00~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa00~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe00~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe00~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1200~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e0c58 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c551a8d80 space 0x558c574e0c58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1200~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1600~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1600~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a00~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a00~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e00~200 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.722+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.722+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e00~200 = 512 2019-12-07T19:34:04.722+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2200~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54d3c328 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c573d8d80 space 0x558c54d3c328 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2200~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2600~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2600~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3200~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54d3c328 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c548de750 space 0x558c54d3c328 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3200~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3600~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3600~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4200~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54d3c4e8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c548de7e0 space 0x558c54d3c4e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4200~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4600~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4600~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4a00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4a00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4e00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4e00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5200~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54d3c4e8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c548de870 space 0x558c54d3c4e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5200~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5600~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5600~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5a00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5a00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5e00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5e00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6200~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2018 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c548de900 space 0x558c574e2018 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6200~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6600~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6600~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6a00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6a00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6e00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6e00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7200~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2018 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c548de990 space 0x558c574e2018 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7200~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7600~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7600~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7a00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7a00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7e00~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.723+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7e00~200 = 512 2019-12-07T19:34:04.723+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8200~200 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.723+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e20f8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c548dea20 space 0x558c574e20f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8200~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8600~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8600~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8a00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8a00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8e00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8e00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9200~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e20f8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdfe60 space 0x558c574e20f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9200~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9600~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9600~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9a00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9a00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9e00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9e00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa200~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e21d8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdfef0 space 0x558c574e21d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa200~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa600~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa600~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xaa00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xaa00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xae00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xae00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb200~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e21d8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdec60 space 0x558c574e21d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb200~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb600~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb600~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xba00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xba00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xbe00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xbe00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc200~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e22b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdecf0 space 0x558c574e22b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc200~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc600~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc600~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xca00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xca00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xce00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xce00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd200~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e22b8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cded80 space 0x558c574e22b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd200~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd600~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd600~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xda00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xda00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xde00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xde00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe200~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2398 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdee10 space 0x558c574e2398 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe200~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe600~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe600~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xea00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xea00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xee00~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.724+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xee00~200 = 512 2019-12-07T19:34:04.724+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf200~200 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.724+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2398 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdeea0 space 0x558c574e2398 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf200~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf600~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf600~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfa00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfa00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfe00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfe00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10200~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2478 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdef30 space 0x558c574e2478 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10200~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10600~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10600~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10a00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10a00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10e00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10e00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11200~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2478 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdefc0 space 0x558c574e2478 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11200~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11600~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11600~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11a00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11a00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11e00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11e00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12200~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2558 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c551a9a70 space 0x558c574e2558 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12200~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12600~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12600~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12a00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12a00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12e00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12e00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13200~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2558 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c551a9560 space 0x558c574e2558 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13200~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13600~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13600~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13a00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13a00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13e00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13e00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14200~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2638 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf050 space 0x558c574e2638 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14200~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14600~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14600~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14a00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14a00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14e00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14e00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15200~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 65 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=71, reqid=client.4540.0:223, at_version=27'18, trim_to=0'0, roll_forward_to=27'17)) v2 ==== 2406+0+0 (unknown 0 0 0) 0x558c54e3e800 con 0x558c54c5bc00 2019-12-07T19:34:04.725+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c57334000 prio 127 cost 0 latency 0.000029 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=71, reqid=client.4540.0:223, at_version=27'18, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.725+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=71, reqid=client.4540.0:223, at_version=27'18, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2638 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf0e0 space 0x558c574e2638 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15200~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15600~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15600~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=71, reqid=client.4540.0:223, at_version=27'18, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15a00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.725+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=71, reqid=client.4540.0:223, at_version=27'18, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15a00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15e00~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.725+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15e00~200 = 512 2019-12-07T19:34:04.725+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16200~200 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334000 prio 127 cost 0 latency 0.000079 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=71, reqid=client.4540.0:223, at_version=27'18, trim_to=0'0, roll_forward_to=27'17)) v2 pg pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=71, reqid=client.4540.0:223, at_version=27'18, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.725+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'17 (0'0,27'17] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'17], crt=27'17) [27'18 (27'17) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:223 2019-12-07T19:34:04.725983+0000 0 ObjectCleanRegions clean_offsets: [0~262144,262144~18446744073709289471], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'18 (0'0,27'18] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'17 lcod 27'17 active+remapped mbc={} ps=[2~1]] add_log_entry 27'18 (27'17) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:223 2019-12-07T19:34:04.725983+0000 0 ObjectCleanRegions clean_offsets: [0~262144,262144~18446744073709289471], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'18 (0'0,27'18] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'17 lcod 27'17 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 18 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'18 (0'0,27'18] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'17 lcod 27'17 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.725+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'18, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.725+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d20600 seq 37 2019-12-07T19:34:04.725+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:04.725+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.725+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.725+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000027.00000000000000000018' <- 0000000027.00000000000000000018 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 6846 (1 ios * 4000 + 2846 bytes) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2718 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes 0x558c575ba780 shared_blobs 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf170 space 0x558c574e2718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba888) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16200~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 702 (541 bytes onode + 2 bytes spanning blobs + 159 bytes inline extents) 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16600~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16600~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16a00~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:04.726+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16a00~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16e00~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16e00~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17200~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'18 (0'0,27'18] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'17 lcod 27'17 active+remapped mbc={} ps=[2~1]] op_applied version 27'18 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.726+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c57334000 finish 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2720 < min_flush_size 524288 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba780 had 1 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2743 < min_flush_size 524288 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2743 < min_flush_size 524288 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x637f6~ab7 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x637f6 0x637f6~ab7 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x337f6 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7f6 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2ad and padding block with 0xd53 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2718 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf200 space 0x558c574e2718 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17200~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17600~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17600~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17a00~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17a00~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17e00~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17e00~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18200~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x642ad 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.726+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000261825s (0.000000081s flush + 0.000261744s kv commit) 2019-12-07T19:34:04.726+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20600> 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.726+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:04.726+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 0x558c575ba780 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.726+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:04.726+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.726+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.726+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=71, last_complete=27'18, committed=1, applied=1)) v2 -- 0x558c54d76200 con 0x558c54c5bc00 2019-12-07T19:34:04.726+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 25 ==== MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 3057+0+0 (crc 0 0 0) 0x558c575c1000 con 0x558c54c64400 2019-12-07T19:34:04.726+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.726+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c5740cdc0 prio 127 cost 0 latency 0.000037 epoch 27 MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.726+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e27f8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf290 space 0x558c574e27f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18200~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18600~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18600~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18a00~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18a00~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18e00~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.726+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18e00~200 = 512 2019-12-07T19:34:04.726+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19200~200 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.726+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.726+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 323360 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.726+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.726+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.726+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.726+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558c5731a000 2019-12-07T19:34:04.726+0000 7f8f07988700 10 osd.7 27 dequeue_op 0x558c5740cdc0 prio 127 cost 0 latency 0.000495 MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] 2019-12-07T19:34:04.726+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.726+0000 7f8f07988700 20 snap_mapper.add_oid 1:41df217a:::smithi13140796-39:4 4 2019-12-07T19:34:04.726+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.726+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:04.726+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.726+0000 7f8f07988700 20 snap_mapper.get_snaps 1:41df217a:::smithi13140796-39:4 got.empty() 2019-12-07T19:34:04.726+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 66 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=72, reqid=client.4540.0:224, at_version=27'19, trim_to=0'0, roll_forward_to=27'17)) v2 ==== 2229+0+0 (unknown 0 0 0) 0x558c574a3000 con 0x558c54c5bc00 2019-12-07T19:34:04.726+0000 7f8f07988700 20 snap_mapper.set_snaps 1:41df217a:::smithi13140796-39:4 4 2019-12-07T19:34:04.726+0000 7f8f07988700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.726+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c578d2000 prio 127 cost 0 latency 0.000030 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=72, reqid=client.4540.0:224, at_version=27'19, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.726+0000 7f8f07988700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.726+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=72, reqid=client.4540.0:224, at_version=27'19, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.726+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 23'10 (0'0,23'10] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log log((0'0,23'10], crt=23'10) [27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0] 2019-12-07T19:34:04.726+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'11 (0'0,27'11] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[2~1]] add_log_entry 27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.726+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[2~1]] add_log_entry 27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.726+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[2~1]] append_log approx pg log length = 12 2019-12-07T19:34:04.726+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.726+0000 7f8f07988700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.726+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.726+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.726+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54836f00 seq 9 2019-12-07T19:34:04.726+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41df217a:::smithi13140796-39:4# key 0x84800000000000000141df21'z!smithi13140796-39!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# -> 4#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1128 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.onode(0x558c574a0a00).flush flush done 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# 0x0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# -> 4#1:41df217a:::smithi13140796-39:4# 0x0~27000 -> 0x0~27000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x0~2000: 0x0~2000 Blob(0x558c575bd8f0 blob([0x2e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f0a0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10294 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c575bd8f0 blob([0x2e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f0a0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=72, reqid=client.4540.0:224, at_version=27'19, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c575bd8f0 blob([0x2e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c574e1ce0 blob([0x2e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=72, reqid=client.4540.0:224, at_version=27'19, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c574e1ce0) get_ref 0x0~2000 Blob(0x558c574e1ce0 blob([0x2e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c574e1ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c578d2000 prio 127 cost 0 latency 0.000178 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=72, reqid=client.4540.0:224, at_version=27'19, trim_to=0'0, roll_forward_to=27'17)) v2 pg pg[1.6s2( v 27'18 (0'0,27'18] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574e1ce0 blob([0x2e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d75180 blob([0x2e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3f80 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10295 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c54d75180 blob([0x2e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3f80 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c54d75180 blob([0x2e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'18 (0'0,27'18] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=72, reqid=client.4540.0:224, at_version=27'19, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c574e0930 blob([0x2e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'18 (0'0,27'18] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'18], crt=27'17) [27'19 (27'18) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:224 2019-12-07T19:34:04.725997+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 0, new_object: 0] 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c574e0930) get_ref 0x0~2000 Blob(0x558c574e0930 blob([0x2e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c574e0930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c574e0930 blob([0x2e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57819d50 blob([0x2e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10296 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c57819d50 blob([0x2e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c57819d50 blob([0x2e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57880bd0 blob([0x2e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57880bd0) get_ref 0x0~2000 Blob(0x558c57880bd0 blob([0x2e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57880bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57880bd0 blob([0x2e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57819e30 blob([0x2e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ea0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10297 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c57819e30 blob([0x2e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ea0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c57819e30 blob([0x2e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c574e0e00 blob([0x2e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c574e0e00) get_ref 0x0~2000 Blob(0x558c574e0e00 blob([0x2e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c574e0e00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c574e0e00 blob([0x2e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57819f10 blob([0x2ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f80 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10298 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c57819f10 blob([0x2ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f80 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c57819f10 blob([0x2ea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c5523a070 blob([0x2ea000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c5523a070) get_ref 0x0~2000 Blob(0x558c5523a070 blob([0x2ea000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c5523a070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5523a070 blob([0x2ea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c578de000 blob([0x2ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de070 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10299 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de000 blob([0x2ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de070 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de000 blob([0x2ec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c55279110 blob([0x2ec000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c55279110) get_ref 0x0~2000 Blob(0x558c55279110 blob([0x2ec000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c55279110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c55279110 blob([0x2ec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c578de0e0 blob([0x2ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de150 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10300 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de0e0 blob([0x2ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de150 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de0e0 blob([0x2ee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c55278ee0 blob([0x2ee000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c55278ee0) get_ref 0x0~2000 Blob(0x558c55278ee0 blob([0x2ee000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c55278ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c55278ee0 blob([0x2ee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c578de1c0 blob([0x2f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de230 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10301 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de1c0 blob([0x2f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de230 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de1c0 blob([0x2f0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c54d3c0e0 blob([0x2f0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c54d3c0e0) get_ref 0x0~2000 Blob(0x558c54d3c0e0 blob([0x2f0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c54d3c0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c54d3c0e0 blob([0x2f0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c578de2a0 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de310 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10302 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de2a0 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de310 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de2a0 blob([0x2f2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c5742bb20 blob([0x2f2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c5742bb20) get_ref 0x0~2000 Blob(0x558c5742bb20 blob([0x2f2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c5742bb20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5742bb20 blob([0x2f2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c578de380 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10303 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de380 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de380 blob([0x2f4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c578813b0 blob([0x2f4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'19 (0'0,27'19] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'18 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] add_log_entry 27'19 (27'18) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:224 2019-12-07T19:34:04.725997+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578813b0) get_ref 0x0~2000 Blob(0x558c578813b0 blob([0x2f4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578813b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c578813b0 blob([0x2f4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c578de460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10304 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de460 blob([0x2f6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57881420 blob([0x2f6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881420) get_ref 0x0~2000 Blob(0x558c57881420 blob([0x2f6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57881420 blob([0x2f6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'19 (0'0,27'19] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'18 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 19 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c578de540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de5b0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10305 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de5b0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de540 blob([0x2f8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'19 (0'0,27'19] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'18 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.727+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57881490 blob([0x2f8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'19, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881490) get_ref 0x0~2000 Blob(0x558c57881490 blob([0x2f8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c57881490 blob([0x2f8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c578de620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10306 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de620 blob([0x2fa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57881500 blob([0x2fa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881500) get_ref 0x0~2000 Blob(0x558c57881500 blob([0x2fa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881500) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57881500 blob([0x2fa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c578de700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de770 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10307 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de770 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de700 blob([0x2fc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57881570 blob([0x2fc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881570) get_ref 0x0~2000 Blob(0x558c57881570 blob([0x2fc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57881570 blob([0x2fc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c578de7e0 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de850 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10308 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de7e0 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de850 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de7e0 blob([0x2fe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0f00 seq 38 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c578815e0 blob([0x2fe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578815e0) get_ref 0x0~2000 Blob(0x558c578815e0 blob([0x2fe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578815e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c578815e0 blob([0x2fe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c578de8c0 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10309 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de8c0 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de8c0 blob([0x300000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57881650 blob([0x300000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881650) get_ref 0x0~2000 Blob(0x558c57881650 blob([0x300000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c57881650 blob([0x300000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x20000~2000: 0x0~2000 Blob(0x558c578de9a0 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea10 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10310 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578de9a0 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea10 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578de9a0 blob([0x302000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c578816c0 blob([0x302000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578816c0) get_ref 0x0~2000 Blob(0x558c578816c0 blob([0x302000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578816c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c578816c0 blob([0x302000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x22000~2000: 0x0~2000 Blob(0x558c578dea80 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10311 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578dea80 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578dea80 blob([0x304000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57881730 blob([0x304000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881730) get_ref 0x0~2000 Blob(0x558c57881730 blob([0x304000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57881730 blob([0x304000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x24000~2000: 0x0~2000 Blob(0x558c578deb60 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578debd0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10312 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578deb60 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578debd0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578deb60 blob([0x306000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c578817a0 blob([0x306000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578817a0) get_ref 0x0~2000 Blob(0x558c578817a0 blob([0x306000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c578817a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c578817a0 blob([0x306000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x26000~1000: 0x0~1000 Blob(0x558c578dec40 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578decb0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10313 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared Blob(0x558c578dec40 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578decb0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) make_blob_shared now Blob(0x558c578dec40 blob([0x308000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57881810 blob([0x308000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881810) get_ref 0x0~1000 Blob(0x558c57881810 blob([0x308000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.blob(0x558c57881810) get_ref init 0x1000, 1000 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x26000~1000: 0x0~1000 Blob(0x558c57881810 blob([0x308000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c5793a108) dirty_range mark inline shard dirty 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# -> 4#1:41df217a:::smithi13140796-39:4# = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# snapset 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# snapset = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# 1 keys 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# 1 keys = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41df217a:::smithi13140796-39:head#c key 0x84800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffe000000000000000c'o' 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# -> 4#1:41df217a:::smithi13140796-39:head#c 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# -> 4#1:41df217a:::smithi13140796-39:head#c = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1129 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 2 keys 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# 2 keys = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.28BF48E5.4.smithi13140796-39..' <- OBJ_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.4_0000000000000001.28BF48E5.4.smithi13140796-39..' <- SNA_1_0000000000000004_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000027.00000000000000000011' <- 0000000027.00000000000000000011 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000027.00000000000000000012' <- 0000000027.00000000000000000012 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 8084 (1 ios * 4000 + 4084 bytes) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes 0x558c574a0a00,0x558c5793a000,0x558c5793a280 shared_blobs 0x558c574d3f80,0x558c5757f0a0,0x558c57819dc0,0x558c57819ea0,0x558c57819f80,0x558c578de070,0x558c578de150,0x558c578de230,0x558c578de310,0x558c578de3f0,0x558c578de4d0,0x558c578de5b0,0x558c578de690,0x558c578de770,0x558c578de850,0x558c578de930,0x558c578dea10,0x558c578deaf0,0x558c578debd0,0x558c578decb0 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) update 4#1:41df217a:::smithi13140796-39:head#c 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) update inline shard 540 bytes from 20 extents 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:41df217a:::smithi13140796-39:head#c is 1031 (485 bytes onode + 2 bytes spanning blobs + 544 bytes inline extents) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c5793a108) update 4#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c5793a108) update inline shard 540 bytes from 20 extents 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:41df217a:::smithi13140796-39:4# is 958 (412 bytes onode + 2 bytes spanning blobs + 544 bytes inline extents) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c5793a388) update 4#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore.extentmap(0x558c5793a388) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:41df217a:::smithi13140796-39:head# is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2837 is 11 SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2836 is 11 SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2838 is 11 SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2839 is 11 SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283a is 11 SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283b is 11 SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283c is 11 SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283d is 11 SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283e is 11 SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283f is 11 SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2840 is 11 SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2841 is 11 SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2842 is 11 SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2843 is 11 SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2844 is 11 SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2845 is 11 SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2846 is 11 SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2847 is 11 SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2848 is 11 SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2849 is 11 SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=2))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:34:04.727+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:34:04.727+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:34:04.727+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[2~1]] op_applied version 27'12 2019-12-07T19:34:04.727+0000 7f8f07988700 10 osd.7 27 dequeue_op 0x558c5740cdc0 finish 2019-12-07T19:34:04.727+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5297 < min_flush_size 524288 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793a000 had 1 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793a280 had 1 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5320 < min_flush_size 524288 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5320 < min_flush_size 524288 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x642ad~14c8 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x642ad 0x642ad~14c8 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x342ad 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2ad 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x775 and padding block with 0x88b 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:head#13 key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0xfffffffffffffffe0000000000000013'o' 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:04.727+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:head#13 2019-12-07T19:34:04.727+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:head#13 = 0 2019-12-07T19:34:04.727+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# -> 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1130 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.onode(0x558c575ba500).flush flush done 2019-12-07T19:34:04.727+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0 2019-12-07T19:34:04.727+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# -> 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 -> 0x0~30000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x0~2000: 0x0~2000 Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57880d90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57880d90) get_ref 0x0~2000 Blob(0x558c57880d90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57880d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57880d90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x2000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578dfe30 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578dfe30) get_ref 0x0~2000 Blob(0x558c578dfe30 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578dfe30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x65775 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c578dfe30 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.727+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578f1ea0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1ea0) get_ref 0x0~2000 Blob(0x558c578f1ea0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c578f1ea0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000153862s (0.000000110s flush + 0.000153752s kv commit) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x6000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578f1f80 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1f80) get_ref 0x0~2000 Blob(0x558c578f1f80 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1f80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54836f00> 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.727+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c578f1f80 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x8000~2000: 0x0~2000 Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 finishing 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 0x558c574a0a00,0x558c5793a000,0x558c5793a280 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578f1e30 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.727+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:34:04.727+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1e30) get_ref 0x0~2000 Blob(0x558c578f1e30 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c578f1e30 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0xa000~2000: 0x0~2000 Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578f1dc0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1dc0) get_ref 0x0~2000 Blob(0x558c578f1dc0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c578f1dc0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 -- 0x558c576b5c00 con 0x558c54c64400 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0xc000~2000: 0x0~2000 Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578f1d50 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1d50) get_ref 0x0~2000 Blob(0x558c578f1d50 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578f1d50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c578f1d50 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0xe000~2000: 0x0~2000 Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881e30 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881e30) get_ref 0x0~2000 Blob(0x558c57881e30 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57881e30 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x10000~2000: 0x0~2000 Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881dc0 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881dc0) get_ref 0x0~2000 Blob(0x558c57881dc0 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57881dc0 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881d50 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881d50) get_ref 0x0~2000 Blob(0x558c57881d50 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881d50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57881d50 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x14000~2000: 0x0~2000 Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881ce0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881ce0) get_ref 0x0~2000 Blob(0x558c57881ce0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57881ce0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x16000~2000: 0x0~2000 Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881c70 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881c70) get_ref 0x0~2000 Blob(0x558c57881c70 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881c70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c57881c70 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x18000~2000: 0x0~2000 Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881c00 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881c00) get_ref 0x0~2000 Blob(0x558c57881c00 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881c00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57881c00 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881b90 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881b90) get_ref 0x0~2000 Blob(0x558c57881b90 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881b90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57881b90 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881b20 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881b20) get_ref 0x0~2000 Blob(0x558c57881b20 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881b20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57881b20 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881ab0 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881ab0) get_ref 0x0~2000 Blob(0x558c57881ab0 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c57881ab0 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x20000~2000: 0x0~2000 Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881a40 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881a40) get_ref 0x0~2000 Blob(0x558c57881a40 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881a40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c57881a40 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x22000~2000: 0x0~2000 Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578819d0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578819d0) get_ref 0x0~2000 Blob(0x558c578819d0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c578819d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c578819d0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x24000~2000: 0x0~2000 Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881960 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881960) get_ref 0x0~2000 Blob(0x558c57881960 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.blob(0x558c57881960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c57881960 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:04.727+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x26000~2000: 0x0~2000 Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c578818f0 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c578818f0) get_ref 0x0~2000 Blob(0x558c578818f0 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c578818f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c578818f0 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x28000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57880a80 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57880a80) get_ref 0x0~2000 Blob(0x558c57880a80 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57880a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c57880a80 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57880a10 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57880a10) get_ref 0x0~2000 Blob(0x558c57880a10 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57880a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c57880a10 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881ea0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57881ea0) get_ref 0x0~2000 Blob(0x558c57881ea0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57881ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c57881ea0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup new Blob(0x558c57881f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57881f10) get_ref 0x0~2000 Blob(0x558c57881f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.blob(0x558c57881f10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c57881f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3508) dirty_range mark inline shard dirty 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# -> 2#1:6ee4dfe7:::smithi13140796-47:head# = 0 2019-12-07T19:34:04.728+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:04.728+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.728+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000027.00000000000000000019' <- 0000000027.00000000000000000019 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5750 (1 ios * 4000 + 1750 bytes) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes 0x558c54ce3400,0x558c575ba780 shared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3508) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3508) update inline shard 651 bytes from 24 extents 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1198 (541 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba888) update 2#1:6ee4dfe7:::smithi13140796-47:head#13 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head#13 is 702 (541 bytes onode + 2 bytes spanning blobs + 159 bytes inline extents) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2))) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'19 (0'0,27'19] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'18 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] op_applied version 27'19 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c578d2000 finish 2019-12-07T19:34:04.728+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3277 < min_flush_size 524288 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3400 had 1 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba780 had 1 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3300 < min_flush_size 524288 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3300 < min_flush_size 524288 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x65775~ce4 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x65775 0x65775~ce4 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x35775 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x775 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x459 and padding block with 0xba7 2019-12-07T19:34:04.728+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 67 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=73, reqid=client.4540.0:225, at_version=27'20, trim_to=0'0, roll_forward_to=27'17)) v2 ==== 2406+0+0 (unknown 0 0 0) 0x558c574a2800 con 0x558c54c5bc00 2019-12-07T19:34:04.728+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c578d2160 prio 127 cost 0 latency 0.000039 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=73, reqid=client.4540.0:225, at_version=27'20, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.728+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=73, reqid=client.4540.0:225, at_version=27'20, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x66459 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.728+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000009 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000236239s (0.000000093s flush + 0.000236146s kv commit) 2019-12-07T19:34:04.728+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.728+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:04.728+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 0x558c54ce3400,0x558c575ba780 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.728+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:04.728+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.728+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=73, reqid=client.4540.0:225, at_version=27'20, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.728+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=73, reqid=client.4540.0:225, at_version=27'20, trim_to=0'0, roll_forward_to=27'17)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.728+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c578d2160 prio 127 cost 0 latency 0.000164 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=73, reqid=client.4540.0:225, at_version=27'20, trim_to=0'0, roll_forward_to=27'17)) v2 pg pg[1.6s2( v 27'19 (0'0,27'19] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'19 (0'0,27'19] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=73, reqid=client.4540.0:225, at_version=27'20, trim_to=0'0, roll_forward_to=27'17)) v2 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'19 (0'0,27'19] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'19], crt=27'17) [27'20 (27'19) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:225 2019-12-07T19:34:04.726001+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] add_log_entry 27'20 (27'19) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:225 2019-12-07T19:34:04.726001+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 20 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.728+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.728+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'20, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740a600 seq 39 2019-12-07T19:34:04.728+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:04.728+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.728+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000027.00000000000000000020' <- 0000000027.00000000000000000020 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5927 (1 ios * 4000 + 1927 bytes) 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes 0x558c54ce3400 shared_blobs 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3508) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1198 (541 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:04.728+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:04.728+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:04.728+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'17 lcod 27'18 active+remapped mbc={} ps=[2~1]] op_applied version 27'20 2019-12-07T19:34:04.728+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c578d2160 finish 2019-12-07T19:34:04.728+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=72, last_complete=27'19, committed=1, applied=1)) v2 -- 0x558c5768b000 con 0x558c54c5bc00 2019-12-07T19:34:04.728+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e27f8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf320 space 0x558c574e27f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19200~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19600~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19600~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19a00~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19a00~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19e00~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19e00~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a200~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e28d8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf3b0 space 0x558c574e28d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a200~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a600~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a600~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1aa00~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1aa00~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ae00~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.728+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ae00~200 = 512 2019-12-07T19:34:04.728+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b200~200 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.728+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2297 < min_flush_size 524288 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e28d8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf440 space 0x558c574e28d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3400 had 1 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.729+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b200~200 = 512 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2320 < min_flush_size 524288 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2320 < min_flush_size 524288 2019-12-07T19:34:04.729+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b600~200 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x66459~910 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.729+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b600~200 = 512 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x66459 0x66459~910 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.729+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ba00~200 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x36459 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x459 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.729+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ba00~200 = 512 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd69 and padding block with 0x297 2019-12-07T19:34:04.729+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1be00~200 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.729+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1be00~200 = 512 2019-12-07T19:34:04.729+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c200~200 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.729+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x66d69 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000236694s (0.000000092s flush + 0.000236602s kv commit) 2019-12-07T19:34:04.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a600> 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:04.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 0x558c54ce3400 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:04.729+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.729+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.729+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=73, last_complete=27'20, committed=1, applied=1)) v2 -- 0x558c54d76200 con 0x558c54c5bc00 2019-12-07T19:34:04.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.730+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 26 ==== MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 ==== 1039+0+0 (crc 0 0 0) 0x558c54d86800 con 0x558c54c64400 2019-12-07T19:34:04.730+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c578d22c0 prio 127 cost 0 latency 0.000020 epoch 27 MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 2019-12-07T19:34:04.730+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.730+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.730+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.730+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2) prio 127 cost 0 e27) pg 0x558c5731a000 2019-12-07T19:34:04.730+0000 7f8f07988700 10 osd.7 27 dequeue_op 0x558c578d22c0 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 pg pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] 2019-12-07T19:34:04.730+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 2019-12-07T19:34:04.730+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log log((0'0,27'12], crt=23'10) [] 2019-12-07T19:34:04.730+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] rollforward: entry=27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.730+0000 7f8f07988700 20 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] rollforward: entry=27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.730+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] append_log approx pg log length = 12 2019-12-07T19:34:04.730+0000 7f8f07988700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.730+0000 7f8f07988700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.730+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.730+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5740b500 seq 10 2019-12-07T19:34:04.730+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41df217a:::smithi13140796-39:head#c onode 0x558c574a0a00 txc 0x558c5740b500 2019-12-07T19:34:04.730+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.2s4_head 4#1:41df217a:::smithi13140796-39:head#c 0x0 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c575bd8f0) put_ref 0x0~2000 Blob(0x558c575bd8f0 blob([0x2e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c54d75180) put_ref 0x0~2000 Blob(0x558c54d75180 blob([0x2e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c57819d50) put_ref 0x0~2000 Blob(0x558c57819d50 blob([0x2e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c57819e30) put_ref 0x0~2000 Blob(0x558c57819e30 blob([0x2e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c57819f10) put_ref 0x0~2000 Blob(0x558c57819f10 blob([0x2ea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de000) put_ref 0x0~2000 Blob(0x558c578de000 blob([0x2ec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de0e0) put_ref 0x0~2000 Blob(0x558c578de0e0 blob([0x2ee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de1c0) put_ref 0x0~2000 Blob(0x558c578de1c0 blob([0x2f0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de2a0) put_ref 0x0~2000 Blob(0x558c578de2a0 blob([0x2f2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de380) put_ref 0x0~2000 Blob(0x558c578de380 blob([0x2f4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de460) put_ref 0x0~2000 Blob(0x558c578de460 blob([0x2f6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de540) put_ref 0x0~2000 Blob(0x558c578de540 blob([0x2f8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de620) put_ref 0x0~2000 Blob(0x558c578de620 blob([0x2fa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de700) put_ref 0x0~2000 Blob(0x558c578de700 blob([0x2fc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de7e0) put_ref 0x0~2000 Blob(0x558c578de7e0 blob([0x2fe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de8c0) put_ref 0x0~2000 Blob(0x558c578de8c0 blob([0x300000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578de9a0) put_ref 0x0~2000 Blob(0x558c578de9a0 blob([0x302000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578dea80) put_ref 0x0~2000 Blob(0x558c578dea80 blob([0x304000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578deb60) put_ref 0x0~2000 Blob(0x558c578deb60 blob([0x306000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.blob(0x558c578dec40) put_ref 0x0~1000 Blob(0x558c578dec40 blob([0x308000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore.extentmap(0x558c574a0b08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c575bd8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e2000~2000] 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=1))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d75180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=2)))) 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e4000~2000] 2019-12-07T19:34:04.730+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57819d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e6000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57819e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e8000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57819f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ea000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c578de000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ec000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c578de0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ee000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c578de1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f0000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c578de2a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f2000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c578de380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f4000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c578de460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f6000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c578de540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f8000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c578de620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fa000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c578de700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fc000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c578de7e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fe000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c578de8c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x300000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c578de9a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x302000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c578dea80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x304000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c578deb60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x306000~2000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~1000: 0x0~1000 Blob(0x558c578dec40 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=2)))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x308000~1000] 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c574d3f80,0x558c5757f0a0,0x558c57819dc0,0x558c57819ea0,0x558c57819f80,0x558c578de070,0x558c578de150,0x558c578de230,0x558c578de310,0x558c578de3f0,0x558c578de4d0,0x558c578de5b0,0x558c578de690,0x558c578de770,0x558c578de850,0x558c578de930,0x558c578dea10,0x558c578deaf0,0x558c578debd0,0x558c578decb0 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c5793a280 4#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.731+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41df217a:::smithi13140796-39:head#c = 0 2019-12-07T19:34:04.731+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.731+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:04.731+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 5574 (1 ios * 4000 + 1574 bytes) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes shared_blobs 0x558c574d3f80,0x558c5757f0a0,0x558c57819dc0,0x558c57819ea0,0x558c57819f80,0x558c578de070,0x558c578de150,0x558c578de230,0x558c578de310,0x558c578de3f0,0x558c578de4d0,0x558c578de5b0,0x558c578de690,0x558c578de770,0x558c578de850,0x558c578de930,0x558c578dea10,0x558c578deaf0,0x558c578debd0,0x558c578decb0 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2837 is 11 SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2836 is 11 SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2838 is 11 SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2839 is 11 SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283a is 11 SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283b is 11 SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283c is 11 SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283d is 11 SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283e is 11 SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283f is 11 SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2840 is 11 SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2841 is 11 SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2842 is 11 SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2843 is 11 SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2844 is 11 SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2845 is 11 SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2846 is 11 SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2847 is 11 SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2848 is 11 SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2849 is 11 SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=1))) 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.731+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:04.731+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:04.731+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:04.731+0000 7f8f07988700 10 osd.7 27 dequeue_op 0x558c578d22c0 finish 2019-12-07T19:34:04.731+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1830 < min_flush_size 524288 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1853 < min_flush_size 524288 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1853 < min_flush_size 524288 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x66d69~73d to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x66d69 0x66d69~73d to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x36d69 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd69 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4a6 and padding block with 0xb5a 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x674a6 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000170568s (0.000000089s flush + 0.000170479s kv commit) 2019-12-07T19:34:04.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:04.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:04.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:04.731+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.731+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.731+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 -- 0x558c54d77000 con 0x558c54c64400 2019-12-07T19:34:04.731+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.731+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 68 ==== MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=74, to_read={1:6ad15c4c:::smithi13140796-31:head=0,126976,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c574a1b80 con 0x558c54c5bc00 2019-12-07T19:34:04.731+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c578d2580 prio 127 cost 0 latency 0.000021 epoch 27 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=74, to_read={1:6ad15c4c:::smithi13140796-31:head=0,126976,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.731+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=74, to_read={1:6ad15c4c:::smithi13140796-31:head=0,126976,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=74, to_read={1:6ad15c4c:::smithi13140796-31:head=0,126976,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=74, to_read={1:6ad15c4c:::smithi13140796-31:head=0,126976,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c578d2580 prio 127 cost 0 latency 0.000070 MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=74, to_read={1:6ad15c4c:::smithi13140796-31:head=0,126976,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.6s2 27/26 ECSubRead(tid=74, to_read={1:6ad15c4c:::smithi13140796-31:head=0,126976,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.731+0000 7f8f0197c700 25 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.731+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1f000 size 0x1f000 (126976) 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 = 126976 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=126976 r=126976 len=126976 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:04.731+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key = 0 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:04.731+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 27/26 ECSubReadReply(tid=74, attrs_read=0)) v2 -- 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:04.731+0000 7f8f0197c700 10 osd.7 27 dequeue_op 0x558c578d2580 finish 2019-12-07T19:34:04.731+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e29b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf4d0 space 0x558c574e29b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c200~200 = 512 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c600~200 = 512 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ca00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ca00~200 = 512 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ce00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ce00~200 = 512 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1da00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1da00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1de00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1de00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ea00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ea00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ee00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ee00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fa00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fa00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fe00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fe00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2aa00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ae00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ba00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2be00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2be00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ca00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ce00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2da00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2da00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2de00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2de00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ea00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ee00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fa00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fe00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39a00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39a00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39e00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39e00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3aa00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ae00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ba00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3be00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3be00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ca00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ce00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3da00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3da00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3de00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3de00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ea00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ee00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f200~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f200~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f600~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f600~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fa00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fe00~200 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x1d000 (118784) 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 20 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=59392 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:c8ce3553:::smithi13140796-29:head 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:c8ce3553:::smithi13140796-29:head 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 118784 2019-12-07T19:34:04.733+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key 2019-12-07T19:34:04.733+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key = 0 2019-12-07T19:34:04.733+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.3s0 27/26 ECSubReadReply(tid=44, attrs_read=0)) v2 -- 0x558c54e45680 con 0x558c54c5b400 2019-12-07T19:34:04.733+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 69 ==== MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=75, reqid=client.4540.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=27'20)) v2 ==== 1039+0+0 (unknown 0 0 0) 0x558c54e3f800 con 0x558c54c5bc00 2019-12-07T19:34:04.734+0000 7f8f23c56700 15 osd.7 27 enqueue_op 0x558c578d26e0 prio 127 cost 0 latency 0.000026 epoch 27 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=75, reqid=client.4540.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=27'20)) v2 2019-12-07T19:34:04.734+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=75, reqid=client.4540.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=27'20)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.734+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c57335600 finish 2019-12-07T19:34:04.734+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=75, reqid=client.4540.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=27'20)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=75, reqid=client.4540.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=27'20)) v2) prio 127 cost 0 e27) pg 0x558c54d5c000 2019-12-07T19:34:04.735+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c578d26e0 prio 127 cost 0 latency 0.001048 MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=75, reqid=client.4540.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=27'20)) v2 pg pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.735+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 27/26 ECSubWrite(tid=75, reqid=client.4540.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=27'20)) v2 2019-12-07T19:34:04.735+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'20], crt=27'17) [] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'18 (27'17) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:223 2019-12-07T19:34:04.725983+0000 0 ObjectCleanRegions clean_offsets: [0~262144,262144~18446744073709289471], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'19 (27'18) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:224 2019-12-07T19:34:04.725997+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'20 (27'19) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:225 2019-12-07T19:34:04.726001+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.735+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 20 2019-12-07T19:34:04.735+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.735+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.735+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.735+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740bb00 seq 40 2019-12-07T19:34:04.735+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#13 onode 0x558c575ba780 txc 0x558c5740bb00 2019-12-07T19:34:04.735+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#13 0x0 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c54d75340) put_ref 0x0~2000 Blob(0x558c54d75340 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75420 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c54d3df80) put_ref 0x0~2000 Blob(0x558c54d3df80 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8620 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8700) put_ref 0x0~2000 Blob(0x558c54cf8700 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d3880) put_ref 0x0~2000 Blob(0x558c574d3880 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3810 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d37a0) put_ref 0x0~2000 Blob(0x558c574d37a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d36c0) put_ref 0x0~2000 Blob(0x558c574d36c0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3650 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d35e0) put_ref 0x0~2000 Blob(0x558c574d35e0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3570 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d3500) put_ref 0x0~2000 Blob(0x558c574d3500 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3490 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba888) dirty_range mark inline shard dirty 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d75340 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75420 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a6000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c54d75340) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d75438 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c54cdebd0 space 0x558c54d75438 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2a6000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c54d75420) put 0x558c54d75420 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d75438 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d3df80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8620 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a8000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c54d3df80) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8638 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c54cde2d0 space 0x558c54cf8638 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2a8000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c54cf8620) put 0x558c54cf8620 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54cf8638 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54cf8700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2aa000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c54cf8700) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c54cde360 space 0x558c574d3908 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2aa000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c574d38f0) put 0x558c574d38f0 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574d3880 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3810 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ac000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d3880) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3828 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c578eeea0 space 0x558c574d3828 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ac000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c574d3810) put 0x558c574d3810 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3828 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c574d37a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ae000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d37a0) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c578eee10 space 0x558c574d3748 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ae000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c574d3730) put 0x558c574d3730 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c574d36c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3650 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b0000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d36c0) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3668 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c578eed80 space 0x558c574d3668 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b0000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c574d3650) put 0x558c574d3650 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3668 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c574d35e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3570 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b2000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d35e0) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3588 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c54cdf950 space 0x558c574d3588 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b2000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c574d3570) put 0x558c574d3570 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d3588 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c574d3500 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3490 sbid 0x0)) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b4000~2000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.blob(0x558c574d3500) discard_unallocated 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d34a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 _rm buffer(0x558c54cdf8c0 space 0x558c574d34a8 0x0~2000 clean) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b4000~2000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.sharedblob(0x558c574d3490) put 0x558c574d3490 removing self from set 0x558c54d38c10 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c574d34a8 in 0x558c53a58fc0) _clear 2019-12-07T19:34:04.735+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#13 = 0 2019-12-07T19:34:04.735+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.735+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:04.735+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5574 (1 ios * 4000 + 1574 bytes) 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[2a6000~10000] 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x2a6000~10000 2019-12-07T19:34:04.735+0000 7f8f05984700 10 freelist release 0x2a6000~10000 2019-12-07T19:34:04.735+0000 7f8f05984700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:04.735+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:04.735+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:04.735+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:04.735+0000 7f8f05984700 10 osd.7 27 dequeue_op 0x558c578d26e0 finish 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1379 < min_flush_size 524288 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba780 had 1 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1402 < min_flush_size 524288 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1402 < min_flush_size 524288 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x674a6~57a to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x674a6 0x674a6~57a to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x374a6 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4a6 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa20 and padding block with 0x5e0 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x67a20 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.735+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000102428s (0.000000122s flush + 0.000102306s kv commit) 2019-12-07T19:34:04.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740bb00> 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:04.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:04.735+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.735+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [2a6000~10000] 2019-12-07T19:34:04.735+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x2a6000~10000 2019-12-07T19:34:04.735+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.735+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.735+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 27/26 ECSubWriteReply(tid=75, last_complete=27'20, committed=1, applied=1)) v2 -- 0x558c576b5c00 con 0x558c54c5bc00 2019-12-07T19:34:04.735+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.737+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 23 ==== MOSDPGPush(1.cs0 27/26 [PushOp(1:33f5bf01:::smithi13140796-2:head, version: 21'11, data_included: [0~225280], data_size: 225280, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:33f5bf01:::smithi13140796-2:head@21'11, size: 901120, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:901120, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 226337+0+0 (unknown 0 0 0) 0x558c54c1ee00 con 0x558c54c19c00 2019-12-07T19:34:04.737+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335760 prio 127 cost 226280 latency 0.000030 epoch 27 MOSDPGPush(1.cs0 27/26 [PushOp(1:33f5bf01:::smithi13140796-2:head, version: 21'11, data_included: [0~225280], data_size: 225280, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:33f5bf01:::smithi13140796-2:head@21'11, size: 901120, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:901120, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.737+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:33f5bf01:::smithi13140796-2:head, version: 21'11, data_included: [0~225280], data_size: 225280, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:33f5bf01:::smithi13140796-2:head@21'11, size: 901120, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:901120, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 226280 e27) 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:33f5bf01:::smithi13140796-2:head, version: 21'11, data_included: [0~225280], data_size: 225280, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:33f5bf01:::smithi13140796-2:head@21'11, size: 901120, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:901120, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 226280 e27) queued 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:33f5bf01:::smithi13140796-2:head, version: 21'11, data_included: [0~225280], data_size: 225280, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:33f5bf01:::smithi13140796-2:head@21'11, size: 901120, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:901120, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 226280 e27) pg 0x558c57676000 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c57335760 prio 127 cost 226280 latency 0.000093 MOSDPGPush(1.cs0 27/26 [PushOp(1:33f5bf01:::smithi13140796-2:head, version: 21'11, data_included: [0~225280], data_size: 225280, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:33f5bf01:::smithi13140796-2:head@21'11, size: 901120, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:901120, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.cs0( v 21'12 lc 21'10 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'10 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=2 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.cs0 27/26 [PushOp(1:33f5bf01:::smithi13140796-2:head, version: 21'11, data_included: [0~225280], data_size: 225280, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:33f5bf01:::smithi13140796-2:head@21'11, size: 901120, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:901120, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'10 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=2 mbc={} ps=[2~1]] on_local_recover: 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'10 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=2 mbc={} ps=[2~1]] got missing 1:33f5bf01:::smithi13140796-2:head v 21'11 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'10 active+remapped m=1 mbc={} ps=[2~1]] last_complete now 21'11 log.complete_to 21'12 2019-12-07T19:34:04.737+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d21b00 seq 7 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:head# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:head# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.737+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:04.737+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1131 2019-12-07T19:34:04.737+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# = 0 2019-12-07T19:34:04.737+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~37000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d21b00 28 blobs 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate 0x37000/1000,37000,0 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2a6000~10000/1000,37000,0 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x350000~27000/1000,37000,0 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2a6000~10000,0x350000~27000] 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573963f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573963f0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57396380) get_ref 0x0~2000 Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573963f0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57396380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57396408 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57396150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396150 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57396460) get_ref 0x0~2000 Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396150 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57396460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57396168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578804d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57880540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880540 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578804d0) get_ref 0x0~2000 Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880540 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578804d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57880558 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578805b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57880620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880620 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578805b0) get_ref 0x0~2000 Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880620 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578805b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57880638 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57880700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880700 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880690) get_ref 0x0~2000 Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880700 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57880718 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578807e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578807e0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880770) get_ref 0x0~2000 Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578807e0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578807f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578808c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578808c0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880850) get_ref 0x0~2000 Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578808c0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578808d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578809a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578809a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880930) get_ref 0x0~2000 Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578809a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578809b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3340) get_ref 0x0~2000 Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3340) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57880e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880e70 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880d20) get_ref 0x0~2000 Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880e70 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880d20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57880e88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880c40) get_ref 0x0~2000 Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57880c40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75340 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c54d75420) get_ref 0x0~2000 Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75340 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c54d75420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54d75358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de620 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c575bd8f0) get_ref 0x0~2000 Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de620 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c575bd8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578de638 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de460 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de540) get_ref 0x0~2000 Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de460 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578de478 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de2a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de380) get_ref 0x0~2000 Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de2a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578de2b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de0e0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de1c0) get_ref 0x0~2000 Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de0e0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578de0f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f10 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de000) get_ref 0x0~2000 Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f10 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578de000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57819f28 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819d50 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57819e30) get_ref 0x0~2000 Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819d50 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c57819e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57819d68 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8700 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf8620) get_ref 0x0~2000 Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8700 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf8620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54cf8718 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3df80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3df80 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1f10) get_ref 0x0~2000 Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3df80 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1f10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54d3df98 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794c4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c460 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c4d0) get_ref 0x0~2000 Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c460 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794c478 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794c3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c2a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c3f0) get_ref 0x0~2000 Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c2a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794c2b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794c310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c230 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c310) get_ref 0x0~2000 Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c230 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794c248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794c1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c150 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c1c0) get_ref 0x0~2000 Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c150 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794c168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794c0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794ce00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ce00 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c0e0) get_ref 0x0~2000 Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ce00 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794ce18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794cd90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d32d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d32d0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cd90) get_ref 0x0~2000 Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d32d0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cd90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d32e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3880 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3810) get_ref 0x0~2000 Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3880 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3810) get_ref init 0x2000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d3898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3260) get_ref 0x0~1000 Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3260) get_ref init 0x1000, 1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~1000: 0x0~1000 Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x37000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1508) dirty_range mark inline shard dirty 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 = 0 2019-12-07T19:34:04.738+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 4 keys 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 4 keys = 0 2019-12-07T19:34:04.738+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000456'.missing/0000000000000001.CCFADF08.head.smithi13140796-2..' <- missing/0000000000000001.CCFADF08.head.smithi13140796-2.. 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.738+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 343513 (29 ios * 4000 + 227513 bytes) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes 0x558c574a1400 shared_blobs 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1508) update inline shard 532 bytes from 28 extents 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1021 (483 bytes onode + 2 bytes spanning blobs + 536 bytes inline extents) 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[2a6000~10000,350000~27000] released 0x[] 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 freelist allocate 0x2a6000~10000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 freelist allocate 0x350000~27000 2019-12-07T19:34:04.738+0000 7f8f0297e700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:04.738+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d21b00 2019-12-07T19:34:04.739+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c57335760 finish 2019-12-07T19:34:04.739+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.739+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'10 active+remapped m=1 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.739+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.739+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 aio_wait 2019-12-07T19:34:04.739+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:04.739+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1504 < min_flush_size 524288 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2382 < min_flush_size 524288 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1400 had 1 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2405 < min_flush_size 524288 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2405 < min_flush_size 524288 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x67a20~965 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x67a20 0x67a20~965 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x37a20 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa20 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x385 and padding block with 0xc7b 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x68385 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.739+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000234373s (0.000076579s flush + 0.000157794s kv commit) 2019-12-07T19:34:04.739+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.739+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:04.739+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 0x558c574a1400 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8718 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb61b0 space 0x558c54cf8718 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3df98 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6240 space 0x558c54d3df98 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578ee510 space 0x558c54d75198 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75358 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578eeea0 space 0x558c54d75358 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396168 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efe60 space 0x558c57396168 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396408 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efef0 space 0x558c57396408 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d32e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb65a0 space 0x558c574d32e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb66c0 space 0x558c574d3748 0x0~1000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3898 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6630 space 0x558c574d3898 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a59ce0) _finish_write discard buffer(0x558c548df7a0 space 0x558c574d3908 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819d68 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6120 space 0x558c57819d68 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819f28 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6090 space 0x558c57819f28 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880558 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efa70 space 0x558c57880558 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880638 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cdebd0 space 0x558c57880638 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880718 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cde2d0 space 0x558c57880718 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578807f8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cde360 space 0x558c578807f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578808d8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cdf950 space 0x558c578808d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578809b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cdf8c0 space 0x558c578809b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880e88 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578eec60 space 0x558c57880e88 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de0f8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6000 space 0x558c578de0f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de2b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578eef30 space 0x558c578de2b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de478 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578eed80 space 0x558c578de478 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de638 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578eee10 space 0x558c578de638 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c168 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6480 space 0x558c5794c168 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c248 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb63f0 space 0x558c5794c248 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c2b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6360 space 0x558c5794c2b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c478 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb62d0 space 0x558c5794c478 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794ce18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6510 space 0x558c5794ce18 0x0~2000 writing nocache) 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.739+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:04.739+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.739+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.739+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.739+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'10 active+remapped m=1 mbc={} ps=[2~1]] recovery_committed_to version 21'11 now ondisk 2019-12-07T19:34:04.739+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGPushReply(1.cs1 27/26 [PushReplyOp(1:33f5bf01:::smithi13140796-2:head)]) v3 -- 0x558c548cc000 con 0x558c54c19c00 2019-12-07T19:34:04.739+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.743+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 24 ==== MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=6, reqid=client.4540.0:232, at_version=27'14, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 183640+0+0 (unknown 0 0 0) 0x558c5748e800 con 0x558c54c19c00 2019-12-07T19:34:04.743+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c573358c0 prio 127 cost 0 latency 0.000038 epoch 27 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=6, reqid=client.4540.0:232, at_version=27'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.743+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=6, reqid=client.4540.0:232, at_version=27'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=6, reqid=client.4540.0:232, at_version=27'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=6, reqid=client.4540.0:232, at_version=27'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558c57676000 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c573358c0 prio 127 cost 0 latency 0.000156 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=6, reqid=client.4540.0:232, at_version=27'14, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.cs0( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=6, reqid=client.4540.0:232, at_version=27'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 snap_mapper.add_oid 1:3a5a3f5b:::smithi13140796-23:4 4 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:4 got.empty() 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 snap_mapper.set_snaps 1:3a5a3f5b:::smithi13140796-23:4 4 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log log((0'0,21'12], crt=21'12) [27'13 (21'8) clone 1:3a5a3f5b:::smithi13140796-23:4 by unknown.0.0:0 2019-12-07T19:33:58.673986+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'14 (21'8) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:232 2019-12-07T19:34:04.737301+0000 0 ObjectCleanRegions clean_offsets: [0~376832,1097728~18446744073708453887], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'13 lc 21'11 (0'0,27'13] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'12 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] add_log_entry 27'13 (21'8) clone 1:3a5a3f5b:::smithi13140796-23:4 by unknown.0.0:0 2019-12-07T19:33:58.673986+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'14 lc 21'11 (0'0,27'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'12 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] add_log_entry 27'14 (21'8) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:232 2019-12-07T19:34:04.737301+0000 0 ObjectCleanRegions clean_offsets: [0~376832,1097728~18446744073708453887], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'14 lc 21'11 (0'0,27'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'12 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log approx pg log length = 14 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'14 lc 21'11 (0'0,27'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'12 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.743+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d20900 seq 8 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:4# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:4# 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1132 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.onode(0x558c574a1180).flush flush done 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# 0x0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:4# 0x0~17000 -> 0x0~17000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x0~2000: 0x0~2000 Blob(0x558c573961c0 blob([0x339000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10314 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c573961c0 blob([0x339000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c574f1ea0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c574f1ea0) get_ref 0x0~2000 Blob(0x558c574f1ea0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c574f1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574f1ea0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x2000~2000: 0x0~2000 Blob(0x558c573962a0 blob([0x33b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10315 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c573962a0 blob([0x33b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57396700 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c57396700) get_ref 0x0~2000 Blob(0x558c57396700 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c57396700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57396700 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x4000~2000: 0x0~2000 Blob(0x558c574f1f80 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10316 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c574f1f80 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c573968c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c573968c0) get_ref 0x0~2000 Blob(0x558c573968c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c573968c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c573968c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x6000~2000: 0x0~2000 Blob(0x558c578f1960 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10317 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578f1960 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57396310 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c57396310) get_ref 0x0~2000 Blob(0x558c57396310 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c57396310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57396310 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x8000~2000: 0x0~2000 Blob(0x558c578f1a40 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10318 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578f1a40 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794cfc0 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cfc0) get_ref 0x0~2000 Blob(0x558c5794cfc0 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5794cfc0 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0xa000~2000: 0x0~2000 Blob(0x558c578f1b20 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10319 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578f1b20 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794d0a0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d0a0) get_ref 0x0~2000 Blob(0x558c5794d0a0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5794d0a0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0xc000~2000: 0x0~2000 Blob(0x558c578f1c00 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10320 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578f1c00 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794d110 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d110) get_ref 0x0~2000 Blob(0x558c5794d110 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c5794d110 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0xe000~2000: 0x0~2000 Blob(0x558c578f1ce0 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10321 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578f1ce0 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794d180 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d180) get_ref 0x0~2000 Blob(0x558c5794d180 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c5794d180 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57314000 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10322 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57314000 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794d1f0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d1f0) get_ref 0x0~2000 Blob(0x558c5794d1f0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5794d1f0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574fa310 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10323 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c574fa310 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794d260 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d260) get_ref 0x0~2000 Blob(0x558c5794d260 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c5794d260 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x14000~2000: 0x0~2000 Blob(0x558c54cf96c0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10324 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c54cf96c0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794d2d0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d2d0) get_ref 0x0~2000 Blob(0x558c5794d2d0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c5794d2d0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup src 0x16000~1000: 0x0~1000 Blob(0x558c574e05b0 blob([0x34f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10325 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c574e05b0 blob([0x34f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c5794d340 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d340) get_ref 0x0~1000 Blob(0x558c5794d340 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d340) get_ref init 0x1000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x16000~1000: 0x0~1000 Blob(0x558c5794d340 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark inline shard dirty 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c5793ab08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:4# = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# snapset 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# snapset = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# 1 keys 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# 1 keys = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x17000~2c000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3a5a3f5b:::smithi13140796-23:head# 0x17000~2c000 - have 0x17000 (94208) bytes fadvise_flags 0x0 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x17000~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d20900 22 blobs 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2c000/1000,2c000,0 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x377000~2c000/1000,2c000,0 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x377000~2c000] 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d420 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d3b0) get_ref 0x0~2000 Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d420 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x17000~2000: 0x0~2000 Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d438 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d500 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d490) get_ref 0x0~2000 Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d500 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x19000~2000: 0x0~2000 Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d518 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d5e0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d570) get_ref 0x0~2000 Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d5e0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d5f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d6c0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d650) get_ref 0x0~2000 Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d6c0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d6d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d7a0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d730) get_ref 0x0~2000 Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d7a0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d7b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d880 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d810) get_ref 0x0~2000 Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d880 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d810) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d960 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d8f0) get_ref 0x0~2000 Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d960 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d978 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794da40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794da40 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d9d0) get_ref 0x0~2000 Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794da40 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794da58 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794dab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dab0) get_ref 0x0~2000 Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dab0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794db38 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794db90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc00 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794db90) get_ref 0x0~2000 Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc00 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794db90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794dc18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794dc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794dce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dce0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dc70) get_ref 0x0~2000 Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dce0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dc70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794dcf8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794dd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794ddc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ddc0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dd50) get_ref 0x0~2000 Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ddc0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dd50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794ddd8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794de30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794dea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dea0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794de30) get_ref 0x0~2000 Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dea0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794de30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794deb8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794df10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794df80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794df80 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794df10) get_ref 0x0~2000 Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794df80 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5794df10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794df98 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c070 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c000) get_ref 0x0~2000 Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c070 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c088 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c150 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c0e0) get_ref 0x0~2000 Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c150 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c230 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c1c0) get_ref 0x0~2000 Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c230 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c2a0) get_ref 0x0~2000 Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c328 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c3f0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c380) get_ref 0x0~2000 Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c3f0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c408 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c4d0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c460) get_ref 0x0~2000 Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c4d0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c4e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c5b0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c540) get_ref 0x0~2000 Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c5b0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c5c8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c690 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c620) get_ref 0x0~2000 Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c690 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c6a8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x43000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark inline shard dirty 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x17000~2c000 = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..' <- SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000027.00000000000000000013' <- 0000000027.00000000000000000013 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000027.00000000000000000014' <- 0000000027.00000000000000000014 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20900 cost 276667 (23 ios * 4000 + 184667 bytes) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20900 onodes 0x558c574a1180,0x558c5793aa00 shared_blobs 0x558c54d75e30,0x558c57396230,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) update inline shard 743 bytes from 34 extents 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1306 (557 bytes onode + 2 bytes spanning blobs + 747 bytes inline extents) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c5793ab08) update 0#1:3a5a3f5b:::smithi13140796-23:4# 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore.extentmap(0x558c5793ab08) update inline shard 324 bytes from 12 extents 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:4# is 742 (412 bytes onode + 2 bytes spanning blobs + 328 bytes inline extents) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2851 is 11 SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284a is 11 SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2853 is 11 SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2852 is 11 SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284b is 11 SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2854 is 11 SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2855 is 11 SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284c is 11 SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284d is 11 SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284e is 11 SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284f is 11 SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2850 is 11 SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2))) 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20900 allocated 0x[377000~2c000] released 0x[] 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 freelist allocate 0x377000~2c000 2019-12-07T19:34:04.743+0000 7f8f0297e700 20 freelist _xor first_key 0x300000 last_key 0x380000 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 prepare 2019-12-07T19:34:04.743+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d20900 2019-12-07T19:34:04.744+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.744+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'14 lc 21'11 (0'0,27'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'12 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] op_applied version 27'14 2019-12-07T19:34:04.744+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c573358c0 finish 2019-12-07T19:34:04.744+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.744+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 aio_wait 2019-12-07T19:34:04.744+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20900 2019-12-07T19:34:04.744+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 io_done 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5046 < min_flush_size 524288 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1180 had 1 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793aa00 had 1 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5069 < min_flush_size 524288 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5069 < min_flush_size 524288 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x68385~13cd to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x68385 0x68385~13cd to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x38385 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x385 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x752 and padding block with 0x8ae 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x69752 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.744+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000144914s (0.000035524s flush + 0.000109390s kv commit) 2019-12-07T19:34:04.744+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20900> 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.744+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 kv_submitted 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20900 2019-12-07T19:34:04.744+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 finishing 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20900 onodes 0x558c574a1180,0x558c5793aa00 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c088 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6f30 space 0x558c5792c088 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c168 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6fc0 space 0x558c5792c168 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c248 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7050 space 0x558c5792c248 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c328 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb70e0 space 0x558c5792c328 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c408 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7170 space 0x558c5792c408 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c4e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7200 space 0x558c5792c4e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c5c8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7290 space 0x558c5792c5c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c6a8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7320 space 0x558c5792c6a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWriteReply(1.cs1 27/26 ECSubWriteReply(tid=6, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558c576b5800 con 0x558c54c19c00 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d438 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efdd0 space 0x558c5794d438 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d518 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6900 space 0x558c5794d518 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d5f8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6990 space 0x558c5794d5f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d6d8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6a20 space 0x558c5794d6d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d7b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6ab0 space 0x558c5794d7b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d898 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6b40 space 0x558c5794d898 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d978 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb62d0 space 0x558c5794d978 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794da58 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6510 space 0x558c5794da58 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794db38 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6bd0 space 0x558c5794db38 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794dc18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6c60 space 0x558c5794dc18 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794dcf8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6cf0 space 0x558c5794dcf8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794ddd8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6d80 space 0x558c5794ddd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794deb8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6e10 space 0x558c5794deb8 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794df98 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb6ea0 space 0x558c5794df98 0x0~2000 writing nocache) 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20900 done 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.744+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20900 [] 2019-12-07T19:34:04.744+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.744+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.745+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 25 ==== MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=7, reqid=client.4540.0:233, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 96798+0+0 (unknown 0 0 0) 0x558c5748d000 con 0x558c54c19c00 2019-12-07T19:34:04.745+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335a20 prio 127 cost 0 latency 0.000034 epoch 27 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=7, reqid=client.4540.0:233, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.745+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=7, reqid=client.4540.0:233, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=7, reqid=client.4540.0:233, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=7, reqid=client.4540.0:233, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558c57676000 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c57335a20 prio 127 cost 0 latency 0.000089 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=7, reqid=client.4540.0:233, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.cs0( v 27'14 lc 21'11 (0'0,27'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'14 lc 21'11 (0'0,27'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=7, reqid=client.4540.0:233, at_version=27'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'14 lc 21'11 (0'0,27'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log log((0'0,27'14], crt=21'12) [27'15 (27'14) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:233 2019-12-07T19:34:04.739951+0000 0 ObjectCleanRegions clean_offsets: [0~1097728,1474560~18446744073708077055], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'15 lc 21'11 (0'0,27'15] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'14 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] add_log_entry 27'15 (27'14) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:233 2019-12-07T19:34:04.739951+0000 0 ObjectCleanRegions clean_offsets: [0~1097728,1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'15 lc 21'11 (0'0,27'15] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'14 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log approx pg log length = 15 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'15 lc 21'11 (0'0,27'15] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'14 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.745+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'15, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea1200 seq 9 2019-12-07T19:34:04.745+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:04.745+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.745+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x43000~17000 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3a5a3f5b:::smithi13140796-23:head# 0x43000~17000 - have 0x43000 (274432) bytes fadvise_flags 0x0 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.745+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x43000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1200 12 blobs 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.745+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate 0x17000/1000,17000,0 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3a3000~17000/1000,17000,0 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3a3000~17000] 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c546777a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c546777a0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d030) get_ref 0x0~2000 Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c546777a0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c546777b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573967e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573967e0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c57396770) get_ref 0x0~2000 Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573967e0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c57396770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c573967f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d180 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d110) get_ref 0x0~2000 Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d180 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d198 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d260 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d1f0) get_ref 0x0~2000 Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d260 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d278 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d340 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d2d0) get_ref 0x0~2000 Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d340 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d420 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d3b0) get_ref 0x0~2000 Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d420 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d438 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d500 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d490) get_ref 0x0~2000 Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d500 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d518 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d5e0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d570) get_ref 0x0~2000 Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d5e0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x51000~2000: 0x0~2000 Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d5f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d6c0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d650) get_ref 0x0~2000 Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d6c0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x53000~2000: 0x0~2000 Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d6d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d7a0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d730) get_ref 0x0~2000 Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d7a0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x55000~2000: 0x0~2000 Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d7b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d880 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d810) get_ref 0x0~2000 Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d880 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d810) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x57000~2000: 0x0~2000 Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792d960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d960 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d8f0) get_ref 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d960 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x59000~1000: 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d978 in 0x558c53a59ce0) _discard 0x1000~1000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5a000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark inline shard dirty 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x43000~17000 = 0 2019-12-07T19:34:04.746+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.746+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000027.00000000000000000015' <- 0000000027.00000000000000000015 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 149238 (13 ios * 4000 + 97238 bytes) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes 0x558c574a1180 shared_blobs 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) update inline shard 984 bytes from 46 extents 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1547 (557 bytes onode + 2 bytes spanning blobs + 988 bytes inline extents) 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[3a3000~17000] released 0x[] 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 freelist allocate 0x3a3000~17000 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea1200 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'15 lc 21'11 (0'0,27'15] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'14 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] op_applied version 27'15 2019-12-07T19:34:04.746+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c57335a20 finish 2019-12-07T19:34:04.746+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.746+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 aio_wait 2019-12-07T19:34:04.746+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:04.746+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3667 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1180 had 1 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3690 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3690 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x69752~e6a to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x69752 0x69752~e6a to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x39752 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x752 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5bc and padding block with 0xa44 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6a5bc 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000198597s (0.000082681s flush + 0.000115916s kv commit) 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200> 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 0x558c574a1180 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c546777b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578eebd0 space 0x558c546777b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573967f8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c578efd40 space 0x558c573967f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d198 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7440 space 0x558c5792d198 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d278 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb74d0 space 0x558c5792d278 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d358 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7560 space 0x558c5792d358 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d438 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb75f0 space 0x558c5792d438 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d518 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7680 space 0x558c5792d518 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d5f8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7710 space 0x558c5792d5f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d6d8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb77a0 space 0x558c5792d6d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d7b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7830 space 0x558c5792d7b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d898 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb78c0 space 0x558c5792d898 0x0~2000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d978 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54cb7950 space 0x558c5792d978 0x1000~1000 writing nocache) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:04.747+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 26 ==== MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=8, reqid=client.4540.0:234, at_version=27'16, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2556+0+0 (unknown 0 0 0) 0x558c575bf800 con 0x558c54c19c00 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.747+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000028 epoch 27 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=8, reqid=client.4540.0:234, at_version=27'16, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.747+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=8, reqid=client.4540.0:234, at_version=27'16, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.747+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.747+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=8, reqid=client.4540.0:234, at_version=27'16, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.747+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.747+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=8, reqid=client.4540.0:234, at_version=27'16, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558c57676000 2019-12-07T19:34:04.747+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 27 dequeue_op 0x558c54e78f20 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=8, reqid=client.4540.0:234, at_version=27'16, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.cs0( v 27'15 lc 21'11 (0'0,27'15] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'15 lc 21'11 (0'0,27'15] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=8, reqid=client.4540.0:234, at_version=27'16, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'15 lc 21'11 (0'0,27'15] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log log((0'0,27'15], crt=21'12) [27'16 (27'15) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:234 2019-12-07T19:34:04.739979+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] add_log_entry 27'16 (27'15) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:234 2019-12-07T19:34:04.739979+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log approx pg log length = 16 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.747+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.747+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.747+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c5740a600 seq 10 2019-12-07T19:34:04.747+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 3 keys 2019-12-07T19:34:04.747+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 3 keys = 0 2019-12-07T19:34:04.747+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.747+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.747+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000027.00000000000000000016' <- 0000000027.00000000000000000016 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.747+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.747+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 6088 (1 ios * 4000 + 2088 bytes) 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes 0x558c574a1180 shared_blobs 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1288) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1547 (557 bytes onode + 2 bytes spanning blobs + 988 bytes inline extents) 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.747+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:04.747+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:04.747+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'15 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] op_applied version 27'16 2019-12-07T19:34:04.747+0000 7f8f06986700 10 osd.7 27 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:34:04.747+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWriteReply(1.cs1 27/26 ECSubWriteReply(tid=7, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558c576b5e00 con 0x558c54c19c00 2019-12-07T19:34:04.747+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2719 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1180 had 1 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2742 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2742 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6a5bc~ab6 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6a5bc 0x6a5bc~ab6 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3a5bc 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5bc 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x72 and padding block with 0xf8e 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6b072 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.747+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122983s (0.000000115s flush + 0.000122868s kv commit) 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.747+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a600> 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 0x558c574a1180 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:04.747+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.747+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.747+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.747+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWriteReply(1.cs1 27/26 ECSubWriteReply(tid=8, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558c576b5800 con 0x558c54c19c00 2019-12-07T19:34:04.747+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.751+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 51 ==== MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=48, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558c57572000 con 0x558c54c5b400 2019-12-07T19:34:04.751+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e79340 prio 3 cost 0 latency 0.000034 epoch 27 MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=48, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.751+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=48, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=48, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.751+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=48, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c5513e000 2019-12-07T19:34:04.751+0000 7f8f07187700 10 osd.7 27 dequeue_op 0x558c54e79340 prio 3 cost 0 latency 0.000086 MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=48, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=48, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.751+0000 7f8f07187700 25 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x200~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0cc8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae000 space 0x558c574e0cc8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x200~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x600~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x600~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa00~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa00~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe00~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe00~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1200~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0cc8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae090 space 0x558c574e0cc8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1200~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1600~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1600~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a00~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a00~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e00~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e00~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2200~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e427f8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae120 space 0x558c54e427f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2200~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2600~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2600~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a00~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a00~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e00~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e00~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3200~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e427f8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.751+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae1b0 space 0x558c54e427f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.751+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3200~200 = 512 2019-12-07T19:34:04.751+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3600~200 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x54000 (344064) 2019-12-07T19:34:04.751+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e428d8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae240 space 0x558c54e428d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e428d8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae2d0 space 0x558c54e428d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e429b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae360 space 0x558c54e429b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e429b8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae3f0 space 0x558c54e429b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1518 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae480 space 0x558c574e1518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1518 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae510 space 0x558c574e1518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1438 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae5a0 space 0x558c574e1438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xaa00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xaa00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xae00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xae00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1438 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae630 space 0x558c574e1438 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xba00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xba00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xbe00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xbe00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1358 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae6c0 space 0x558c574e1358 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xca00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xca00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xce00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xce00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 6 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.210690433s delta_ub 0.021951027s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b5c00 con 0x558c54c19000 2019-12-07T19:34:04.752+0000 7f8f23c56700 20 osd.7 27 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021328793s,0.021951027s]) 2019-12-07T19:34:04.752+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.189361640s delta_ub 18446744073.688222823s) v5 -- 0x558c54d76200 con 0x558c54c19000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1358 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae750 space 0x558c574e1358 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xda00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xda00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xde00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xde00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1278 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae7e0 space 0x558c574e1278 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xea00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xea00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xee00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xee00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1278 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae870 space 0x558c574e1278 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfa00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfa00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfe00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfe00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1198 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae900 space 0x558c574e1198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1198 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576ae990 space 0x558c574e1198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11200~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11600~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11600~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11a00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11a00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11e00~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.752+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11e00~200 = 512 2019-12-07T19:34:04.752+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12200~200 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x54000 (344064) 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.752+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e10b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aea20 space 0x558c574e10b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e10b8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aeab0 space 0x558c574e10b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0ef8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aeb40 space 0x558c574e0ef8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0ef8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aebd0 space 0x558c574e0ef8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0f68 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aec60 space 0x558c574e0f68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0f68 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aecf0 space 0x558c574e0f68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0a98 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aed80 space 0x558c574e0a98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 6 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.210690433s delta_ub 0.021951027s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77000 con 0x558c54c18800 2019-12-07T19:34:04.753+0000 7f8f23c56700 20 osd.7 27 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.020473142s,0.021951027s]) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0a98 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aee10 space 0x558c574e0a98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.190217291s delta_ub 18446744073.689078474s) v5 -- 0x558c54d76200 con 0x558c54c18800 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19a00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19a00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19e00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19e00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e0a28 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aeea0 space 0x558c574e0a28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1aa00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1aa00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ae00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ae00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need {<0x0, 0x1000> : [0x1b200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1b200:200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54d3c9b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aef30 space 0x558c54d3c9b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b200~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b600~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b600~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ba00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ba00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1be00~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.753+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1be00~200 = 512 2019-12-07T19:34:04.753+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c200~200 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x54000 (344064) 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1c200:1200~200]} 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.753+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1c200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54d3c9b8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576aefc0 space 0x558c54d3c9b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ca00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ca00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ce00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ce00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need {<0x0, 0x1000> : [0x1d200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1d200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574f19e8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af050 space 0x558c574f19e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1da00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1da00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1de00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1de00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1e200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1e200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574f19e8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af0e0 space 0x558c574f19e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ea00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ea00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ee00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ee00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need {<0x0, 0x1000> : [0x1f200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1f200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9828 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af170 space 0x558c54cf9828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fa00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fa00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fe00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fe00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need {<0x1000, 0x1000> : [0x20200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x20200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf9828 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af200 space 0x558c54cf9828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20a00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20a00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20e00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20e00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need {<0x0, 0x1000> : [0x21200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x21200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1d68 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af290 space 0x558c574e1d68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21a00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21a00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21e00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21e00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need {<0x1000, 0x1000> : [0x22200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x22200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1d68 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af320 space 0x558c574e1d68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22a00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22a00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22e00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22e00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need {<0x0, 0x1000> : [0x23200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x23200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1e48 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af3b0 space 0x558c574e1e48 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23a00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23a00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23e00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23e00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need {<0x1000, 0x1000> : [0x24200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x24200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1e48 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af440 space 0x558c574e1e48 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24a00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24a00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24e00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24e00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need {<0x0, 0x1000> : [0x25200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x25200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x25200:200~200]} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1f28 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.754+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 27 ==== MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=10, reqid=client.4540.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=27'16)) v2 ==== 1028+0+0 (unknown 0 0 0) 0x558c575c1000 con 0x558c54c19c00 2019-12-07T19:34:04.754+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af4d0 space 0x558c574e1f28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.754+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000020 epoch 27 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=10, reqid=client.4540.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=27'16)) v2 2019-12-07T19:34:04.754+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=10, reqid=client.4540.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=27'16)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25200~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25600~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25600~200 = 512 2019-12-07T19:34:04.754+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=10, reqid=client.4540.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=27'16)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25a00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25a00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=10, reqid=client.4540.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=27'16)) v2) prio 127 cost 0 e27) pg 0x558c57676000 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25e00~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.754+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25e00~200 = 512 2019-12-07T19:34:04.754+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26200~200 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x54000 (344064) 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need {<0x1000, 0x1000> : [0x26200:1200~200]} 2019-12-07T19:34:04.754+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c54e79b80 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=10, reqid=client.4540.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=27'16)) v2 pg pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.754+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.754+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.cs0 27/26 ECSubWrite(tid=10, reqid=client.4540.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=27'16)) v2 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped m=1 mbc={} ps=[2~1]] append_log log((0'0,27'16], crt=21'12) [] 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] rollforward: entry=27'13 (21'8) clone 1:3a5a3f5b:::smithi13140796-23:4 by unknown.0.0:0 2019-12-07T19:33:58.673986+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] rollforward: entry=27'14 (21'8) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:232 2019-12-07T19:34:04.737301+0000 0 ObjectCleanRegions clean_offsets: [0~376832,1097728~18446744073708453887], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] rollforward: entry=27'15 (27'14) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:233 2019-12-07T19:34:04.739951+0000 0 ObjectCleanRegions clean_offsets: [0~1097728,1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] rollforward: entry=27'16 (27'15) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:234 2019-12-07T19:34:04.739979+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] append_log approx pg log length = 16 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x26200:1200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e1f28 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.755+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.755+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af560 space 0x558c574e1f28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26200~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26600~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26600~200 = 512 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea1500 seq 11 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26a00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26a00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26e00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26e00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5421 (1 ios * 4000 + 1421 bytes) 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27200~200 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need {<0x0, 0x1000> : [0x27200:200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x27200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.755+0000 7f8f0297e700 10 osd.7 27 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:34:04.755+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x27200:200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757f828 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af5f0 space 0x558c5757f828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27200~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27600~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27600~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27a00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27a00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27e00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27e00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28200~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need {<0x1000, 0x1000> : [0x28200:1200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1232 < min_flush_size 524288 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6b072~4e7 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6b072 0x6b072~4e7 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3b072 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x72 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x559 and padding block with 0xaa7 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x28200:1200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757f828 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af680 space 0x558c5757f828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28200~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28600~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28600~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28a00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28a00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28e00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28e00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29200~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need {<0x0, 0x1000> : [0x29200:200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x29200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6b559 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.755+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000144077s (0.000000098s flush + 0.000143979s kv commit) 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.755+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.755+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:04.755+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.755+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:04.755+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.755+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWriteReply(1.cs1 27/26 ECSubWriteReply(tid=10, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558c54d76a00 con 0x558c54c19c00 2019-12-07T19:34:04.755+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x29200:200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757f668 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af710 space 0x558c5757f668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29200~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29600~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29600~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29a00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29a00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29e00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29e00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a200~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need {<0x1000, 0x1000> : [0x2a200:1200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2a200:1200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757f668 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af7a0 space 0x558c5757f668 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a200~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a600~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a600~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2aa00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2aa00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ae00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ae00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b200~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need {<0x0, 0x1000> : [0x2b200:200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2b200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2b200:200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fd68 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af830 space 0x558c5757fd68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b200~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b600~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b600~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ba00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ba00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2be00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2be00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c200~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need {<0x1000, 0x1000> : [0x2c200:1200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2c200:1200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fd68 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af8c0 space 0x558c5757fd68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c200~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c600~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c600~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ca00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ca00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ce00~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.755+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ce00~200 = 512 2019-12-07T19:34:04.755+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d200~200 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x54000 (344064) 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need {<0x0, 0x1000> : [0x2d200:200~200]} 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2d200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.755+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.756+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 28 ==== MOSDPGPush(1.cs0 27/26 [PushOp(1:3f7a742d:::smithi13140796-9:head, version: 21'12, data_included: [0~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3f7a742d:::smithi13140796-9:head@21'12, size: 212992, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:212992, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 54306+0+0 (unknown 0 0 0) 0x558c54d76a00 con 0x558c54c19c00 2019-12-07T19:34:04.756+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e78c60 prio 127 cost 54248 latency 0.000025 epoch 27 MOSDPGPush(1.cs0 27/26 [PushOp(1:3f7a742d:::smithi13140796-9:head, version: 21'12, data_included: [0~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3f7a742d:::smithi13140796-9:head@21'12, size: 212992, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:212992, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2d200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3f7a742d:::smithi13140796-9:head, version: 21'12, data_included: [0~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3f7a742d:::smithi13140796-9:head@21'12, size: 212992, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:212992, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 54248 e27) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af950 space 0x558c5757eda8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3f7a742d:::smithi13140796-9:head, version: 21'12, data_included: [0~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3f7a742d:::smithi13140796-9:head@21'12, size: 212992, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:212992, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 54248 e27) queued 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2da00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2da00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2de00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDPGPush(1.cs0 27/26 [PushOp(1:3f7a742d:::smithi13140796-9:head, version: 21'12, data_included: [0~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3f7a742d:::smithi13140796-9:head@21'12, size: 212992, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:212992, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 54248 e27) pg 0x558c57676000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2de00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need {<0x1000, 0x1000> : [0x2e200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f06986700 10 osd.7 27 dequeue_op 0x558c54e78c60 prio 127 cost 54248 latency 0.000082 MOSDPGPush(1.cs0 27/26 [PushOp(1:3f7a742d:::smithi13140796-9:head, version: 21'12, data_included: [0~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3f7a742d:::smithi13140796-9:head@21'12, size: 212992, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:212992, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.cs0 27/26 [PushOp(1:3f7a742d:::smithi13140796-9:head, version: 21'12, data_included: [0~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:3f7a742d:::smithi13140796-9:head@21'12, size: 212992, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:212992, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.756+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] on_local_recover: 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:34:04.756+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 lc 21'11 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped m=1 mbc={} ps=[2~1]] got missing 1:3f7a742d:::smithi13140796-9:head v 21'12 2019-12-07T19:34:04.756+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 lcod 21'11 active+remapped mbc={} ps=[2~1]] last_complete now 27'16 log.complete_to at end 2019-12-07T19:34:04.756+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2e200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757eda8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576af9e0 space 0x558c5757eda8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c5740b500 seq 12 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3f7a742d:::smithi13140796-9:head# key 0x8080000000000000013f7a74'-!smithi13140796-9!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ea00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ea00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ee00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ee00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need {<0x0, 0x1000> : [0x2f200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2f200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3f7a742d:::smithi13140796-9:head# key 0x8080000000000000013f7a74'-!smithi13140796-9!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:04.756+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1133 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# = 0 2019-12-07T19:34:04.756+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~d000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3f7a742d:::smithi13140796-9:head# 0x0~d000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740b500 7 blobs 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.756+0000 7f8f06986700 10 fbmap_alloc 0x558c53a72e00 allocate 0xd000/1000,d000,0 2019-12-07T19:34:04.756+0000 7f8f06986700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3ba000~d000/1000,d000,0 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3ba000~d000] 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573964d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265b20 blob([0x3ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573964d0 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c55265b20) get_ref 0x0~2000 Blob(0x558c55265b20 blob([0x3ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573964d0 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c55265b20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c55265b20 blob([0x3ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573964d0 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c573964e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57490 blob([0x3bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9a40 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e57490) get_ref 0x0~2000 Blob(0x558c54e57490 blob([0x3bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9a40 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e57490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54e57490 blob([0x3bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9a40 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54cf9a58 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf9960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf9960 blob([0x3be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42ee0 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54cf9960) get_ref 0x0~2000 Blob(0x558c54cf9960 blob([0x3be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42ee0 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54cf9960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2f200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54cf9960 blob([0x3be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42ee0 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e42ef8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fdd8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afa70 space 0x558c5757fdd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e42e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f600~200 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42e70 blob([0x3c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42e00 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e42e70) get_ref 0x0~2000 Blob(0x558c54e42e70 blob([0x3c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42e00 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e42e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54e42e70 blob([0x3c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42e00 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e42e18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fa00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42d90 blob([0x3c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57570 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fa00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e42d90) get_ref 0x0~2000 Blob(0x558c54e42d90 blob([0x3c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57570 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fe00~200 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e42d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c54e42d90 blob([0x3c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57570 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e57588 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fe00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30200~200 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e575e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e575e0 blob([0x3c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57650 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e575e0) get_ref 0x0~2000 Blob(0x558c54e575e0 blob([0x3c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57650 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e575e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x30200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54e575e0 blob([0x3c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57650 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e57668 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e576c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e57730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e576c0 blob([0x3c6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57730 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e576c0) get_ref 0x0~1000 Blob(0x558c54e576c0 blob([0x3c6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e57730 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.blob(0x558c54e576c0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~1000: 0x0~1000 Blob(0x558c54e576c0 blob([0x3c6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e57730 sbid 0x0)) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54e57748 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xd000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.extentmap(0x558c5793b508) dirty_range mark inline shard dirty 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~d000 = 0 2019-12-07T19:34:04.756+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 4 keys 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 4 keys = 0 2019-12-07T19:34:04.756+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000456'.missing/0000000000000001.CFE5E24B.head.smithi13140796-9..' <- missing/0000000000000001.CFE5E24B.head.smithi13140796-9.. 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.756+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 87600 (8 ios * 4000 + 55600 bytes) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes 0x558c5793b400 shared_blobs 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.extentmap(0x558c5793b508) update 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore.extentmap(0x558c5793b508) update inline shard 133 bytes from 7 extents 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3f7a742d:::smithi13140796-9:head# is 623 (484 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:34:04.756+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[3ba000~d000] released 0x[] 2019-12-07T19:34:04.756+0000 7f8f06986700 10 freelist allocate 0x3ba000~d000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x30200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fdd8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afb00 space 0x558c5757fdd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740b500 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need {<0x0, 0x1000> : [0x31200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x31200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f06986700 10 osd.7 27 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:34:04.756+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x31200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f06986700 20 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 lcod 21'11 active+remapped mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afb90 space 0x558c5757ea98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need {<0x1000, 0x1000> : [0x32200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 aio_wait 2019-12-07T19:34:04.756+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x32200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757ea98 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afc20 space 0x558c5757ea98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need {<0x0, 0x1000> : [0x33200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x33200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2062 < min_flush_size 524288 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793b400 had 1 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2085 < min_flush_size 524288 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2085 < min_flush_size 524288 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6b559~825 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6b559 0x6b559~825 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x33200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fba8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3b559 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x559 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afcb0 space 0x558c5757fba8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd7e and padding block with 0x282 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need {<0x1000, 0x1000> : [0x34200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x34200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x34200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fba8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afd40 space 0x558c5757fba8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6bd7e 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need {<0x0, 0x1000> : [0x35200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x35200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000157262s (0.000037299s flush + 0.000119963s kv commit) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.756+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.756+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:04.756+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 0x558c5793b400 2019-12-07T19:34:04.756+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9a58 in 0x558c53a59ce0) _finish_write discard buffer(0x558c548de510 space 0x558c54cf9a58 0x0~2000 writing nocache) 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42e18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c551a8e10 space 0x558c54e42e18 0x0~2000 writing nocache) 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42ef8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c548de5a0 space 0x558c54e42ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57588 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54c9ae10 space 0x558c54e57588 0x0~2000 writing nocache) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x35200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57668 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54c9ad80 space 0x558c54e57668 0x0~2000 writing nocache) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e57748 in 0x558c53a59ce0) _finish_write discard buffer(0x558c54c9ac60 space 0x558c54e57748 0x0~1000 writing nocache) 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573964e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c548de3f0 space 0x558c573964e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afdd0 space 0x558c5757eb78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:04.756+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 lcod 21'11 active+remapped mbc={} ps=[2~1]] recovery_committed_to version 27'16 now ondisk 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35600~200 2019-12-07T19:34:04.756+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_trim(1.cs1 27'16 e27/27) v2 -- 0x558c54d9f4a0 con 0x558c54c19c00 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGPushReply(1.cs1 27/26 [PushReplyOp(1:3f7a742d:::smithi13140796-9:head)]) v3 -- 0x558c54c58c00 con 0x558c54c19c00 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need {<0x1000, 0x1000> : [0x36200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x36200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x36200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757eb78 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afe60 space 0x558c5757eb78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need {<0x0, 0x1000> : [0x37200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x37200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x37200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fc18 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c576afef0 space 0x558c5757fc18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need {<0x1000, 0x1000> : [0x38200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x38200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x38200:1200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fc18 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57982510 space 0x558c5757fc18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38200~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38600~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38600~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38a00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38a00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38e00~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.756+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38e00~200 = 512 2019-12-07T19:34:04.756+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39200~200 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x54000 (344064) 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need {<0x0, 0x1000> : [0x39200:200~200]} 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x39200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.756+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x39200:200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fc88 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c579825a0 space 0x558c5757fc88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39200~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39600~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39600~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39a00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39a00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39e00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39e00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a200~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need {<0x1000, 0x1000> : [0x3a200:1200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3a200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3a200:1200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757fc88 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57982630 space 0x558c5757fc88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a200~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a600~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a600~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3aa00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3aa00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ae00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ae00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b200~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need {<0x0, 0x1000> : [0x3b200:200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3b200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3b200:200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c579826c0 space 0x558c5757ebe8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b200~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b600~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b600~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ba00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ba00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3be00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3be00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c200~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3c200:1200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3c200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3c200:1200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757ebe8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57982750 space 0x558c5757ebe8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c200~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c600~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c600~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ca00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ca00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ce00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ce00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d200~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need {<0x0, 0x1000> : [0x3d200:200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3d200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3d200:200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574f1278 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c579827e0 space 0x558c574f1278 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d200~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d600~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d600~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3da00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3da00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3de00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3de00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e200~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need {<0x1000, 0x1000> : [0x3e200:1200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3e200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3e200:1200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574f1278 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.757+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57982870 space 0x558c574f1278 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.757+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 29 ==== MRecoveryReserve(1.cs0 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54d9f4a0 con 0x558c54c19c00 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e200~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e600~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.757+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e600~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ea00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ea00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ee00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.757+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558c57676000 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ee00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f200~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need {<0x0, 0x1000> : [0x3f200:200~200]} 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3f200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.757+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.757+0000 7f8f0297e700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.758844+0000 1 0.000038 2019-12-07T19:34:04.757+0000 7f8f0297e700 5 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3f200:200~200]} 2019-12-07T19:34:04.757+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.757+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.757+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57982900 space 0x558c54e42328 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f200~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f600~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f600~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fa00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fa00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fe00~200 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.757+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fe00~200 = 512 2019-12-07T19:34:04.757+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=512 len=131072 2019-12-07T19:34:04.757+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:04.757+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:04.757+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 344064 2019-12-07T19:34:04.757+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key 2019-12-07T19:34:04.757+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key = 0 2019-12-07T19:34:04.757+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.3s0 27/26 ECSubReadReply(tid=48, attrs_read=0)) v2 -- 0x558c57454500 con 0x558c54c5b400 2019-12-07T19:34:04.757+0000 7f8f07187700 10 osd.7 27 dequeue_op 0x558c54e79340 finish 2019-12-07T19:34:04.757+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.758+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 52 ==== MRecoveryReserve(1.bs3 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c575cc000 con 0x558c54c5b400 2019-12-07T19:34:04.758+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.758+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.758+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.758+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.758+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.758+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.758+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.758+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.759643+0000 3 0.000219 2019-12-07T19:34:04.758+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.758+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:04.758+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.758+0000 7f8f0b990700 15 osd.7 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.758+0000 7f8f0b990700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.758+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.758+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.758+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.758+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.758+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.758+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.bs0 GRANT e27/27) v3 -- 0x558c576dcb40 con 0x558c54c5b400 2019-12-07T19:34:04.758+0000 7f8f07187700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.759714+0000 1 0.000029 2019-12-07T19:34:04.758+0000 7f8f07187700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.758+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:04.758+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.761+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 53 ==== MOSDPGPush(1.bs3 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 74790+0+0 (unknown 0 0 0) 0x558c54c1e600 con 0x558c54c5b400 2019-12-07T19:34:04.761+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c573358c0 prio 127 cost 74728 latency 0.000024 epoch 27 MOSDPGPush(1.bs3 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.761+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e27) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e27) queued 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c573358c0 prio 127 cost 74728 latency 0.000075 MOSDPGPush(1.bs3 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.bs3 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] on_local_recover: 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] got missing 1:db85c185:::smithi13140796-18:head v 21'2 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[2~1]] last_complete now 21'8 log.complete_to 21'9 2019-12-07T19:34:04.761+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730a600 seq 4 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:db85c185:::smithi13140796-18:head# key 0x838000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:db85c185:::smithi13140796-18:head# key 0x838000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.761+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1134 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# = 0 2019-12-07T19:34:04.761+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~12000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:db85c185:::smithi13140796-18:head# 0x0~12000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~12000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730a600 9 blobs 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x12000/1000,12000,0 2019-12-07T19:34:04.761+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3c7000~12000/1000,12000,0 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3c7000~12000] 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f11f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1960 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c574f1960) get_ref 0x0~2000 Blob(0x558c574f1960 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c574f1960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c574f1960 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad730 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad730) get_ref 0x0~2000 Blob(0x558c576ad730 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c576ad730 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad880 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad8f0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad880) get_ref 0x0~2000 Blob(0x558c576ad880 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad8f0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c576ad880 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad8f0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad908 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad960 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad960) get_ref 0x0~2000 Blob(0x558c576ad960 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c576ad960 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad9e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ada40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ada40 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adab0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ada40) get_ref 0x0~2000 Blob(0x558c576ada40 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adab0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576ada40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c576ada40 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adab0 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576adb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576adb20 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576adb20) get_ref 0x0~2000 Blob(0x558c576adb20 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore.blob(0x558c576adb20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.761+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c576adb20 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576adc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576adc00 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.blob(0x558c576adc00) get_ref 0x0~2000 Blob(0x558c576adc00 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.blob(0x558c576adc00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c576adc00 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adc88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576adce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576add50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576adce0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576add50 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.blob(0x558c576adce0) get_ref 0x0~2000 Blob(0x558c576adce0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576add50 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.blob(0x558c576adce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c576adce0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576add50 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576add68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576addc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ade30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576addc0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ade30 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.blob(0x558c576addc0) get_ref 0x0~2000 Blob(0x558c576addc0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ade30 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.blob(0x558c576addc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c576addc0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ade30 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ade48 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x12000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.extentmap(0x558c575bb788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~12000 = 0 2019-12-07T19:34:04.762+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 4 keys 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 4 keys = 0 2019-12-07T19:34:04.762+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045b'.missing/0000000000000001.BD1A381A.head.smithi13140796-18..' <- missing/0000000000000001.BD1A381A.head.smithi13140796-18.. 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.762+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a600 cost 115965 (10 ios * 4000 + 75965 bytes) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a600 onodes 0x558c575bb680 shared_blobs 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.extentmap(0x558c575bb788) update 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore.extentmap(0x558c575bb788) update inline shard 174 bytes from 9 extents 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:db85c185:::smithi13140796-18:head# is 665 (485 bytes onode + 2 bytes spanning blobs + 178 bytes inline extents) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a600 allocated 0x[3c7000~12000] released 0x[] 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 freelist allocate 0x3c7000~12000 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 prepare 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730a600 2019-12-07T19:34:04.762+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c573358c0 finish 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.762+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.762+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.762+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 aio_wait 2019-12-07T19:34:04.762+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a600 2019-12-07T19:34:04.762+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 io_done 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1988 < min_flush_size 524288 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb680 had 1 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2011 < min_flush_size 524288 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2011 < min_flush_size 524288 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6bd7e~7db to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6bd7e 0x6bd7e~7db to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3bd7e 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd7e 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x559 and padding block with 0xaa7 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6c559 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.762+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000177234s (0.000042179s flush + 0.000135055s kv commit) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a600> 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 kv_submitted 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a600 2019-12-07T19:34:04.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 finishing 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a600 onodes 0x558c575bb680 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _finish_write discard buffer(0x558c54cdf680 space 0x558c574f1208 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _finish_write discard buffer(0x558c54cdf7a0 space 0x558c576ad828 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad908 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982d80 space 0x558c576ad908 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad9e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982e10 space 0x558c576ad9e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adac8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982ea0 space 0x558c576adac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982f30 space 0x558c576adba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adc88 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982fc0 space 0x558c576adc88 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576add68 in 0x558c53a59c00) _finish_write discard buffer(0x558c57983050 space 0x558c576add68 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ade48 in 0x558c53a59c00) _finish_write discard buffer(0x558c579830e0 space 0x558c576ade48 0x0~2000 writing nocache) 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a600 done 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:04.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a600 [] 2019-12-07T19:34:04.762+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.762+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[2~1]] recovery_committed_to version 21'8 now ondisk 2019-12-07T19:34:04.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.762+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 27/26 [PushReplyOp(1:db85c185:::smithi13140796-18:head)]) v3 -- 0x558c54c1fc00 con 0x558c54c5b400 2019-12-07T19:34:04.762+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.763+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 54 ==== MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=50, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558c57454500 con 0x558c54c5b400 2019-12-07T19:34:04.763+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335a20 prio 127 cost 0 latency 0.000021 epoch 27 MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=50, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.763+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=50, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=50, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=50, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558c5513e000 2019-12-07T19:34:04.763+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c57335a20 prio 127 cost 0 latency 0.000316 MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=50, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.763+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.3s4 27/26 ECSubRead(tid=50, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.763+0000 7f8f0317f700 25 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.763+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40200~200 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40200~200 size 0x54000 (344064) 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need {<0x1000, 0x1000> : [0x40200:1200~200]} 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x40200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x40200:1200~200]} 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42328 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c54cdf710 space 0x558c54e42328 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.763+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40200~200 = 512 2019-12-07T19:34:04.763+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40600~200 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40600~200 size 0x54000 (344064) 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.763+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40600~200 = 512 2019-12-07T19:34:04.763+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40a00~200 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.763+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40a00~200 = 512 2019-12-07T19:34:04.763+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40e00~200 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.763+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40e00~200 = 512 2019-12-07T19:34:04.763+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41200~200 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41200~200 size 0x54000 (344064) 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need {<0x0, 0x1000> : [0x41200:200~200]} 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x41200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x41200:200~200]} 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e421d8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983290 space 0x558c54e421d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.763+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41200~200 = 512 2019-12-07T19:34:04.763+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41600~200 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41600~200 size 0x54000 (344064) 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.763+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x42200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x42200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x42200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e421d8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983320 space 0x558c54e421d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42200~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42600~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42600~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need {<0x0, 0x1000> : [0x43200:200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x43200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x43200:200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42478 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c579833b0 space 0x558c54e42478 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43200~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43600~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43600~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need {<0x1000, 0x1000> : [0x44200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x44200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x44200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42478 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983440 space 0x558c54e42478 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44200~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44600~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44600~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need {<0x0, 0x1000> : [0x45200:200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x45200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x45200:200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42398 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c579834d0 space 0x558c54e42398 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45200~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45600~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45600~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need {<0x1000, 0x1000> : [0x46200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x46200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x46200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42398 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983560 space 0x558c54e42398 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46200~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46600~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46600~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need {<0x0, 0x1000> : [0x47200:200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x47200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x47200:200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e420f8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c579835f0 space 0x558c54e420f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47200~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47600~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47600~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x48200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x48200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x48200:1200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e420f8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983680 space 0x558c54e420f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48200~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48600~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48600~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48600~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48a00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48a00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48e00~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.764+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48e00~200 = 512 2019-12-07T19:34:04.764+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49200~200 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49200~200 size 0x54000 (344064) 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need {<0x0, 0x1000> : [0x49200:200~200]} 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x49200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.764+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x49200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42638 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983710 space 0x558c54e42638 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49a00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49e00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need {<0x1000, 0x1000> : [0x4a200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4a200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4a200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42638 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c579837a0 space 0x558c54e42638 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4aa00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4aa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4aa00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ae00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ae00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4b200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need {<0x0, 0x1000> : [0x4b200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4b200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4b200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983830 space 0x558c54e42088 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4b600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ba00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ba00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4be00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4be00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4be00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need {<0x1000, 0x1000> : [0x4c200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4c200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4c200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e42088 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c579838c0 space 0x558c54e42088 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ca00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ca00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ce00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ce00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4d200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need {<0x0, 0x1000> : [0x4d200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4d200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4d200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983950 space 0x558c54e43f98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4d600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4da00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4da00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4da00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4de00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4de00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4de00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need {<0x1000, 0x1000> : [0x4e200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4e200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4e200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e43f98 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c579839e0 space 0x558c54e43f98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ea00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ea00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ee00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ee00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4f200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need {<0x0, 0x1000> : [0x4f200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4f200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4f200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e43ba8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983a70 space 0x558c54e43ba8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4f600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fa00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4fa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fa00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fe00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4fe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fe00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need {<0x1000, 0x1000> : [0x50200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x50200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x50200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e43ba8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983b00 space 0x558c54e43ba8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50a00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50e00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need {<0x0, 0x1000> : [0x51200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x51200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x51200:200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e439e8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983b90 space 0x558c54e439e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51a00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51e00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x52200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x52200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x52200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e439e8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983c20 space 0x558c54e439e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52a00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52e00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need {<0x1000, 0x1000> : [0x53200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x53200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x53200:1200~200]} 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54e43ac8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983cb0 space 0x558c54e43ac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53200~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53600~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53a00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53e00~200 = 512 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54200~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54600~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54a00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54e00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55200~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55600~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55a00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55e00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56200~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56600~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56a00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56e00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57200~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57600~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57a00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57e00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58200~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58200~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58600~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58600~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58600~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58a00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58a00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58e00~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.765+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58e00~200 = 0 2019-12-07T19:34:04.765+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59200~200 2019-12-07T19:34:04.765+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5aa00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5aa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5aa00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ae00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ae00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5b200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5b600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ba00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ba00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5be00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5be00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5be00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ca00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ca00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ce00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ce00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5d200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5d600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5da00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5da00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5da00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5de00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5de00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5de00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ea00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ea00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ee00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ee00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5f200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5f600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fa00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5fa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fa00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fe00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5fe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fe00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6aa00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6aa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6aa00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ae00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ae00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6b200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6b600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ba00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ba00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6be00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6be00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6be00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ca00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ca00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ce00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ce00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6d200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6d600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6da00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6da00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6da00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6de00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6de00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6de00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ea00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ea00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ee00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ee00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6f200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6f600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fa00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6fa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fa00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fe00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6fe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fe00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79a00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79a00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79a00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79e00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79e00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79e00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7aa00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7aa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7aa00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ae00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ae00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ae00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7b200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7b600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ba00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ba00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ba00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7be00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7be00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7be00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ca00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ca00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ca00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ce00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ce00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ce00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7d200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7d600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7da00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7da00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7da00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7de00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7de00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7de00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ea00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ea00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ea00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ee00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ee00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ee00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f200~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7f200~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f200~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f600~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7f600~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f600~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fa00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7fa00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fa00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fe00~200 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7fe00~200 size 0x54000 (344064) 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fe00~200 = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=40960 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 344064 2019-12-07T19:34:04.766+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key = 0 2019-12-07T19:34:04.766+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.3s0 27/26 ECSubReadReply(tid=50, attrs_read=0)) v2 -- 0x558c57573400 con 0x558c54c5b400 2019-12-07T19:34:04.766+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c57335a20 finish 2019-12-07T19:34:04.766+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.767+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 55 ==== MOSDPGPush(1.bs3 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 209958+0+0 (unknown 0 0 0) 0x558c54c1fc00 con 0x558c54c5b400 2019-12-07T19:34:04.767+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334840 prio 127 cost 209896 latency 0.000029 epoch 27 MOSDPGPush(1.bs3 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.767+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e27) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.767+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e27) queued 2019-12-07T19:34:04.767+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.767+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.767+0000 7f8f07187700 10 osd.7 27 dequeue_op 0x558c57334840 prio 127 cost 209896 latency 0.000071 MOSDPGPush(1.bs3 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs3( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] 2019-12-07T19:34:04.767+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.bs3 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.767+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] on_local_recover: 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:04.767+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] got missing 1:dce63cc3:::smithi13140796-50:head v 21'9 2019-12-07T19:34:04.767+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[2~1]] last_complete now 21'9 log.complete_to 21'10 2019-12-07T19:34:04.767+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.767+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730b200 seq 5 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:dce63cc3:::smithi13140796-50:head# key 0x838000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.767+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:dce63cc3:::smithi13140796-50:head# key 0x838000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.767+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1135 2019-12-07T19:34:04.767+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:34:04.767+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.767+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~33000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730b200 26 blobs 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.767+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x33000/1000,33000,0 2019-12-07T19:34:04.767+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3d9000~33000/1000,33000,0 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3d9000~33000] 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57566d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac000 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57566d20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c576ac000) get_ref 0x0~2000 Blob(0x558c576ac000 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57566d20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c576ac000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c576ac000 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57566d20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57566d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e424d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e424d0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c54e424d0) get_ref 0x0~2000 Blob(0x558c54e424d0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c54e424d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54e424d0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fdc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1ab0 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdc70 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c574f1ab0) get_ref 0x0~2000 Blob(0x558c574f1ab0 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdc70 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c574f1ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c574f1ab0 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdc70 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fdc88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fdc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fdb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fdc00 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdb90 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c574fdc00) get_ref 0x0~2000 Blob(0x558c574fdc00 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdb90 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c574fdc00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c574fdc00 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdb90 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fdba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b29a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2930 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b29a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2930) get_ref 0x0~2000 Blob(0x558c579b2930 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b29a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c579b2930 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b29a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b29b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2a10 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2a80 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2a10) get_ref 0x0~2000 Blob(0x558c579b2a10 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2a80 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c579b2a10 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2a80 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2a98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2af0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2b60 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2af0) get_ref 0x0~2000 Blob(0x558c579b2af0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2b60 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c579b2af0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2b60 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2bd0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2c40 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2bd0) get_ref 0x0~2000 Blob(0x558c579b2bd0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2c40 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c579b2bd0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2c40 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2cb0 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2cb0) get_ref 0x0~2000 Blob(0x558c579b2cb0 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c579b2cb0 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2d90 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2d90) get_ref 0x0~2000 Blob(0x558c579b2d90 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c579b2d90 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2e70 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2e70) get_ref 0x0~2000 Blob(0x558c579b2e70 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c579b2e70 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2f50 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2fc0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2f50) get_ref 0x0~2000 Blob(0x558c579b2f50 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2fc0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b2f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c579b2f50 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2fc0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b30a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3030 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3030) get_ref 0x0~2000 Blob(0x558c579b3030 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c579b3030 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3110 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3180 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3110) get_ref 0x0~2000 Blob(0x558c579b3110 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3180 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c579b3110 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3180 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b31f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b31f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3260 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b31f0) get_ref 0x0~2000 Blob(0x558c579b31f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3260 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b31f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c579b31f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3260 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3278 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b32d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b32d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3340 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b32d0) get_ref 0x0~2000 Blob(0x558c579b32d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3340 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b32d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c579b32d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3340 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3358 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b33b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b33b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3420 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b33b0) get_ref 0x0~2000 Blob(0x558c579b33b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3420 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b33b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c579b33b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3420 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3438 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3490 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3500 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3490) get_ref 0x0~2000 Blob(0x558c579b3490 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3500 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c579b3490 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3500 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3518 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b35e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3570 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b35e0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3570) get_ref 0x0~2000 Blob(0x558c579b3570 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b35e0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c579b3570 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b35e0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b35f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b36c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3650 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b36c0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3650) get_ref 0x0~2000 Blob(0x558c579b3650 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b36c0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c579b3650 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b36c0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b36d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3730 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3730) get_ref 0x0~2000 Blob(0x558c579b3730 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c579b3730 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b37b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3810 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3810) get_ref 0x0~2000 Blob(0x558c579b3810 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3810) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c579b3810 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b38f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b38f0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b38f0) get_ref 0x0~2000 Blob(0x558c579b38f0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b38f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c579b38f0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b39d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b39d0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b39d0) get_ref 0x0~2000 Blob(0x558c579b39d0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c579b39d0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3ab0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3b20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3ab0) get_ref 0x0~2000 Blob(0x558c579b3ab0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3b20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c579b3ab0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3b20 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3b90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b90) get_ref 0x0~1000 Blob(0x558c579b3b90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b90) get_ref init 0x1000, 1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~1000: 0x0~1000 Blob(0x558c579b3b90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x33000 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore.extentmap(0x558c57572108) dirty_range mark inline shard dirty 2019-12-07T19:34:04.767+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 = 0 2019-12-07T19:34:04.767+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 4 keys 2019-12-07T19:34:04.767+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 4 keys = 0 2019-12-07T19:34:04.767+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.767+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045b'.missing/0000000000000001.B376C33C.head.smithi13140796-50..' <- missing/0000000000000001.B376C33C.head.smithi13140796-50.. 2019-12-07T19:34:04.768+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.768+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.768+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.768+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 319133 (27 ios * 4000 + 211133 bytes) 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes 0x558c57572000 shared_blobs 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore.extentmap(0x558c57572108) update 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore.extentmap(0x558c57572108) update inline shard 494 bytes from 26 extents 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:dce63cc3:::smithi13140796-50:head# is 985 (485 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:04.768+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[3d9000~33000] released 0x[] 2019-12-07T19:34:04.768+0000 7f8f07187700 10 freelist allocate 0x3d9000~33000 2019-12-07T19:34:04.768+0000 7f8f07187700 20 freelist _xor first_key 0x380000 last_key 0x400000 2019-12-07T19:34:04.768+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:04.768+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730b200 2019-12-07T19:34:04.768+0000 7f8f07187700 10 osd.7 27 dequeue_op 0x558c57334840 finish 2019-12-07T19:34:04.768+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.768+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.768+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.768+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 aio_wait 2019-12-07T19:34:04.768+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:04.768+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2337 < min_flush_size 524288 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572000 had 1 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2360 < min_flush_size 524288 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2360 < min_flush_size 524288 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6c559~938 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6c559 0x6c559~938 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3c559 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x559 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe91 and padding block with 0x16f 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6ce91 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.768+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000153467s (0.000040871s flush + 0.000112596s kv commit) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.768+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b200> 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.768+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:04.768+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 0x558c57572000 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fdba8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982bd0 space 0x558c574fdba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fdc88 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982b40 space 0x558c574fdc88 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57566d38 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982a20 space 0x558c57566d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982ab0 space 0x558c5757e9b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b29b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982c60 space 0x558c579b29b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2a98 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982cf0 space 0x558c579b2a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2b78 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e000 space 0x558c579b2b78 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[2~1]] recovery_committed_to version 21'9 now ondisk 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2c58 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e090 space 0x558c579b2c58 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2d38 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e120 space 0x558c579b2d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2e18 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e1b0 space 0x558c579b2e18 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2ef8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e240 space 0x558c579b2ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2fd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e2d0 space 0x558c579b2fd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e360 space 0x558c579b30b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 27/26 [PushReplyOp(1:dce63cc3:::smithi13140796-50:head)]) v3 -- 0x558c57569200 con 0x558c54c5b400 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3198 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e3f0 space 0x558c579b3198 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3278 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e480 space 0x558c579b3278 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3358 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e510 space 0x558c579b3358 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3438 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e5a0 space 0x558c579b3438 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3518 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e630 space 0x558c579b3518 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b35f8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e6c0 space 0x558c579b35f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b36d8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e750 space 0x558c579b36d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b37b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e7e0 space 0x558c579b37b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3898 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e870 space 0x558c579b3898 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3978 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e900 space 0x558c579b3978 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3a58 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796e990 space 0x558c579b3a58 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3b38 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796ea20 space 0x558c579b3b38 0x0~2000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796eab0 space 0x558c579b3c18 0x0~1000 writing nocache) 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:04.768+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [] 2019-12-07T19:34:04.768+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.768+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.769+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 56 ==== MRecoveryReserve(1.3s4 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c576dcb40 con 0x558c54c5b400 2019-12-07T19:34:04.769+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.769+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.769+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.769+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.769+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.769+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558c5513e000 2019-12-07T19:34:04.769+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.769+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.770886+0000 1 0.000042 2019-12-07T19:34:04.769+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.769+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:04.769+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.774+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 57 ==== MOSDPGPush(1.bs3 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 205862+0+0 (unknown 0 0 0) 0x558c576b5800 con 0x558c54c5b400 2019-12-07T19:34:04.774+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335e40 prio 127 cost 205800 latency 0.000024 epoch 27 MOSDPGPush(1.bs3 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.774+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e27) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.774+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e27) queued 2019-12-07T19:34:04.774+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.774+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGPush(1.bs3 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.774+0000 7f8f07187700 10 osd.7 27 dequeue_op 0x558c57335e40 prio 127 cost 205800 latency 0.000063 MOSDPGPush(1.bs3 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs3( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.774+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.bs3 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.774+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] on_local_recover: 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:04.774+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] got missing 1:d313ddd1:::smithi13140796-49:head v 21'10 2019-12-07T19:34:04.774+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[2~1]] last_complete now 21'11 log.complete_to 21'12 2019-12-07T19:34:04.774+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730a000 seq 6 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:d313ddd1:::smithi13140796-49:head# key 0x838000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:d313ddd1:::smithi13140796-49:head# key 0x838000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.774+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1136 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# = 0 2019-12-07T19:34:04.774+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~32000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730a000 25 blobs 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x32000/1000,32000,0 2019-12-07T19:34:04.774+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x40c000~32000/1000,32000,0 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x40c000~32000] 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac1c0) get_ref 0x0~2000 Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac478 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57396620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396620 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac4d0) get_ref 0x0~2000 Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396620 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57396638 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579707e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579707e0) get_ref 0x0~2000 Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579707e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579708c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579708c0) get_ref 0x0~2000 Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579708c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579709a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579709a0) get_ref 0x0~2000 Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579709a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54677260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c54677260) get_ref 0x0~2000 Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c54677260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad7b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac230 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac2a0) get_ref 0x0~2000 Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac230 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac248 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac380) get_ref 0x0~2000 Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c576ac380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573965b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c54e42700) get_ref 0x0~2000 Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c54e42700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c573965c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b28c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2850 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b28c0) get_ref 0x0~2000 Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2850 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b28c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b27e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b27e0) get_ref 0x0~2000 Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b27e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2690 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2700) get_ref 0x0~2000 Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2690 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b26a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b25b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b25b0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2620) get_ref 0x0~2000 Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b25b0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b25c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b24d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b24d0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2540) get_ref 0x0~2000 Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b24d0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b24e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b23f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b23f0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2460) get_ref 0x0~2000 Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b23f0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2408 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2310 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2380) get_ref 0x0~2000 Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2310 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b2380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b22a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970a80 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b22a0) get_ref 0x0~2000 Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970a80 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c579b22a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970b60 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970af0) get_ref 0x0~2000 Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970b60 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970c40 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970bd0) get_ref 0x0~2000 Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970c40 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d20 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970cb0) get_ref 0x0~2000 Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d20 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e00 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970d90) get_ref 0x0~2000 Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e00 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970ee0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970e70) get_ref 0x0~2000 Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970ee0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970fc0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970f50) get_ref 0x0~2000 Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970fc0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57970f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579710a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579710a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57971030) get_ref 0x0~2000 Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579710a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57971030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579710b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57971110) get_ref 0x0~2000 Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.blob(0x558c57971110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x32000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976108) dirty_range mark inline shard dirty 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 0 2019-12-07T19:34:04.774+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 4 keys 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 4 keys = 0 2019-12-07T19:34:04.774+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045b'.missing/0000000000000001.BC8CBBB8.head.smithi13140796-49..' <- missing/0000000000000001.BC8CBBB8.head.smithi13140796-49.. 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.774+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a000 cost 311037 (26 ios * 4000 + 207037 bytes) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a000 onodes 0x558c57976000 shared_blobs 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976108) update 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976108) update inline shard 478 bytes from 25 extents 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:d313ddd1:::smithi13140796-49:head# is 969 (485 bytes onode + 2 bytes spanning blobs + 482 bytes inline extents) 2019-12-07T19:34:04.774+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a000 allocated 0x[40c000~32000] released 0x[] 2019-12-07T19:34:04.774+0000 7f8f07187700 10 freelist allocate 0x40c000~32000 2019-12-07T19:34:04.774+0000 7f8f07187700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 prepare 2019-12-07T19:34:04.774+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730a000 2019-12-07T19:34:04.774+0000 7f8f07187700 10 osd.7 27 dequeue_op 0x558c57335e40 finish 2019-12-07T19:34:04.774+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.774+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.774+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.775+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 aio_wait 2019-12-07T19:34:04.775+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a000 2019-12-07T19:34:04.775+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 io_done 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2292 < min_flush_size 524288 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2315 < min_flush_size 524288 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2315 < min_flush_size 524288 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6ce91~90b to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6ce91 0x6ce91~90b to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3ce91 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe91 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x79c and padding block with 0x864 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6d79c 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.775+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124571s (0.000041793s flush + 0.000082778s kv commit) 2019-12-07T19:34:04.775+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a000> 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.775+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 kv_submitted 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a000 2019-12-07T19:34:04.775+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 finishing 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a000 onodes 0x558c57976000 2019-12-07T19:34:04.775+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573965c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f0e0 space 0x558c573965c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396638 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796ecf0 space 0x558c57396638 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac248 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796efc0 space 0x558c576ac248 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac328 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f050 space 0x558c576ac328 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac478 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796ec60 space 0x558c576ac478 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad7b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796ef30 space 0x558c576ad7b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970868 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796ed80 space 0x558c57970868 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970948 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796ee10 space 0x558c57970948 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970a28 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796eea0 space 0x558c57970a28 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970a98 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f560 space 0x558c57970a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970b78 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f5f0 space 0x558c57970b78 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[2~1]] recovery_committed_to version 21'11 now ondisk 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970c58 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f680 space 0x558c57970c58 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970d38 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f710 space 0x558c57970d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970e18 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f7a0 space 0x558c57970e18 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970ef8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f830 space 0x558c57970ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970fd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f8c0 space 0x558c57970fd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579710b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f950 space 0x558c579710b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f9e0 space 0x558c57971198 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 27/26 [PushReplyOp(1:d313ddd1:::smithi13140796-49:head)]) v3 -- 0x558c54c1fc00 con 0x558c54c5b400 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2328 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f4d0 space 0x558c579b2328 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2408 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f440 space 0x558c579b2408 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b24e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f3b0 space 0x558c579b24e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b25c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f320 space 0x558c579b25c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b26a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f290 space 0x558c579b26a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2788 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f200 space 0x558c579b2788 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2868 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796f170 space 0x558c579b2868 0x0~2000 writing nocache) 2019-12-07T19:34:04.775+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 done 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:04.775+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a000 [] 2019-12-07T19:34:04.775+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.775+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.776+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 58 ==== MOSDPGRecoveryDelete(1.bs3 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 ==== 123+0+0 (unknown 0 0 0) 0x558c54c1fc00 con 0x558c54c5b400 2019-12-07T19:34:04.776+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335080 prio 3 cost 1000 latency 0.000013 epoch 27 MOSDPGRecoveryDelete(1.bs3 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 2019-12-07T19:34:04.776+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGRecoveryDelete(1.bs3 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e27) 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGRecoveryDelete(1.bs3 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e27) queued 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDPGRecoveryDelete(1.bs3 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.776+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c57335080 prio 3 cost 1000 latency 0.000036 MOSDPGRecoveryDelete(1.bs3 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 pg pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] handle_recovery_delete 0x558c57335080 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] remove_missing_object 1:d07269f7:::smithi13140796-28:head 21'12 2019-12-07T19:34:04.776+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730a900 seq 7 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) get_onode oid 3#1:d07269f7:::smithi13140796-28:head# key 0x838000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs3_head 0x558c54d0f2c0) r -2 v.len 0 2019-12-07T19:34:04.776+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 3#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:04.776+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a900 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a900 onodes shared_blobs 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a900 allocated 0x[] released 0x[] 2019-12-07T19:34:04.776+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 prepare 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a900 2019-12-07T19:34:04.776+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 io_done 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.776+0000 7f8f0317f700 10 osd.7 27 dequeue_op 0x558c57335080 finish 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.776+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6d79c~2e to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6d79c 0x6d79c~2e to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3d79c 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x79c 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7ca and padding block with 0x836 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6d7ca 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000089537s (0.000000101s flush + 0.000089436s kv commit) 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a900> 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 kv_submitted 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a900 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 finishing 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a900 onodes 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a900 done 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a900 [] 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.776+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] on_local_recover: 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:04.776+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] got missing 1:d07269f7:::smithi13140796-28:head v 21'12 2019-12-07T19:34:04.776+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[2~1]] last_complete now 21'12 log.complete_to at end 2019-12-07T19:34:04.776+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.776+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730b500 seq 8 2019-12-07T19:34:04.776+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000045b'.missing/0000000000000001.B0E469FE.head.smithi13140796-28..' <- missing/0000000000000001.B0E469FE.head.smithi13140796-28.. 2019-12-07T19:34:04.776+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.776+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.776+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.776+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b500 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b500 onodes shared_blobs 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.776+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 prepare 2019-12-07T19:34:04.776+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b500 2019-12-07T19:34:04.776+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 io_done 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7f8f07187700 20 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.776+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 406301 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6d7ca~4b8 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6d7ca 0x6d7ca~4b8 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3d7ca 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7ca 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc82 and padding block with 0x37e 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6dc82 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:04.776+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000102665s (0.000000050s flush + 0.000102615s kv commit) 2019-12-07T19:34:04.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b500> 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 kv_submitted 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b500 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 finishing 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b500 onodes 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 done 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:04.776+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b500 [] 2019-12-07T19:34:04.776+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:04.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:04.776+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[2~1]] recovery_committed_to version 21'12 now ondisk 2019-12-07T19:34:04.776+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_trim(1.bs0 21'12 e27/27) v2 -- 0x558c54d9e000 con 0x558c54c5b400 2019-12-07T19:34:04.776+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGRecoveryDeleteReply(1.bs0 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 -- 0x558c54c1e600 con 0x558c54c5b400 2019-12-07T19:34:04.776+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.777+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 59 ==== MRecoveryReserve(1.bs3 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54d9e000 con 0x558c54c5b400 2019-12-07T19:34:04.777+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.777+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.777+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.777+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.777+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.777+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558c54d9b000 2019-12-07T19:34:04.777+0000 7f8f0317f700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.777+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.778830+0000 1 0.000036 2019-12-07T19:34:04.777+0000 7f8f0317f700 5 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.777+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:04.777+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.781+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 94 ==== MRecoveryReserve(1.9s4 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54d9e000 con 0x558c54c64000 2019-12-07T19:34:04.781+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.9s4 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.781+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.781+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.781+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.781+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.781+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558c5500a000 2019-12-07T19:34:04.781+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.781+0000 7f8f04181700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.782912+0000 3 0.000126 2019-12-07T19:34:04.781+0000 7f8f04181700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.781+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:04.781+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.781+0000 7f8f0b990700 15 osd.7 27 enqueue_peering_evt 1.9s4 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.781+0000 7f8f0b990700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.781+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.781+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.781+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.781+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558c5500a000 2019-12-07T19:34:04.781+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.781+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.9s0 GRANT e27/27) v3 -- 0x558c576dc960 con 0x558c54c64000 2019-12-07T19:34:04.781+0000 7f8f08189700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.782997+0000 1 0.000030 2019-12-07T19:34:04.781+0000 7f8f08189700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.781+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:04.781+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.782+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 95 ==== MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 263+0+0 (crc 0 0 0) 0x558c57573400 con 0x558c54c64000 2019-12-07T19:34:04.782+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57334160 prio 3 cost 0 latency 0.000017 epoch 27 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.782+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.782+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.782+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.782+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c5500a000 2019-12-07T19:34:04.782+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c57334160 prio 3 cost 0 latency 0.000117 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.782+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.782+0000 7f8f04181700 25 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.782+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x200~200 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x6000 (24576) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c55279748 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.782+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c54cdea20 space 0x558c55279748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.782+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x200~200 = 512 2019-12-07T19:34:04.782+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x600~200 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x6000 (24576) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.782+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x600~200 = 512 2019-12-07T19:34:04.782+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa00~200 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.782+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa00~200 = 512 2019-12-07T19:34:04.782+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe00~200 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.782+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe00~200 = 512 2019-12-07T19:34:04.782+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1200~200 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x6000 (24576) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c55279748 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.782+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c53a699e0 space 0x558c55279748 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.782+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1200~200 = 512 2019-12-07T19:34:04.782+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1600~200 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x6000 (24576) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.782+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1600~200 = 512 2019-12-07T19:34:04.782+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a00~200 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3dac8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c54c9acf0 space 0x558c54d3dac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2200~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2600~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3dac8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c54cdeab0 space 0x558c54d3dac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3200~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3600~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3dc18 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c54cdeb40 space 0x558c54d3dc18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4200~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4600~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4a00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4e00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3dc18 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.783+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c5796fcb0 space 0x558c54d3dc18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5200~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5600~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5a00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5e00~200 = 512 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xaa00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xaa00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xae00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xae00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xb200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xb200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xb600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xb600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xba00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xba00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xbe00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xbe00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xca00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xca00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xce00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xce00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xd200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xd200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xd600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xd600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xda00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xda00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xde00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xde00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xea00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xea00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xee00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xee00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xf200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xf200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xf600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xf600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xfa00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xfa00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xfe00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xfe00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1aa00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1aa00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ae00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ae00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1b200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1b200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1b600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1b600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ba00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ba00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1be00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1be00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ca00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ca00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ce00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ce00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1d200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1d200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1d600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1d600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1da00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1da00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1de00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1de00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ea00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ea00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ee00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ee00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1f200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1f200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1f600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1f600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1fa00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1fa00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1fe00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1fe00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2aa00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ae00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2b200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2b200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2b600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2b600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ba00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2be00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2be00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ca00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ce00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2d200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2d200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2d600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2d600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2da00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2da00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2de00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2de00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ea00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ee00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2f200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2f200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2f600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2f600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2fa00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2fe00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39a00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39a00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39e00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39e00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3aa00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ae00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3b200~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3b200~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3b600~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3b600~200 = 0 2019-12-07T19:34:04.783+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ba00~200 2019-12-07T19:34:04.783+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3be00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3be00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c200~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c200~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c600~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c600~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ca00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ce00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3d200~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3d200~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3d600~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3d600~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3da00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3da00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3de00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3de00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e200~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e200~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e600~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e600~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ea00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ee00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3f200~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3f200~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3f600~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3f600~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3fa00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3fe00~200 2019-12-07T19:34:04.784+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=12288 2019-12-07T19:34:04.784+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:04.784+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:04.784+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 24576 2019-12-07T19:34:04.784+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key 2019-12-07T19:34:04.784+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key = 0 2019-12-07T19:34:04.784+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=119, attrs_read=0)) v2 -- 0x558c54d97b80 con 0x558c54c64000 2019-12-07T19:34:04.784+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c57334160 finish 2019-12-07T19:34:04.784+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.788+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 96 ==== MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (crc 0 0 0) 0x558c54d97b80 con 0x558c54c64000 2019-12-07T19:34:04.788+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c57335b80 prio 3 cost 0 latency 0.000014 epoch 27 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.788+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.788+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c5500a000 2019-12-07T19:34:04.788+0000 7f8f08189700 10 osd.7 27 dequeue_op 0x558c57335b80 prio 3 cost 0 latency 0.000065 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.788+0000 7f8f08189700 25 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x200~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d75c18 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe000 space 0x558c54d75c18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x200~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x600~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x600~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1200~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d75c18 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe090 space 0x558c54d75c18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1200~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1600~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1600~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2200~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d757b8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe120 space 0x558c54d757b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2200~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2600~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2600~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3200~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d757b8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe1b0 space 0x558c54d757b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3200~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3600~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3600~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4200~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d75908 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe240 space 0x558c54d75908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4200~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4600~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4600~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4a00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4a00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4e00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4e00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5200~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d75908 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe2d0 space 0x558c54d75908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5200~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5600~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5600~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5a00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5a00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5e00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5e00~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6200~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d759e8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.788+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe360 space 0x558c54d759e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6200~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6600~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6600~200 = 512 2019-12-07T19:34:04.788+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6a00~200 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.788+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.788+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6a00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6e00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d759e8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe3f0 space 0x558c54d759e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7200~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7600~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7a00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7e00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf85c8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe480 space 0x558c54cf85c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8200~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8600~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8a00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8e00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf85c8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe510 space 0x558c54cf85c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9200~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9600~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9a00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9e00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf8478 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe5a0 space 0x558c54cf8478 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa200~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa600~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xaa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xaa00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xae00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xae00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xb200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf8478 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe630 space 0x558c54cf8478 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xb200~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xb600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xb600~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xba00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xba00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xbe00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xbe00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c55265ac8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.789+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe6c0 space 0x558c55265ac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc200~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc600~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xca00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xca00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xce00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xce00~200 = 512 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xda00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xda00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xde00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xde00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xea00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xea00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xee00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xee00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xf200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xf200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xf600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xf600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xfa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xfa00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xfe00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xfe00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1aa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1aa00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ae00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ae00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1b200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1b200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1b600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1b600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ba00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ba00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1be00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1be00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ca00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ca00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ce00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ce00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1d200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1d200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1d600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1d600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1da00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1da00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1de00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1de00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ea00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ea00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ee00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ee00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1f200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1f200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1f600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1f600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1fa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1fa00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1fe00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1fe00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2aa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ae00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2b200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2b200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2b600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2b600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ba00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2be00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2be00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ca00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ce00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2d200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2d200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2d600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2d600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2da00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2da00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2de00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2de00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ea00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ee00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2f200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2f200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2f600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2f600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2fa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2fe00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39a00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39a00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39e00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39e00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3aa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ae00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ba00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3be00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3be00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ca00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ce00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3d200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3d200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3d600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3d600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3da00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3da00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3de00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3de00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ea00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ee00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3f200~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3f200~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3f600~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3f600~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3fa00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3fe00~200 2019-12-07T19:34:04.789+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=26624 2019-12-07T19:34:04.789+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:04.789+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:04.789+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 53248 2019-12-07T19:34:04.789+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:04.789+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:04.789+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=120, attrs_read=0)) v2 -- 0x558c57976a00 con 0x558c54c64000 2019-12-07T19:34:04.789+0000 7f8f08189700 10 osd.7 27 dequeue_op 0x558c57335b80 finish 2019-12-07T19:34:04.789+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.796+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 97 ==== MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 263+0+0 (crc 0 0 0) 0x558c57976a00 con 0x558c54c64000 2019-12-07T19:34:04.797+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e789a0 prio 3 cost 0 latency 0.000036 epoch 27 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.797+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.797+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c5500a000 2019-12-07T19:34:04.797+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c54e789a0 prio 3 cost 0 latency 0.000097 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.797+0000 7f8f04181700 25 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3de48 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.797+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c5796fd40 space 0x558c54d3de48 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x200~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x600~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xa00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xa00~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xe00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xe00~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3de48 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.797+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c5796fdd0 space 0x558c54d3de48 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1200~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1600~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1a00~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1e00~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3c638 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.797+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c5796fe60 space 0x558c54d3c638 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2200~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2600~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2a00~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2e00~200 = 512 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x4e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x5e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x6e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x7e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x8e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x9e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xa200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xa200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xa600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xa600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xaa00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xaa00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xae00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xae00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xb200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xb200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xb600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xb600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xba00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xba00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xbe00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xbe00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xc200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xc200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xc600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xc600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xca00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xca00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xce00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xce00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xd200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xd200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xd600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xd600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xda00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xda00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xde00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xde00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xe200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xe200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xe600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xe600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xea00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xea00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xee00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xee00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xf200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xf200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xf600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xf600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xfa00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xfa00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xfe00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0xfe00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x10e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x11e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x12e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x13e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x14e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x15e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x16e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x17e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x18e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x19e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1a200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1a200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1a600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1a600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1aa00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1aa00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ae00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ae00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1b200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1b200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1b600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1b600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ba00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ba00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1be00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1be00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1c200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1c200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1c600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1c600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ca00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ca00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ce00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ce00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1d200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1d200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1d600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1d600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1da00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1da00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1de00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1de00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1e200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1e200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1e600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1e600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ea00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ea00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ee00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1ee00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1f200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1f200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1f600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1f600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1fa00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1fa00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1fe00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x1fe00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x20e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x21e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x22e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x23e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x24e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x25e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x26e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x27e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x28e00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29200~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29200~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29600~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29600~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29a00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29a00~200 = 0 2019-12-07T19:34:04.797+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29e00~200 2019-12-07T19:34:04.797+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x29e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2a200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2a200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2a600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2a600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2aa00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2aa00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ae00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ae00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2b200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2b200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2b600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2b600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ba00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ba00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2be00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2be00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2c200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2c200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2c600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2c600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ca00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ca00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ce00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ce00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2d200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2d200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2d600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2d600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2da00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2da00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2de00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2de00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2e200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2e200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2e600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2e600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ea00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ea00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ee00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2ee00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2f200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2f200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2f600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2f600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2fa00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2fa00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2fe00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x2fe00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x30e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x31e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x32e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x33e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x34e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x35e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x36e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x37e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x38e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39a00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39a00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39e00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x39e00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3a200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3a200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3a600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3a600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3aa00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3aa00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ae00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ae00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3b200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3b200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3b600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3b600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ba00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ba00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3be00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3be00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3c200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3c200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3c600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3c600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ca00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ca00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ce00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ce00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3d200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3d200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3d600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3d600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3da00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3da00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3de00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3de00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3e200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3e200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3e600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3e600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ea00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ea00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ee00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3ee00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3f200~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3f200~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3f600~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3f600~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3fa00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3fa00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3fe00~200 2019-12-07T19:34:04.798+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 0x3fe00~200 = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=6144 2019-12-07T19:34:04.798+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:04.798+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# 2019-12-07T19:34:04.798+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 12288 2019-12-07T19:34:04.798+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# hinfo_key 2019-12-07T19:34:04.798+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# hinfo_key = 0 2019-12-07T19:34:04.798+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=121, attrs_read=0)) v2 -- 0x558c57573400 con 0x558c54c64000 2019-12-07T19:34:04.798+0000 7f8f04181700 10 osd.7 27 dequeue_op 0x558c54e789a0 finish 2019-12-07T19:34:04.798+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.802+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 98 ==== MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 263+0+0 (crc 0 0 0) 0x558c57573400 con 0x558c54c64000 2019-12-07T19:34:04.802+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e79760 prio 3 cost 0 latency 0.000016 epoch 27 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.802+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.802+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558c5500a000 2019-12-07T19:34:04.802+0000 7f8f08189700 10 osd.7 27 dequeue_op 0x558c54e79760 prio 3 cost 0 latency 0.000056 MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s4 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.802+0000 7f8f08189700 25 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf88d8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe750 space 0x558c54cf88d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf88d8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe7e0 space 0x558c54cf88d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf8a28 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe870 space 0x558c54cf8a28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54cf8a28 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe900 space 0x558c54cf8a28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315898 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fe990 space 0x558c57315898 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4a00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4a00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4e00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4e00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315898 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fea20 space 0x558c57315898 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5a00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5a00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5e00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5e00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573157b8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577feab0 space 0x558c573157b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6a00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6a00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6e00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6e00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573157b8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577feb40 space 0x558c573157b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7600~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7a00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7a00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7e00~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7e00~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8200~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573156d8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.802+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577febd0 space 0x558c573156d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8200~200 = 512 2019-12-07T19:34:04.802+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8600~200 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8a00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8a00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8e00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8e00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573156d8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fec60 space 0x558c573156d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9200~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9600~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9a00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9a00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9e00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9e00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573155f8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fecf0 space 0x558c573155f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa200~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa600~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xaa00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xaa00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xae00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xae00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xb200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573155f8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fed80 space 0x558c573155f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xb200~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xb600~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xb600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xba00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xba00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xbe00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xbe00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57567588 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fee10 space 0x558c57567588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc200~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc600~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xca00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xca00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xce00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xce00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xd200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57567588 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577feea0 space 0x558c57567588 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xd200~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xd600~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xd600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xda00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xda00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xde00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xde00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c574fa248 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fef30 space 0x558c574fa248 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe200~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe600~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xea00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xea00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xee00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xee00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xf200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c574fa248 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577fefc0 space 0x558c574fa248 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xf200~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xf600~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xf600~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xfa00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xfa00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xfe00~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.803+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xfe00~200 = 512 2019-12-07T19:34:04.803+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10200~200 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315518 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff050 space 0x558c57315518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315518 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff0e0 space 0x558c57315518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315438 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff170 space 0x558c57315438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315438 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff200 space 0x558c57315438 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315358 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff290 space 0x558c57315358 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315358 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff320 space 0x558c57315358 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315278 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff3b0 space 0x558c57315278 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315278 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff440 space 0x558c57315278 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17e00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18200~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315198 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff4d0 space 0x558c57315198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18200~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18600~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18600~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18a00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.804+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18a00~200 = 512 2019-12-07T19:34:04.804+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18e00~200 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18e00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315198 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff560 space 0x558c57315198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19a00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19a00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19e00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19e00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573150b8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff5f0 space 0x558c573150b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1aa00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1aa00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ae00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ae00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1b200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573150b8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff680 space 0x558c573150b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1b200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1b600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1b600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ba00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ba00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1be00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1be00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57314fd8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff710 space 0x558c57314fd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ca00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ca00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ce00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ce00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1d200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57314fd8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff7a0 space 0x558c57314fd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1d200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1d600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1d600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1da00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1da00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1de00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1de00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need {<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57314ef8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff830 space 0x558c57314ef8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ea00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ea00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ee00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ee00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1f200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57314ef8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff8c0 space 0x558c57314ef8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1f200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1f600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1f600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1fa00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1fa00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1fe00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1fe00~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20200~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need {<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573159e8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.805+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff950 space 0x558c573159e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20200~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20600~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.805+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20600~200 = 512 2019-12-07T19:34:04.805+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20a00~200 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20a00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20e00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c573159e8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.806+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ff9e0 space 0x558c573159e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21200~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21600~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21a00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21e00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need {<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315ac8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:04.806+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ffa70 space 0x558c57315ac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22200~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22600~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22a00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22e00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need {<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57315ac8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:04.806+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c577ffb00 space 0x558c57315ac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23200~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23600~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23a00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23e00~200 = 512 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2aa00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ae00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2b200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2b200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2b600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2b600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ba00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2be00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2be00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ca00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ce00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2d200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2d200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2d600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2d600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2da00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2da00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2de00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2de00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ea00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ee00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2f200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2f200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2f600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2f600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2fa00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2fe00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39a00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39a00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39e00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39e00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3aa00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ae00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3b200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3b200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3b600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3b600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ba00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3be00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3be00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ca00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ce00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3d200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3d200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3d600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3d600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3da00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3da00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3de00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3de00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ea00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ee00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3f200~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3f200~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3f600~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3f600~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3fa00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3fe00~200 2019-12-07T19:34:04.806+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 20 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=73728 2019-12-07T19:34:04.806+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:34:04.806+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.806+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 147456 2019-12-07T19:34:04.806+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key 2019-12-07T19:34:04.806+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key = 0 2019-12-07T19:34:04.806+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=122, attrs_read=0)) v2 -- 0x558c54d97b80 con 0x558c54c64000 2019-12-07T19:34:04.806+0000 7f8f08189700 10 osd.7 27 dequeue_op 0x558c54e79760 finish 2019-12-07T19:34:04.806+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.811+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 99 ==== MRecoveryReserve(1.9s4 RELEASE e27/27) v3 ==== 30+0+0 (crc 0 0 0) 0x558c576dc960 con 0x558c54c64000 2019-12-07T19:34:04.811+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.9s4 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.811+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.811+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.811+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.811+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.811+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558c5500a000 2019-12-07T19:34:04.811+0000 7f8f04181700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.811+0000 7f8f04181700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.812685+0000 1 0.000040 2019-12-07T19:34:04.811+0000 7f8f04181700 5 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.811+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:04.811+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.826+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.876+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.926+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.977+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.027+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.077+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.127+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.154+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:05.177+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.227+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.278+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.327+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.356+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:05.378+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.382+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c64800 0x558c54c70680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:05.382+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c64800 0x558c54c70680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:05.382+0000 7f8f24457700 10 osd.7 27 ms_handle_authentication new session 0x558c57405040 con 0x558c54c64800 entity client.admin addr 2019-12-07T19:34:05.382+0000 7f8f24457700 10 osd.7 27 ms_handle_authentication session 0x558c57405040 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:05.382+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2559427579 conn(0x558c54c64800 0x558c54c70680 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4609 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:05.382+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4609 172.21.15.131:0/2559427579 1 ==== command(tid 14: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c576dc960 con 0x558c54c64800 2019-12-07T19:34:05.382+0000 7f8f164bb700 20 osd.7 27 OSD::ms_dispatch: command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:05.382+0000 7f8f164bb700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:05.382+0000 7f8f164bb700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:05.382+0000 7f8f164bb700 20 osd.7 27 _dispatch 0x558c576dc960 command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:05.383+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2559427579 -- command_reply(tid 14: 0 ) v1 -- 0x558c54ce0fc0 con 0x558c54c64800 2019-12-07T19:34:05.398+0000 7f8f21451700 10 osd.7 27 tick 2019-12-07T19:34:05.398+0000 7f8f21451700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:05.398+0000 7f8f21451700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:05.398+0000 7f8f21451700 20 osd.7 27 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:05.405+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4609 172.21.15.131:0/2559427579 2 ==== command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c54d9e000 con 0x558c54c64800 2019-12-07T19:34:05.405+0000 7f8f164bb700 20 osd.7 27 OSD::ms_dispatch: command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:05.405+0000 7f8f164bb700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:05.405+0000 7f8f164bb700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:05.405+0000 7f8f164bb700 20 osd.7 27 _dispatch 0x558c54d9e000 command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:05.406+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2559427579 -- command_reply(tid 15: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce1180 con 0x558c54c64800 2019-12-07T19:34:05.409+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2559427579 conn(0x558c54c64800 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 64 2019-12-07T19:34:05.409+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2559427579 conn(0x558c54c64800 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:05.409+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2559427579 conn(0x558c54c64800 0x558c54c70680 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:05.409+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2559427579 conn(0x558c54c64800 0x558c54c70680 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:05.409+0000 7f8f164bb700 2 osd.7 27 ms_handle_reset con 0x558c54c64800 session 0x558c57405040 2019-12-07T19:34:05.428+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.452+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 11 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.910862610s delta_ub 0.022524611s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b5c00 con 0x558c54c18000 2019-12-07T19:34:05.452+0000 7f8f23c56700 20 osd.7 27 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021742854s,0.022524611s]) 2019-12-07T19:34:05.452+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.889119756s delta_ub 18446744073.687808762s) v5 -- 0x558c54d76200 con 0x558c54c18000 2019-12-07T19:34:05.452+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 11 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.910862610s delta_ub 0.022524611s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1e600 con 0x558c552d1400 2019-12-07T19:34:05.452+0000 7f8f24457700 20 osd.7 27 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021704896s,0.022524611s]) 2019-12-07T19:34:05.452+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.889157714s delta_ub 18446744073.687846720s) v5 -- 0x558c57569200 con 0x558c552d1400 2019-12-07T19:34:05.478+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.521+0000 7f8f1219d700 20 osd.7 27 reports for 0 queries 2019-12-07T19:34:05.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c579b4a80 con 0x558c5476d800 2019-12-07T19:34:05.521+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:05.521+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x469000/0x3f5000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:05.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(1 pgs tid 0 v 0) v2 -- 0x558c579b4e00 con 0x558c5476d800 2019-12-07T19:34:05.528+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.578+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.628+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 416973 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.652+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 60 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c54d97b80 con 0x558c54c5b400 2019-12-07T19:34:05.652+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.bs3 epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.652+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 61 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558c54d2f680 con 0x558c54c5b400 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 osd.7 27 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 osd.7 27 _dispatch 0x558c54d2f680 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.652+0000 7f8f1199c700 3 osd.7 27 handle_osd_map epochs [28,28], i have 27, src has [1,28] 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 osd.7 27 handle_osd_map got inc map for epoch 28 2019-12-07T19:34:05.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ea0e826:::osdmap.27:0# 0x0~0 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1671 size 0x1671 (5745) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567420 blob([0x26d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575673b0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567340 blob([!~1000,0x26e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558c575672d0 sbid 0x0)) need 0x1000~671 cache has 0x[1000~671] 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ea0e826:::osdmap.27:0# 0x0~1671 = 5745 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 osd.7 27 add_map_bl 27 5745 bytes 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 osd.7 27 got_full_map 28, nothing requested 2019-12-07T19:34:05.652+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 70 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (unknown 0 0 0) 0x558c576ba300 con 0x558c54c5bc00 2019-12-07T19:34:05.652+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.652+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 osd.7 27 handle_osd_map pg_num_history pg_num_history(e28 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {28={}} 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 osd.7 27 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e28 [1,28] lci=[13,28]) 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f7500 seq 19 2019-12-07T19:34:05.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:ccbb972f:::inc_osdmap.28:0# key 0x7f7fffffffffffffffccbb97'/!inc_osdmap.28!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:05.652+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 28, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:05.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1137 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~3aa 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc56 on front/back, now 0x0~1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7500 1 blobs 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x43e000~1000/1000,1000,0 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x43e000~1000] 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75dc0 blob([0x43e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74000 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.blob(0x558c54d75dc0) get_ref 0x0~3aa Blob(0x558c54d75dc0 blob([0x43e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74000 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.blob(0x558c54d75dc0) get_ref init 0x1000, 1000 2019-12-07T19:34:05.652+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c54d9b000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~3aa: 0x0~3aa Blob(0x558c54d75dc0 blob([0x43e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3aa) SharedBlob(0x558c54d74000 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 28, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d74018 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:05.652+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3aa 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e388) dirty_range mark inline shard dirty 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa = 0 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3058e826:::osdmap.28:0# key 0x7f7fffffffffffffff3058e8'&!osdmap.28!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:05.652+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3058e826:::osdmap.28:0# 0x0~1454 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1138 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3058e826:::osdmap.28:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~454 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7500 2 blobs 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:05.652+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x43f000~2000/1000,2000,0 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x43f000~2000] 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2a80 blob([0x43f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2af0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.blob(0x558c577f2a80) get_ref 0x0~1000 Blob(0x558c577f2a80 blob([0x43f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2af0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.blob(0x558c577f2a80) get_ref init 0x1000, 1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c577f2a80 blob([0x43f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c577f2af0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c577f2b08 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2b60 blob([!~1000,0x440000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2bd0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.blob(0x558c577f2b60) get_ref 0x1000~454 Blob(0x558c577f2b60 blob([!~1000,0x440000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c577f2bd0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.blob(0x558c577f2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558c577f2b60 blob([!~1000,0x440000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c577f2bd0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c577f2be8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:05.652+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1454 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e608) dirty_range mark inline shard dirty 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3058e826:::osdmap.28:0# 0x0~1454 = 0 2019-12-07T19:34:05.653+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7500 0 blobs 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:05.653+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:05.653+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:05.653+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7500 0 blobs 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7500 cost 23623 (4 ios * 4000 + 7623 bytes) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7500 onodes 0x558c53a91900,0x558c55276c80,0x558c5756e280,0x558c5756e500 shared_blobs 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e388) update #-1:ccbb972f:::inc_osdmap.28:0# 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e388) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:ccbb972f:::inc_osdmap.28:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e608) update #-1:3058e826:::osdmap.28:0# 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e608) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3058e826:::osdmap.28:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7500 allocated 0x[43e000~3000] released 0x[] 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 freelist allocate 0x43e000~3000 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 prepare 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f7500 2019-12-07T19:34:05.653+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 71 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558c57944240 con 0x558c54c5bc00 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 osd.7 27 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:05.653+0000 7f8f1199c700 20 osd.7 27 _dispatch 0x558c57944240 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.653+0000 7f8f1199c700 3 osd.7 27 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.653+0000 7f8f1199c700 10 osd.7 27 no new maps here, dropping 2019-12-07T19:34:05.653+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 62 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c54d97b80 con 0x558c54c5b400 2019-12-07T19:34:05.653+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.3s4 epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.653+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c5513e000 2019-12-07T19:34:05.653+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 28, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.653+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 63 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (unknown 0 0 0) 0x558c54e9aa00 con 0x558c54c5b400 2019-12-07T19:34:05.654+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.654+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {28=0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {28=0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.654+0000 7f8f0297e700 20 osd.7 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 28, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 100 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (crc 0 0 0) 0x558c54e9aa00 con 0x558c54c64000 2019-12-07T19:34:05.654+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.654+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 101 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54d97b80 con 0x558c54c64000 2019-12-07T19:34:05.654+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {28=0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.654+0000 7f8f24457700 15 osd.7 27 enqueue_peering_evt 1.7s4 epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.654+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.654+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {28=0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.654+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.654+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 28, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.654+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 102 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (crc 0 0 0) 0x558c57571680 con 0x558c54c64000 2019-12-07T19:34:05.654+0000 7f8f1199c700 20 osd.7 27 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.654+0000 7f8f1199c700 10 osd.7 27 do_waiters -- start 2019-12-07T19:34:05.654+0000 7f8f1199c700 10 osd.7 27 do_waiters -- finish 2019-12-07T19:34:05.654+0000 7f8f1199c700 20 osd.7 27 _dispatch 0x558c57571680 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.654+0000 7f8f1199c700 3 osd.7 27 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.654+0000 7f8f1199c700 10 osd.7 27 no new maps here, dropping 2019-12-07T19:34:05.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c54d5f000 2019-12-07T19:34:05.654+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 28, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.655+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 72 ==== pg_query2(1.6s2 1.6s2 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:05.655+0000 7f8f23c56700 15 osd.7 27 enqueue_peering_evt 1.6s2 epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.655+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c54d5c000 2019-12-07T19:34:05.655+0000 7f8f0197c700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 28, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.655+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 aio_wait 2019-12-07T19:34:05.655+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7500 2019-12-07T19:34:05.655+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 io_done 2019-12-07T19:34:05.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:05.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756e280 had 1 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756e500 had 1 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860398080 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998678) bluestore 86 GiB free (0.959953), bluefs_ratio 0.0399515 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6dc82~226c to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6dc82 0x6dc82~226c to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3dc82 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc82 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xeee and padding block with 0x112 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x6feee 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000728674s (0.000444888s flush + 0.000283786s kv commit) 2019-12-07T19:34:05.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7500> 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 kv_submitted 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7500 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f7500 osr 0x558c54ea4b40 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5754d7c0) _discard drop 16 0x1b000~1000 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5754d7c0) prepare_write seq 17 0x1b000~1000 crc cdd76f83 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5754d7c0) _discard drop 16 0x2000~1000 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c5754d7c0) prepare_write seq 17 0x2000~1000 crc c12d782e 2019-12-07T19:34:05.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 4 ==== osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 ==== 301+0+4 (crc 0 0 0) 0x558c57606b00 con 0x558c54c9f000 2019-12-07T19:34:05.657+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e78f20 prio 63 cost 4 latency 0.000026 epoch 28 osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:05.657+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) 2019-12-07T19:34:05.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 5 ==== osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 ==== 263+0+4 (crc 0 0 0) 0x558c57606dc0 con 0x558c54c9f000 2019-12-07T19:34:05.657+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e79b80 prio 63 cost 4 latency 0.000008 epoch 28 osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:05.657+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) 2019-12-07T19:34:05.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 6 ==== osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 ==== 263+0+4 (crc 0 0 0) 0x558c55230b00 con 0x558c54c9f000 2019-12-07T19:34:05.657+0000 7f8f24457700 15 osd.7 27 enqueue_op 0x558c54e78c60 prio 63 cost 4 latency 0.000009 epoch 28 osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:05.657+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7 27 _committed_osd_maps 28..28 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7 27 advance to epoch 28 (<= last 28 <= newest_map 28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 7 osd.7 28 consume_map version 28 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.0s3 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.9s4 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.2s4 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.bs3 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.3s4 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.cs0 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.4s0 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.ds2 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.6s2 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 identify_splits_and_merges 1.7s4 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 28 prune_pg_created 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:0.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:1.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:2.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:3.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:4.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.cs0 to_process <> waiting <> waiting_peering {28=0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:5.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:6.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7:7.consume_map 28 (was 27) 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.657+0000 7f8f141a1700 10 osd.7 28 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.0s3 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.9s4 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.2s4 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.bs3 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.3s4 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.4s0 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.ds2 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.6s2 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 15 osd.7 28 enqueue_peering_evt 1.7s4 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f141a1700 7 osd.7 28 activate_map version 28 2019-12-07T19:34:05.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 31 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1344+0+0 (unknown 0 0 0) 0x558c54e9aa00 con 0x558c54c19c00 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c54d5c000 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c5731a000 2019-12-07T19:34:05.657+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.657+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 27 -> 28 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 27 -> 28 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 27 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 27 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c5500a000 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c55141000 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] new interval newup [3,2,7,1,4,5] newacting [3,2,7,1,4,5] 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,5,7,3] 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 27 -> 28 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.658616+0000 1 0.000023 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.658618+0000 1 0.000027 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 27 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 27 -> 28 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.658629+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.658628+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 27 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.658638+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.658639+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.658670+0000 1 0.000027 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 28 2019-12-07T19:34:05.657+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.658682+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,5,7,3] 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.657+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.658692+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] new interval newup [3,2,7,1,4,5] newacting [3,2,7,1,4,5] 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.657+0000 7f8f03980700 15 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]p0(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]/[3,2147483647,7,1,2147483647,5]p3(0) r=2 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.657+0000 7f8f05984700 15 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'20 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [0,4,1,5,7,3](0) acting [0,4,1,2147483647,2147483647,3](0)) : acting set is too small 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [3,2,7,1,4,5](3) acting [3,2147483647,7,1,2147483647,5](3)) : acting set is too small 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2))) 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c54676310] 2019-12-07T19:34:05.657+0000 7f8f08189700 15 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c54ea3ce0] 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.094210890s + 0s) 2019-12-07T19:34:05.657+0000 7f8f0918b700 20 osd.7 28 agent_entry empty queue 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active pruub 44.040316297s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.040316297s (mnow 29.094215532s + 14.946100765s) 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active pruub 44.040316297s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active pruub 44.040316297s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,2147483647](1)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,2147483647](1) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.657+0000 7f8f03980700 1 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] start_peering_interval up [0,4,1,5,7,3] -> [0,4,1,5,7,3], acting [0,4,1,2147483647,2147483647,3] -> [0,4,1,5,7,3], acting_primary 0(0) -> 0, up_primary 0(0) -> 0, role -1 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 103 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54d97b80 con 0x558c54c64000 2019-12-07T19:34:05.657+0000 7f8f05984700 1 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active pruub 44.040316297s@ mbc={} ps=[2~1]] start_peering_interval up [3,2,7,1,4,5] -> [3,2,7,1,4,5], acting [3,2147483647,7,1,2147483647,5] -> [3,2,7,1,4,5], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 2 -> 2, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.657+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.9s4 epoch_sent: 28 epoch_requested: 28 MQuery 1.9s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.657+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.9s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4))) 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] release_backoffs [1:60000000::::0,1:70000000::::head) 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:05.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.9s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.041350232s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.041350232s (mnow 29.094279064s + 14.947071168s) 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.657+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 30 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558c575718c0 con 0x558c54c19c00 2019-12-07T19:34:05.657+0000 7f8f1199c700 20 osd.7 28 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.657+0000 7f8f1199c700 10 osd.7 28 do_waiters -- start 2019-12-07T19:34:05.657+0000 7f8f1199c700 10 osd.7 28 do_waiters -- finish 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.657+0000 7f8f1199c700 20 osd.7 28 _dispatch 0x558c575718c0 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.041350232s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.657+0000 7f8f1199c700 3 osd.7 28 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.657+0000 7f8f1199c700 10 osd.7 28 no new maps here, dropping 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.041350232s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.657+0000 7f8f08189700 1 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.041350232s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,3] -> [1,4,5,0,7,3], acting [1,4,5,0,7,2147483647] -> [1,4,5,0,7,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:05.657+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.657+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.658922+0000 1 0.000297 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.657+0000 7f8f03980700 1 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.658940+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7f8f03980700 5 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946100765s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.657+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.658960+0000 1 0.000318 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 28 send_incremental_map 27 -> 28 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~0 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~3aa size 0x3aa (938) 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75dc0 blob([0x43e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3aa) SharedBlob(0x558c54d74000 sbid 0x0)) need 0x0~3aa cache has 0x[0~3aa] 2019-12-07T19:34:05.657+0000 7f8f05984700 1 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.657+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa = 938 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 28 add_map_inc_bl 28 938 bytes 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(28..28 src has 1..28) v4 -- 0x558c576c0240 con 0x558c54c64400 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.658990+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7f8f03980700 10 osd.7 28 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.657+0000 7f8f05984700 5 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947071168s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.657+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 -- 0x558c57450000 con 0x558c54c64400 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.659014+0000 1 0.000327 2019-12-07T19:34:05.657+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c574ff800 seq 11 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.657+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.6s2 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:05.657+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.657+0000 7f8f08189700 1 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.657+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff800 cost 5576 (1 ios * 4000 + 1576 bytes) 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff800 onodes shared_blobs 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff800 allocated 0x[] released 0x[] 2019-12-07T19:34:05.657+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.659045+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 prepare 2019-12-07T19:34:05.657+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff800 2019-12-07T19:34:05.657+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 io_done 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 28 send_incremental_map 27 -> 28 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:05.657+0000 7f8f08189700 5 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.657+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.657+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(28..28 src has 1..28) v4 -- 0x558c579458c0 con 0x558c54c5bc00 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 28 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.657+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:28 sent:28 1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945925355s) 2->0 ([18,27] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e28/28) v1 -- 0x558c576bb800 con 0x558c54c5bc00 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 28 send_incremental_map 27 -> 28 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:05.657+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(28..28 src has 1..28) v4 -- 0x558c57570d80 con 0x558c54c64000 2019-12-07T19:34:05.657+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:28 sent:28 1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) 2->0 ([18,27] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e28/28) v1 -- 0x558c57450700 con 0x558c54c5bc00 2019-12-07T19:34:05.657+0000 7f8f08189700 10 osd.7 28 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.657+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:05.657+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5740ac00 seq 41 2019-12-07T19:34:05.657+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:28 sent:28 1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) 4->0 ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4)))) e28/28) v1 -- 0x558c576dee00 con 0x558c54c64000 2019-12-07T19:34:05.657+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:05.657+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:05.657+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:05.657+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:05.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:05.657+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:05.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d59200 seq 21 2019-12-07T19:34:05.657+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740ac00 cost 5576 (1 ios * 4000 + 1576 bytes) 2019-12-07T19:34:05.657+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740ac00 onodes shared_blobs 2019-12-07T19:34:05.657+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.657+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:05.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:05.657+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 prepare 2019-12-07T19:34:05.657+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740ac00 2019-12-07T19:34:05.657+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 io_done 2019-12-07T19:34:05.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:05.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:05.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:05.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:05.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:05.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c54d5c000 2019-12-07T19:34:05.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:05.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:05.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:05.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.657+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.9s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c5500a000 2019-12-07T19:34:05.657+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c5513e000 2019-12-07T19:34:05.657+0000 7f8f04181700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946904229s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.9s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.658+0000 7f8f04181700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 27 -> 28 2019-12-07T19:34:05.658+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 274 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1431 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:28 sent:28 1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) 4->0 ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4)))) e28/28) v1 -- 0x558c576de700 con 0x558c54c64000 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,0] 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2851 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4238 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.659263+0000 1 0.000018 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4261 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4261 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.659273+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x6feee~10a5 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x6feee 0x6feee~10a5 to file(ino 16 size 0x70000 mtime 2019-12-07T19:34:04.661243+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _allocate len 0xf93 from 0 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _allocate len 0xf93 from 1 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.659283+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd1a0000 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd1a0000~10000/10000,10000,acd1a0000 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x80000 to include allocated 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 29 (was clean) 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x3feee 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xeee 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf93 and padding block with 0x6d 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x80000 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,0] 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f07187700 15 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,2147483647](4)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,2147483647](4) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4))) 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:05.658+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active pruub 44.043602923s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.043602923s (mnow 29.094852473s + 14.948750450s) 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active pruub 44.043602923s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active pruub 44.043602923s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f07187700 1 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active pruub 44.043602923s@ mbc={} ps=[2~1]] start_peering_interval up [4,1,2,5,7,0] -> [4,1,2,5,7,0], acting [4,1,2,5,7,2147483647] -> [4,1,2,5,7,0], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c54d5f000 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 27 -> 28 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 27 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.659503+0000 1 0.000019 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.659518+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.659527+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948750450s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.659566+0000 1 0.000290 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x70f93 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.658+0000 7f8f07187700 1 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.659594+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.658+0000 7f8f05183700 15 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.658+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (29) on file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]), flushing log 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [1,4,3,0,7,2](1) acting [1,2147483647,2147483647,0,7,2](1)) : acting set is too small 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.3s4 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 29 len 0x49 crc 0x1034706f) 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x1c000~1000 to file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:05.658+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x1c000 0x1c000~1000 to file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1c000 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 28 send_incremental_map 27 -> 28 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.658+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(28..28 src has 1..28) v4 -- 0x558c576b2b40 con 0x558c54c5b400 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 28 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.041897808s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.041897808s (mnow 29.095103633s + 14.946794175s) 2019-12-07T19:34:05.658+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:28 sent:28 1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948602262s) 4->0 ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4)))) e28/28) v1 -- 0x558c574e4700 con 0x558c54c5b400 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.041897808s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.041897808s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x1d000 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:05.658+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:28 sent:28 1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) 4->0 ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4)))) e28/28) v1 -- 0x558c574e4000 con 0x558c54c5b400 2019-12-07T19:34:05.658+0000 7f8f05183700 1 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.041897808s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,3,0,7,2] -> [1,4,3,0,7,2], acting [1,2147483647,2147483647,0,7,2] -> [1,4,3,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5730a300 seq 26 2019-12-07T19:34:05.658+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:05.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:05.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:05.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a300 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:05.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a300 onodes shared_blobs 2019-12-07T19:34:05.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a300 allocated 0x[] released 0x[] 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 prepare 2019-12-07T19:34:05.658+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a300 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.658+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 io_done 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c54d9b000 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 27 -> 28 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.658+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.659806+0000 1 0.000023 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.659816+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946794175s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.659824+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.659832+0000 1 0.000304 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.658+0000 7f8f05183700 1 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.659859+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7f8f05183700 5 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:05.658+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000189 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.658+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.658+0000 7f8f07187700 15 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.658+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [4,0,2,7,1,5](4) acting [4,2147483647,2,2147483647,1,5](4)) : acting set is too small 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.7s4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 27 -> 28 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 29 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 27 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:05.658+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000703883s (0.000000106s flush + 0.000703777s kv commit) 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.658+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:28 sent:28 1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946632142s) 4->0 ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e28/28) v1 -- 0x558c574e7800 con 0x558c54c64000 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] new interval newup [7,5,4,1,2,3] newacting [7,5,4,1,2,3] 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.095380030s + 0s) 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.659957+0000 1 0.000023 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:28 sent:28 1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) 4->0 ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e28/28) v1 -- 0x558c574e7100 con 0x558c54c64000 2019-12-07T19:34:05.658+0000 7f8f07187700 1 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] start_peering_interval up [4,0,2,7,1,5] -> [4,0,2,7,1,5], acting [4,2147483647,2,2147483647,1,5] -> [4,0,2,7,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x70f93~50c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x70f93 0x70f93~50c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.659968+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x40f93 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf93 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x49f and padding block with 0xb61 2019-12-07T19:34:05.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:05.658+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.659978+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:05.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54dde000 seq 29 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.658+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:05.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde000 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:05.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde000 onodes shared_blobs 2019-12-07T19:34:05.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde000 allocated 0x[] released 0x[] 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff800,0x558c5740ac00,0x558c54d59200> 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 prepare 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.658+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde000 2019-12-07T19:34:05.658+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 io_done 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff800 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 finishing 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff800 onodes 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 done 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff800 [] 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] new interval newup [7,5,4,1,2,3] newacting [7,5,4,1,2,3] 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c54d5f000 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 kv_submitted 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740ac00 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 finishing 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740ac00 onodes 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740ac00 done 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740ac00 [] 2019-12-07T19:34:05.658+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.658+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]/[2147483647,5,4,1,2,3]p5(1) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'16 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f06986700 15 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]p7(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'16 mlcod 0'0 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.658+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c5500d000 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7149f 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 27 -> 28 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [7,5,4,1,2,3](7) acting [2147483647,5,4,1,2,3](5)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [7,5,4,1,2,3](7) acting [2147483647,5,4,1,2,3](5) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000136655s (0.000000124s flush + 0.000136531s kv commit) 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 27 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a300> 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 kv_submitted 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a300 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 finishing 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a300 onodes 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a300 done 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1))) 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a300 [] 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.660091+0000 1 0.000272 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7149f~522 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7149f 0x7149f~522 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] init_hb_stamps now [0x558c54677f10,0x558c5523a3f0,0x558c5523a230,0x558c5523a150,0x558c54ea3ce0] 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4149f 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x49f 2019-12-07T19:34:05.658+0000 7f8f07187700 1 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9c1 and padding block with 0x63f 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.095555694s + 0s) 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.660119+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.658+0000 7f8f07187700 5 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.658+0000 7f8f07187700 20 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 28 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:10.399358+0000, must = 0 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.658+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7f8f06986700 1 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'16 mlcod 0'0 active mbc={} ps=[2~1]] start_peering_interval up [7,5,4,1,2,3] -> [7,5,4,1,2,3], acting [2147483647,5,4,1,2,3] -> [7,5,4,1,2,3], acting_primary 5(1) -> 7, up_primary 7(0) -> 7, role -1 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.658+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.658+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.658+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] release_backoffs [1:30000000::::0,1:40000000::::head) 2019-12-07T19:34:05.658+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x719c1 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:28 sent:28 1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e28/28) v1 -- 0x558c54e9a300 con 0x558c54c5b400 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000104260s (0.000000076s flush + 0.000104184s kv commit) 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde000> 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 kv_submitted 2019-12-07T19:34:05.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde000 2019-12-07T19:34:05.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:05.658+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 finishing 2019-12-07T19:34:05.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde000 onodes 2019-12-07T19:34:05.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde000 done 2019-12-07T19:34:05.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:05.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde000 [] 2019-12-07T19:34:05.659+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.659+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:28 sent:28 1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e28/28) v1 -- 0x558c54e99c00 con 0x558c54c5b400 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.659+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:05.659+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730b800 seq 9 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.659+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:05.659+0000 7f8f06986700 15 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] publish_stats_to_osd 0:0 2019-12-07T19:34:05.659+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:05.659+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.659+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:05.659+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:05.659+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b800 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.659+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b800 onodes shared_blobs 2019-12-07T19:34:05.659+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b800 allocated 0x[] released 0x[] 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.659+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 prepare 2019-12-07T19:34:05.659+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b800 2019-12-07T19:34:05.659+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 io_done 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.659+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.659+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c5513e000 2019-12-07T19:34:05.659+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.659+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x719c1~522 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.659+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x719c1 0x719c1~522 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.659+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x419c1 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9c1 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.659+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xee3 and padding block with 0x11d 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:05.659+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.660313+0000 1 0.000339 2019-12-07T19:34:05.659+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.659+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.659+0000 7f8f06986700 1 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] state: transitioning to Primary 2019-12-07T19:34:05.659+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.660340+0000 0 0.000000 2019-12-07T19:34:05.659+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started/Primary 2019-12-07T19:34:05.659+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] enter Started/Primary/Peering 2019-12-07T19:34:05.659+0000 7f8f06986700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] build_prior all_probe 1(3),2(4),3(5),4(2),5(1),6(0),7(0) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] build_prior maybe_rw interval:26, acting: 1(3),2(4),3(5),4(2),5(1) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] build_prior final: probe 1(3),2(4),3(5),4(2),5(1),7(0) down 6 blocked_by {} 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] up_thru 25 < same_since 28, must notify monitor 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: querying info from osd.1(3) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: querying info from osd.2(4) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: querying info from osd.3(5) 2019-12-07T19:34:05.659+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: querying info from osd.4(2) 2019-12-07T19:34:05.659+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: querying info from osd.5(1) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] check_prior_readable_down_osds prior_readable_down_osds osd.6 is dead as of epoch 28 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.659+0000 7f8f06986700 15 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] publish_stats_to_osd 27:58 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.659+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got osd.3(5) 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] update_history prior_readable_until_ub 44.052406331s (mnow 29.095954216s + 14.956452115s) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:24.612947+0000, must = 0 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:05.659+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] state: last_epoch_started moved forward, rebuilding prior 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] build_prior all_probe 1(3),2(4),3(5),4(2),5(1),6(0),7(0) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] build_prior maybe_rw interval:26, acting: 1(3),2(4),3(5),4(2),5(1) 2019-12-07T19:34:05.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] build_prior final: probe 1(3),2(4),3(5),4(2),5(1),7(0) down 6 blocked_by {} 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] up_thru 25 < same_since 28, must notify monitor 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@6 mbc={} ps=[2~1]] state: already requested info from osd.1(3) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@6 mbc={} ps=[2~1]] state: already requested info from osd.2(4) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@6 mbc={} ps=[2~1]] state: have osd.3(5) info 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@6 mbc={} ps=[2~1]] state: already requested info from osd.4(2) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@6 mbc={} ps=[2~1]] state: already requested info from osd.5(1) 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@6 mbc={} ps=[2~1]] check_prior_readable_down_osds prior_readable_down_osds osd.6 is dead as of epoch 28 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering pruub 44.052406331s@ mbc={} ps=[2~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.659+0000 7f8f06986700 15 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] publish_stats_to_osd 28:59 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.659+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.cs3 1.cs3 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558c5793b680 con 0x558c54c64000 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 28 send_incremental_map 25 -> 28 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(26..28 src has 1..28) v4 -- 0x558c57570480 con 0x558c54c19800 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 28 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 25 -> 28 (shared) 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.cs4 1.cs4 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558c5793b900 con 0x558c54c19800 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.cs5 1.cs5 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558c5793bb80 con 0x558c54c5bc00 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.cs2 1.cs2 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558c57977180 con 0x558c54c5b400 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 28 send_incremental_map 27 -> 28 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(28..28 src has 1..28) v4 -- 0x558c575706c0 con 0x558c54c19c00 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 28 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.cs1 1.cs1 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558c57977400 con 0x558c54c19c00 2019-12-07T19:34:05.659+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea0f00 seq 13 2019-12-07T19:34:05.659+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:05.659+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:05.659+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:05.659+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:05.659+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 6530 (1 ios * 4000 + 2530 bytes) 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.659+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:05.659+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:05.659+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 28 queue_want_up_thru now 28 (was 25), currently 25 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 28 send_alive up_thru currently 25 want 28 2019-12-07T19:34:05.659+0000 7f8f06986700 10 osd.7 28 send_alive want 28 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_alive(want up_thru 28 have 28) v1 -- 0x558c576dde00 con 0x558c5476d000 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.659+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c54d9b000 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got osd.4(2) 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2019-12-07T19:34:05.659+0000 7f8f0317f700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.659+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:11.744919+0000, must = 0 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got osd.1(3) 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:10.174806+0000, must = 0 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c548ab000 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 27 -> 28 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 27 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:05.659+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1]] state: Active advmap 2019-12-07T19:34:05.659+0000 7f8f1fbbe700 10 osd.7 28 tick_without_osd_lock 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] on_active_advmap new removed_snaps [4~1], snap_trimq now [4~1] 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.659+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] agent_stop 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] activate_map 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] Active: kicking snap trim 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:05.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] enter Trimming 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] enter Trimming/WaitReservation 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.659+0000 7f8f0297e700 15 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] publish_stats_to_osd 26:26 2019-12-07T19:34:05.659+0000 7f8f0297e700 7 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.659+0000 7f8f0297e700 15 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] publish_stats_to_osd 27:27 2019-12-07T19:34:05.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 28 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.660+0000 7f8f0b990700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0b990700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] SnapTrimmer state: NotTrimming: trimming 4 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) queued 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.660+0000 7f8f0b990700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] exit Trimming/WaitReservation 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.660+0000 7f8f0b990700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.660+0000 7f8f0b990700 15 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] publish_stats_to_osd 28:28 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got osd.5(1) 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 2019-12-07T19:34:05.660+0000 7f8f0b990700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:24.574785+0000, must = 0 2019-12-07T19:34:05.660+0000 7f8f0b990700 10 osd.7 28 queueing pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[4~1] ps=[2~1]] for snaptrim 2019-12-07T19:34:05.660+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=28) prio 5 cost 1048576 e28) 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:05.660+0000 7f8f0b990700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.660+0000 7f8f0b990700 15 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 28:29 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) queued 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) pg 0x558c57676000 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 dequeue_op 0x558c54e78f20 prio 63 cost 4 latency 0.003121 osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 21 -> 28 (as per caller) 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) queued 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) pg 0x558c57676000 2019-12-07T19:34:05.660+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 32 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948655453s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1344+0+0 (unknown 0 0 0) 0x558c54e99c00 con 0x558c54c19c00 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 dequeue_op 0x558c54e79b80 prio 63 cost 4 latency 0.003127 osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=28) prio 5 cost 1048576 e28) queued 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=28) prio 5 cost 1048576 e28) pg 0x558c548ab000 2019-12-07T19:34:05.660+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948655453s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[4~1] ps=[2~1]] snap_trimmer posting 2019-12-07T19:34:05.660+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948655453s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 4 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:05.660+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 64 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (unknown 0 0 0) 0x558c54e99c00 con 0x558c54c5b400 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54ce5640) upper_bound after SNA_1_0000000000000004_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.4' 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54ce5640) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..' 2019-12-07T19:34:05.660+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000004_.0_0000000000000001.4) returns 0 SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55238fd0 in raw 0x558c55238fd0 len 70 nref 1) ) 2019-12-07T19:34:05.660+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1]] SnapTrimmer state: adding snap 4 to purged_snaps 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1,4~1]] SnapTrimmer state: purged_snaps now [2~1,4~1], snap_trimq now [] 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54ea0300 seq 11 2019-12-07T19:34:05.660+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes shared_blobs 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1,4~1]] share_pg_info 2019-12-07T19:34:05.660+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 -- 0x558c576bb100 con 0x558c54c64400 2019-12-07T19:34:05.660+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 -- 0x558c576e0300 con 0x558c54c64000 2019-12-07T19:34:05.660+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 -- 0x558c576e0a00 con 0x558c54c19800 2019-12-07T19:34:05.660+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 -- 0x558c54e98e00 con 0x558c54c5b400 2019-12-07T19:34:05.660+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 -- 0x558c57452a00 con 0x558c54c19c00 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1,4~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[2~1,4~1]] exit Trimming 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] prepare_stats_for_publish reporting purged_snaps [2~1,4~1] 2019-12-07T19:34:05.660+0000 7f8f0297e700 15 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] publish_stats_to_osd 28:30 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] enter NotTrimming 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] snap_trimmer complete 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948655453s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e28) pg 0x558c57676000 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 dequeue_op 0x558c54e78c60 prio 63 cost 4 latency 0.003394 osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948655453s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 5(1) notify: (query:28 sent:28 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948655453s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got dup osd.5(1) info 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948655453s), identical to ours 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.660+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.660+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 4(2) notify: (query:28 sent:28 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.660+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got dup osd.4(2) info 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948932841s), identical to ours 2019-12-07T19:34:05.660+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:05.660+0000 7f8f06986700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 104 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947989491s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (crc 0 0 0) 0x558c57452a00 con 0x558c54c64000 2019-12-07T19:34:05.661+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947989491s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.661+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947989491s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.661+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.661+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947989491s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.661+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.661+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947989491s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.661+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 105 ==== MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=133, reqid=osd.1.0:133, at_version=28'10, trim_to=0'0, roll_forward_to=24'8)) v2 ==== 2656+0+0 (crc 0 0 0) 0x558c575c0000 con 0x558c54c64000 2019-12-07T19:34:05.661+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 1(3) notify: (query:28 sent:28 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947989491s) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.661+0000 7f8f24457700 15 osd.7 28 enqueue_op 0x558c54e79080 prio 127 cost 0 latency 0.000018 epoch 28 MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=133, reqid=osd.1.0:133, at_version=28'10, trim_to=0'0, roll_forward_to=24'8)) v2 2019-12-07T19:34:05.661+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=133, reqid=osd.1.0:133, at_version=28'10, trim_to=0'0, roll_forward_to=24'8)) v2) prio 127 cost 0 e28) 2019-12-07T19:34:05.661+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got dup osd.1(3) info 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947989491s), identical to ours 2019-12-07T19:34:05.661+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.661+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.661+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=133, reqid=osd.1.0:133, at_version=28'10, trim_to=0'0, roll_forward_to=24'8)) v2) prio 127 cost 0 e28) queued 2019-12-07T19:34:05.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=133, reqid=osd.1.0:133, at_version=28'10, trim_to=0'0, roll_forward_to=24'8)) v2) prio 127 cost 0 e28) pg 0x558c5500d000 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 28 dequeue_op 0x558c54e79080 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=133, reqid=osd.1.0:133, at_version=28'10, trim_to=0'0, roll_forward_to=24'8)) v2 pg pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=133, reqid=osd.1.0:133, at_version=28'10, trim_to=0'0, roll_forward_to=24'8)) v2 2019-12-07T19:34:05.661+0000 7f8f04982700 20 snap_mapper.remove_oid 1:0064fe1d:::smithi13140796-44:4 2019-12-07T19:34:05.661+0000 7f8f04982700 20 snap_mapper._remove_oid 1:0064fe1d:::smithi13140796-44:4 2019-12-07T19:34:05.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:05.661+0000 7f8f04982700 20 snap_mapper.get_snaps 1:0064fe1d:::smithi13140796-44:4 4 2019-12-07T19:34:05.661+0000 7f8f04982700 20 snap_mapper.clear_snaps 1:0064fe1d:::smithi13140796-44:4 2019-12-07T19:34:05.661+0000 7f8f04982700 20 snap_mapper.clear_snaps rm OBJ_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:05.661+0000 7f8f04982700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 24'8 (0'0,24'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] append_log log((0'0,24'8], crt=24'8) [28'9 (24'5) delete 1:64fe1d00:::smithi13140796-44:4 by unknown.0.0:0 2019-12-07T19:34:05.659723+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,28'10 (24'8) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:05.659723+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'9 (0'0,28'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'8 crt=24'8 lcod 24'8 active mbc={} ps=[2~1]] add_log_entry 28'9 (24'5) delete 1:64fe1d00:::smithi13140796-44:4 by unknown.0.0:0 2019-12-07T19:34:05.659723+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'8 crt=24'8 lcod 24'8 active mbc={} ps=[2~1]] add_log_entry 28'10 (24'8) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:05.659723+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'8 crt=24'8 lcod 24'8 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'8 crt=24'8 lcod 24'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:05.661+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:05.661+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 28'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d58000 seq 18 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:0064fe1d:::smithi13140796-44:4#9 key 0x8380000000000000010064fe1d21736d'ithi13140796-44!='0x00000000000000040000000000000009'o' 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) r -2 v.len 0 2019-12-07T19:34:05.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# -> 3#1:0064fe1d:::smithi13140796-44:4#9 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4# -> 3#1:0064fe1d:::smithi13140796-44:4#9 = 0 2019-12-07T19:34:05.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:05.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:05.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.0062F78B.4.smithi13140796-44..' <- OBJ_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:05.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.3_0000000000000001.0062F78B.4.smithi13140796-44..' <- SNA_1_0000000000000004_.3_0000000000000001.0062F78B.4.smithi13140796-44.. 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:05.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000028.00000000000000000009' <- 0000000028.00000000000000000009 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000028.00000000000000000010' <- 0000000028.00000000000000000010 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 7549 (1 ios * 4000 + 3549 bytes) 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes 0x558c54e45b80,0x558c575bb900 shared_blobs 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) update 3#1:0064fe1d:::smithi13140796-44:4#9 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:4#9 is 1039 (412 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 1112 (485 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:34:05.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:34:05.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=24'8 crt=24'8 lcod 24'8 active mbc={} ps=[2~1]] op_applied version 28'10 2019-12-07T19:34:05.661+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f04982700 10 osd.7 28 dequeue_op 0x558c54e79080 finish 2019-12-07T19:34:05.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x71ee3 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fbbf000/0x40000000/0x1680000000, data 0x48514d/0x43f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002660304s (0.000000099s flush + 0.002660205s kv commit) 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b800> 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 kv_submitted 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b800 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 finishing 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:03.373892+0000 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b800 onodes 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 done 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b800 [] 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2305 < min_flush_size 524288 2019-12-07T19:34:05.661+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373810+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3439 < min_flush_size 524288 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 20 osd.7 28 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 20 osd.7 28 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8059 < min_flush_size 524288 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45b80 had 1 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57455180 had 1 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8082 < min_flush_size 524288 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8082 < min_flush_size 524288 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x71ee3~1f92 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x71ee3 0x71ee3~1f92 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x41ee3 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xee3 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe75 and padding block with 0x18b 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 20 osd.7 28 scrub_load_below_threshold loadavg per cpu 0.0575 < max 5 = yes 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 20 osd.7 28 sched_scrub load_is_low=1 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 10 osd.7 28 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:05.662961+0000 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 20 osd.7 28 sched_scrub done 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 10 osd.7 28 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 20 osd.7 28 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:05.661+0000 7f8f1fbbe700 10 osd.7 28 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:05.661+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x73e75 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000201600s (0.000000105s flush + 0.000201495s kv commit) 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00,0x558c54ea0300,0x558c54d58000> 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:05.661+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:34:05.661+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:34:05.661+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 0x558c54e45b80,0x558c575bb900 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:34:05.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.661+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 28/18 ECSubWriteReply(tid=133, last_complete=28'10, committed=1, applied=1)) v2 -- 0x558c57568200 con 0x558c54c64000 2019-12-07T19:34:05.661+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 27 ==== pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (crc 0 0 0) 0x558c5793bb80 con 0x558c54c64400 2019-12-07T19:34:05.662+0000 7f8f23c56700 15 osd.7 28 enqueue_peering_evt 1.2s4 epoch_sent: 28 epoch_requested: 28 MQuery 1.2s4 from 0(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.662+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.2s4 from 0(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.662+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.2s4 from 0(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.2s4 from 0(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558c5731a000 2019-12-07T19:34:05.662+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.2s4 from 0(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.662+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 28 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (crc 0 0 0) 0x558c576c0240 con 0x558c54c64400 2019-12-07T19:34:05.662+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s 2019-12-07T19:34:05.662+0000 7f8f1199c700 20 osd.7 28 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7f8f1199c700 10 osd.7 28 do_waiters -- start 2019-12-07T19:34:05.662+0000 7f8f1199c700 10 osd.7 28 do_waiters -- finish 2019-12-07T19:34:05.662+0000 7f8f1199c700 20 osd.7 28 _dispatch 0x558c576c0240 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7f8f03980700 20 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] update_history prior_readable_until_ub 44.051723552s (mnow 29.098891509s + 14.952832043s) 2019-12-07T19:34:05.662+0000 7f8f1199c700 3 osd.7 28 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.662+0000 7f8f1199c700 10 osd.7 28 no new maps here, dropping 2019-12-07T19:34:05.662+0000 7f8f03980700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.662+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 -- 0x558c576baa00 con 0x558c54c64400 2019-12-07T19:34:05.662+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:05.662+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54d59b00 seq 12 2019-12-07T19:34:05.662+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:05.662+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:05.662+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:05.662+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:05.662+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:05.662+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.662+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:05.662+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:05.662+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:05.662+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.662+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1152 < min_flush_size 524288 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:05.662+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1175 < min_flush_size 524288 2019-12-07T19:34:05.662+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1175 < min_flush_size 524288 2019-12-07T19:34:05.662+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.662+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x73e75~497 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.662+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x73e75 0x73e75~497 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x43e75 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe75 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.662+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x30c and padding block with 0xcf4 2019-12-07T19:34:05.662+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 38 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (crc 0 0 0) 0x558c548a3800 con 0x558c54c19800 2019-12-07T19:34:05.662+0000 7f8f24c58700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.662+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 39 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947506058s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (crc 0 0 0) 0x558c576e0a00 con 0x558c54c19800 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7f8f24c58700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947506058s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947506058s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got osd.2(4) 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 2019-12-07T19:34:05.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947506058s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:34:58.167300+0000, must = 0 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:05.662+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 40 ==== osd_map(26..28 src has 1..28) v4 ==== 2151+0+0 (crc 0 0 0) 0x558c57570480 con 0x558c54c19800 2019-12-07T19:34:05.662+0000 7f8f1199c700 20 osd.7 28 OSD::ms_dispatch: osd_map(26..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7f8f1199c700 10 osd.7 28 do_waiters -- start 2019-12-07T19:34:05.662+0000 7f8f1199c700 10 osd.7 28 do_waiters -- finish 2019-12-07T19:34:05.662+0000 7f8f1199c700 20 osd.7 28 _dispatch 0x558c57570480 osd_map(26..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.662+0000 7f8f1199c700 3 osd.7 28 handle_osd_map epochs [26,28], i have 28, src has [1,28] 2019-12-07T19:34:05.662+0000 7f8f1199c700 10 osd.7 28 no new maps here, dropping 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:05.662+0000 7f8f0297e700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:05.664075+0000 9 0.000489 2019-12-07T19:34:05.662+0000 7f8f0297e700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_acting all_info osd.1(3) 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955107098s) 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_acting all_info osd.2(4) 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_acting all_info osd.3(5) 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_acting all_info osd.4(2) 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_acting all_info osd.5(1) 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_acting all_info osd.7(0) 1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:05.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 5(1) For position 2: selecting up[i]: 4(2) For position 3: selecting up[i]: 1(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:05.662+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_async_recovery_ec result want=[7,5,4,1,2,3] async_recovery= 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] acting_recovery_backfill is 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] choose_acting want=[7,5,4,1,2,3] backfill_targets= async_recovery_targets= 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: requesting log from osd.1(3) 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.663+0000 7f8f0297e700 15 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] publish_stats_to_osd 28:60 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:05.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.cs3 1.cs3 query(log 27'16 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 epoch_sent 28) e28/28) v1 -- 0x558c54e45680 con 0x558c54c64000 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.663+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947506058s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.663+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 2(4) notify: (query:28 sent:28 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947506058s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.663+0000 7f8f06986700 7 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.2(4) 2019-12-07T19:34:05.663+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got dup osd.2(4) info 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947506058s), identical to ours 2019-12-07T19:34:05.663+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:05.663+0000 7f8f06986700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.663+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.663+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 106 ==== PGlog(1.cs0 log log((27'16,27'16], crt=27'16) pi ([0,0] all_participants= intervals=) e28/28) v6 ==== 2203+0+0 (crc 0 0 0) 0x558c575c0800 con 0x558c54c64000 2019-12-07T19:34:05.663+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MLogRec from 1(3) log log((27'16,27'16], crt=27'16) pi ([0,0] all_participants= intervals=) +create_info 2019-12-07T19:34:05.663+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MLogRec from 1(3) log log((27'16,27'16], crt=27'16) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MLogRec from 1(3) log log((27'16,27'16], crt=27'16) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MLogRec from 1(3) log log((27'16,27'16], crt=27'16) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MLogRec from 1(3) log log((27'16,27'16], crt=27'16) pi ([0,0] all_participants= intervals=) +create_info 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: GetLog: received master log from osd.1(3) 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: leaving GetLog 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: processing master log 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] proc_master_log for osd.1(3): log((27'16,27'16], crt=27'16) missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 merge_log log((27'16,27'16], crt=27'16) from osd.1(3) into log((0'0,27'16], crt=27'16) 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 merge_log copying olog dups to log 21'1 to 27'16 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 merge_log removed dups overlapping log entries (0'0,27'16] 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 merge_log result log((0'0,27'16], crt=27'16) missing(0 may_include_deletes = 1) changed=1 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] peer osd.1(3) now 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:23.739266+0000, must = 0 2019-12-07T19:34:05.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:05.664765+0000 2 0.000233 2019-12-07T19:34:05.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetMissing 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: osd.1(3) has no missing, identical log 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: osd.2(4) has no missing, identical log 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: osd.3(5) has no missing, identical log 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: osd.4(2) has no missing, identical log 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: osd.5(1) has no missing, identical log 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: still need up_thru update before going active 2019-12-07T19:34:05.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetMissing 2019-12-07T19:34:05.664795+0000 0 0.000000 2019-12-07T19:34:05.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:05.663+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d21b00 seq 14 2019-12-07T19:34:05.663+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore.onode(0x558c5756f680).flush flush done 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x0000000000000456'.dup_0000000000.00000000000000000000' end: 0x0000000000000456'.dup_4294967295.18446744073709551615' 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:05.663+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5754 (1 ios * 4000 + 1754 bytes) 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes shared_blobs 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:05.663+0000 7f8f0297e700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.663+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 41 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c576e0a00 con 0x558c54c19800 2019-12-07T19:34:05.663+0000 7f8f24c58700 15 osd.7 28 enqueue_peering_evt 1.ds2 epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:05.663+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e28) 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e28) pg 0x558c55141000 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54ea1800 seq 17 2019-12-07T19:34:05.663+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1800 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1800 onodes shared_blobs 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1800 allocated 0x[] released 0x[] 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 prepare 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1800 2019-12-07T19:34:05.663+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 io_done 2019-12-07T19:34:05.663+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7430c 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002156224s (0.000000115s flush + 0.002156109s kv commit) 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00> 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1454 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2588 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.664+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 73 ==== pg_notify2(1.cs0 1.cs0 (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.944302151s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e28/28) v1 ==== 1333+0+0 (unknown 0 0 0) 0x558c576baa00 con 0x558c54c5bc00 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2611 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2611 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7430c~a33 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7430c 0x7430c~a33 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4430c 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x30c 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.664+0000 7f8f23c56700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.944302151s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd3f and padding block with 0x2c1 2019-12-07T19:34:05.664+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.944302151s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.664+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.664+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.944302151s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.664+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.664+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.944302151s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:05.664+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.cs0 from 3(5) notify: (query:28 sent:28 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.944302151s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.664+0000 7f8f06986700 7 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.3(5) 2019-12-07T19:34:05.664+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] got dup osd.3(5) info 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.944302151s), identical to ours 2019-12-07T19:34:05.664+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:05.664+0000 7f8f06986700 10 osd.7 28 queue_want_up_thru want 28 <= queued 28, currently 25 2019-12-07T19:34:05.664+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x74d3f 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000115902s (0.000000083s flush + 0.000115819s kv commit) 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00,0x558c54ea1800> 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.664+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 kv_submitted 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1800 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 finishing 2019-12-07T19:34:05.664+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1800 onodes 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1800 done 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1800 [] 2019-12-07T19:34:05.664+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.664+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.665+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 107 ==== MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=134, reqid=osd.1.0:133, at_version=0'0, trim_to=0'0, roll_forward_to=28'10)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c575c1800 con 0x558c54c64000 2019-12-07T19:34:05.665+0000 7f8f24457700 15 osd.7 28 enqueue_op 0x558c54e79600 prio 127 cost 0 latency 0.000020 epoch 28 MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=134, reqid=osd.1.0:133, at_version=0'0, trim_to=0'0, roll_forward_to=28'10)) v2 2019-12-07T19:34:05.665+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=134, reqid=osd.1.0:133, at_version=0'0, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e28) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=134, reqid=osd.1.0:133, at_version=0'0, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e28) queued 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=134, reqid=osd.1.0:133, at_version=0'0, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e28) pg 0x558c5500d000 2019-12-07T19:34:05.665+0000 7f8f04982700 10 osd.7 28 dequeue_op 0x558c54e79600 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=134, reqid=osd.1.0:133, at_version=0'0, trim_to=0'0, roll_forward_to=28'10)) v2 pg pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:05.665+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.0s3 28/18 ECSubWrite(tid=134, reqid=osd.1.0:133, at_version=0'0, trim_to=0'0, roll_forward_to=28'10)) v2 2019-12-07T19:34:05.665+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=24'8 active mbc={} ps=[2~1]] append_log log((0'0,28'10], crt=24'8) [] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1]] rollforward: entry=28'9 (24'5) delete 1:64fe1d00:::smithi13140796-44:4 by unknown.0.0:0 2019-12-07T19:34:05.659723+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1]] rollforward: entry=28'10 (24'8) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:05.659723+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:05.665+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:05.665+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:05.665+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:05.665+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:05.665+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d58c00 seq 19 2019-12-07T19:34:05.665+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4#9 onode 0x558c54e45b80 txc 0x558c54d58c00 2019-12-07T19:34:05.665+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4#9 0x0 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c5742b810) put_ref 0x0~2000 Blob(0x558c5742b810 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c5742b8f0) put_ref 0x0~2000 Blob(0x558c5742b8f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c574f1c00) put_ref 0x0~2000 Blob(0x558c574f1c00 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c5742b960) put_ref 0x0~2000 Blob(0x558c5742b960 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c5742b7a0) put_ref 0x0~2000 Blob(0x558c5742b7a0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c5742b730) put_ref 0x0~2000 Blob(0x558c5742b730 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c57538770) put_ref 0x0~2000 Blob(0x558c57538770 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c54ea3260) put_ref 0x0~2000 Blob(0x558c54ea3260 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c54677490) put_ref 0x0~2000 Blob(0x558c54677490 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c54677340) put_ref 0x0~2000 Blob(0x558c54677340 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c54676000) put_ref 0x0~2000 Blob(0x558c54676000 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c546761c0) put_ref 0x0~2000 Blob(0x558c546761c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c575bd730) put_ref 0x0~2000 Blob(0x558c575bd730 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c575bdab0) put_ref 0x0~2000 Blob(0x558c575bdab0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c55279570) put_ref 0x0~2000 Blob(0x558c55279570 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c55278930) put_ref 0x0~2000 Blob(0x558c55278930 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c55278e00) put_ref 0x0~2000 Blob(0x558c55278e00 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c574f1d50) put_ref 0x0~2000 Blob(0x558c574f1d50 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c5523a9a0) put_ref 0x0~2000 Blob(0x558c5523a9a0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c54ea3570) put_ref 0x0~2000 Blob(0x558c54ea3570 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c552652d0) put_ref 0x0~2000 Blob(0x558c552652d0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c55264690) put_ref 0x0~2000 Blob(0x558c55264690 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.blob(0x558c55264150) put_ref 0x0~1000 Blob(0x558c55264150 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore.extentmap(0x558c54e45c88) dirty_range mark inline shard dirty 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c5742b810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1cf000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c5742b8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d1000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c574f1c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d3000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c5742b960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d5000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c5742b7a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d7000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c5742b730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d9000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57538770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1db000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c54ea3260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1dd000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c54677490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1df000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c54677340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e1000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c54676000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e3000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c546761c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e5000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c575bd730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e7000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c575bdab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e9000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c55279570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1eb000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c55278930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1ed000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c55278e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1ef000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c574f1d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f1000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c5523a9a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f3000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c54ea3570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f5000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c552652d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f7000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c55264690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f9000~2000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~1000: 0x0~1000 Blob(0x558c55264150 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1fb000~1000] 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:05.665+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:4#9 = 0 2019-12-07T19:34:05.665+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:05.665+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:05.665+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=1))) 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.665+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:34:05.665+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:34:05.665+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.665+0000 7f8f04982700 10 osd.7 28 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1773 < min_flush_size 524288 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45b80 had 1 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1796 < min_flush_size 524288 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1796 < min_flush_size 524288 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x74d3f~704 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x74d3f 0x74d3f~704 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x44d3f 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd3f 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x443 and padding block with 0xbbd 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x75443 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.665+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127849s (0.000000081s flush + 0.000127768s kv commit) 2019-12-07T19:34:05.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:34:05.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:05.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:34:05.665+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.665+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.665+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 28/18 ECSubWriteReply(tid=134, last_complete=28'10, committed=1, applied=1)) v2 -- 0x558c57568400 con 0x558c54c64000 2019-12-07T19:34:05.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.665+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.666+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 108 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c576e0300 con 0x558c54c64000 2019-12-07T19:34:05.666+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.0s3 epoch_sent: 28 epoch_requested: 28 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:05.666+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e28) 2019-12-07T19:34:05.666+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.666+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.666+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.666+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e28) pg 0x558c5500d000 2019-12-07T19:34:05.666+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:05.666+0000 7f8f04982700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:05.666+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:05.666+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:05.666+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54ea0600 seq 20 2019-12-07T19:34:05.666+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:05.666+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:05.666+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:05.666+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:05.666+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:05.666+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0600 onodes shared_blobs 2019-12-07T19:34:05.666+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0600 allocated 0x[] released 0x[] 2019-12-07T19:34:05.666+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 prepare 2019-12-07T19:34:05.666+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0600 2019-12-07T19:34:05.666+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 io_done 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.666+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x75443~46d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x75443 0x75443~46d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x45443 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x443 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8b0 and padding block with 0x750 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x758b0 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:05.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000100758s (0.000000079s flush + 0.000100679s kv commit) 2019-12-07T19:34:05.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0600> 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 kv_submitted 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0600 2019-12-07T19:34:05.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 finishing 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0600 onodes 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0600 done 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:05.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0600 [] 2019-12-07T19:34:05.666+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:05.666+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:05.666+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.678+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.728+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.779+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.829+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.879+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.929+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.979+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.029+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.079+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.129+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.179+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.230+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.280+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.330+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.342+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 8 ==== osd_ping(ping e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.815119378s delta_ub 0.036494033s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c552d1800 2019-12-07T19:34:06.342+0000 7f8f24c58700 20 osd.7 28 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036036494s,0.036494033s]) 2019-12-07T19:34:06.342+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.779082884s delta_ub 18446744073.673515122s) v5 -- 0x558c54d77e00 con 0x558c552d1800 2019-12-07T19:34:06.342+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 8 ==== osd_ping(ping e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.815119378s delta_ub 0.036494033s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57568400 con 0x558c552d1c00 2019-12-07T19:34:06.342+0000 7f8f24457700 20 osd.7 28 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036017393s,0.036494033s]) 2019-12-07T19:34:06.342+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.779101985s delta_ub 18446744073.673534223s) v5 -- 0x558c57568200 con 0x558c552d1c00 2019-12-07T19:34:06.380+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.385+0000 7f8f21451700 10 osd.7 28 tick 2019-12-07T19:34:06.385+0000 7f8f21451700 10 osd.7 28 do_waiters -- start 2019-12-07T19:34:06.385+0000 7f8f21451700 10 osd.7 28 do_waiters -- finish 2019-12-07T19:34:06.385+0000 7f8f21451700 20 osd.7 28 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:06.430+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.480+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.515+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:06.530+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.580+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 10 osd.7 28 tick_without_osd_lock 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fbbf000/0x40000000/0x1680000000, data 0x45814d/0x43f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:03.373892+0000 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373810+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 25 osd.7 28 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 20 osd.7 28 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 10 osd.7 28 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 20 osd.7 28 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:06.623+0000 7f8f1fbbe700 10 osd.7 28 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258911 2019-12-07T19:34:06.630+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:34:06.630+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:06.630+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 17 0x2000~1000 2019-12-07T19:34:06.630+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc c12d782e 2019-12-07T19:34:06.630+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 17 0x1b000~1000 2019-12-07T19:34:06.630+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc cdd76f83 2019-12-07T19:34:06.630+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414608 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.630+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:06.630+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:06.654+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 18 ==== osd_map(28..29 src has 1..29) v4 ==== 1627+0+0 (crc 0 0 0) 0x558c579458c0 con 0x558c5476d000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 osd.7 28 OSD::ms_dispatch: osd_map(28..29 src has 1..29) v4 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 osd.7 28 do_waiters -- start 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 osd.7 28 do_waiters -- finish 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 osd.7 28 _dispatch 0x558c579458c0 osd_map(28..29 src has 1..29) v4 2019-12-07T19:34:06.654+0000 7f8f164bb700 3 osd.7 28 handle_osd_map epochs [28,29], i have 28, src has [1,29] 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 osd.7 28 handle_osd_map got inc map for epoch 29 2019-12-07T19:34:06.654+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3058e826:::osdmap.28:0# 0x0~0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2a80 blob([0x43f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c577f2af0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2b60 blob([!~1000,0x440000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c577f2bd0 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3058e826:::osdmap.28:0# 0x0~1454 = 5204 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 osd.7 28 add_map_bl 28 5204 bytes 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 osd.7 28 got_full_map 29, nothing requested 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 osd.7 28 handle_osd_map pg_num_history pg_num_history(e29 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {29={}} 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 osd.7 28 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e29 [1,29] lci=[13,29]) 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c548f6000 seq 20 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c34b972f:::inc_osdmap.29:0# key 0x7f7fffffffffffffffc34b97'/!inc_osdmap.29!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1139 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~27d 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd83 on front/back, now 0x0~1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 1 blobs 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x441000~1000/1000,1000,0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x441000~1000] 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac930 blob([0x441000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac7e0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c576ac930) get_ref 0x0~27d Blob(0x558c576ac930 blob([0x441000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac7e0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c576ac930) get_ref init 0x1000, 1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~27d: 0x0~27d Blob(0x558c576ac930 blob([0x441000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x27d) SharedBlob(0x558c576ac7e0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c576ac7f8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x27d 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756fc88) dirty_range mark inline shard dirty 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d = 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3bd8e826:::osdmap.29:0# key 0x7f7fffffffffffffff3bd8e8'&!osdmap.29!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3bd8e826:::osdmap.29:0# 0x0~1438 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1140 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3bd8e826:::osdmap.29:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~438 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 2 blobs 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x442000~2000/1000,2000,0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x442000~2000] 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576aca80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576aca80 blob([0x442000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1c00 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c576aca80) get_ref 0x0~1000 Blob(0x558c576aca80 blob([0x442000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1c00 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c576aca80) get_ref init 0x1000, 1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c576aca80 blob([0x442000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574f1c00 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c574f1c18 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e0700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e09a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e0700 blob([!~1000,0x443000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e09a0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c574e0700) get_ref 0x1000~438 Blob(0x558c574e0700 blob([!~1000,0x443000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c574e09a0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c574e0700) get_ref init 0x2000, 1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558c574e0700 blob([!~1000,0x443000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c574e09a0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c574e09b8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1438 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c574a1a08) dirty_range mark inline shard dirty 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3bd8e826:::osdmap.29:0# 0x0~1438 = 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 0 blobs 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 65 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558c576b2b40 con 0x558c54c5b400 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f1199c700 20 osd.7 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 0 blobs 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:06.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 66 ==== pg_notify2(1.cs0 1.cs0 (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e29/29) v1 ==== 1333+0+0 (unknown 0 0 0) 0x558c576e0300 con 0x558c54c5b400 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6000 cost 23294 (4 ios * 4000 + 7294 bytes) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6000 onodes 0x558c53a91900,0x558c55276c80,0x558c574a1900,0x558c5756fb80 shared_blobs 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:06.654+0000 7f8f24457700 15 osd.7 28 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c574a1a08) update #-1:3bd8e826:::osdmap.29:0# 2019-12-07T19:34:06.654+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c574a1a08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3bd8e826:::osdmap.29:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756fc88) update #-1:c34b972f:::inc_osdmap.29:0# 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5756fc88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c34b972f:::inc_osdmap.29:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:06.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6000 allocated 0x[441000~3000] released 0x[] 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 freelist allocate 0x441000~3000 2019-12-07T19:34:06.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.654+0000 7f8f164bb700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:06.654+0000 7f8f0297e700 20 osd.7 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 29, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.654+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 prepare 2019-12-07T19:34:06.654+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6000 2019-12-07T19:34:06.655+0000 7f8f1199c700 10 osd.7 28 do_waiters -- start 2019-12-07T19:34:06.655+0000 7f8f1199c700 10 osd.7 28 do_waiters -- finish 2019-12-07T19:34:06.655+0000 7f8f1199c700 20 osd.7 28 _dispatch 0x558c576b2b40 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.655+0000 7f8f1199c700 3 osd.7 28 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.655+0000 7f8f1199c700 10 osd.7 28 no new maps here, dropping 2019-12-07T19:34:06.655+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 aio_wait 2019-12-07T19:34:06.655+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6000 2019-12-07T19:34:06.655+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 io_done 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1900 had 1 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756fb80 had 1 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8836 < min_flush_size 524288 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8836 < min_flush_size 524288 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x758b0~2284 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x758b0 0x758b0~2284 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x458b0 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8b0 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb34 and padding block with 0x4cc 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x77b34 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000192650s (0.000045903s flush + 0.000146747s kv commit) 2019-12-07T19:34:06.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6000> 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c5754d7c0> 2019-12-07T19:34:06.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 kv_submitted 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6000 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c548f6000 osr 0x558c54ea4b40 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 18 0x1b000~1000 crc d708c37e 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 18 0x2000~1000 crc 7a166c13 2019-12-07T19:34:06.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 deferred_cleanup 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe000 onodes 0x558c53a91900,0x558c55276c80,0x558c5756f400,0x558c5756f900 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54ea3d68 in 0x558c53a59340) _finish_write discard buffer(0x558c548df200 space 0x558c54ea3d68 0x0~1000 writing nocache) 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575672e8 in 0x558c53a59340) _finish_write discard buffer(0x558c548df320 space 0x558c575672e8 0x1000~1000 writing nocache) 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575673c8 in 0x558c53a59340) _finish_write discard buffer(0x558c548df290 space 0x558c575673c8 0x0~1000 writing nocache) 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 done 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7500 deferred_cleanup 2019-12-07T19:34:06.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe000 [] 2019-12-07T19:34:06.655+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 deferred_cleanup 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7500 onodes 0x558c53a91900,0x558c55276c80,0x558c5756e280,0x558c5756e500 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74018 in 0x558c53a59340) _finish_write discard buffer(0x558c548def30 space 0x558c54d74018 0x0~1000 writing nocache) 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2b08 in 0x558c53a59340) _finish_write discard buffer(0x558c548defc0 space 0x558c577f2b08 0x0~1000 writing nocache) 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2be8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba480 space 0x558c577f2be8 0x1000~1000 writing nocache) 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7500 done 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 deferred_queued 2019-12-07T19:34:06.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7500 [] 2019-12-07T19:34:06.655+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7 28 _committed_osd_maps 29..29 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7 28 advance to epoch 29 (<= last 29 <= newest_map 29) 2019-12-07T19:34:06.655+0000 7f8f141a1700 7 osd.7 29 consume_map version 29 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.0s3 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.9s4 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.2s4 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.bs3 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.3s4 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.cs0 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.4s0 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.ds2 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.6s2 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 identify_splits_and_merges 1.7s4 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 29 prune_pg_created 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:0.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:1.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:2.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:3.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:4.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.cs0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:5.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:6.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7:7.consume_map 29 (was 28) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:06.655+0000 7f8f141a1700 10 osd.7 29 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.0s3 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.9s4 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 67 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c576e0300 con 0x558c54c5b400 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.2s4 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s) 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c5500d000 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.3s4 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c5500a000 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.4s0 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.ds2 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.6s2 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f141a1700 15 osd.7 29 enqueue_peering_evt 1.7s4 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 28 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:06.655+0000 7f8f141a1700 7 osd.7 29 activate_map version 29 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 28 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.655+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 29 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 28 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Peering advmap 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946699724s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] adjust_need_up_thru now 28, need_up_thru now false 2019-12-07T19:34:06.655+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.655+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c5731a000 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 29 send_incremental_map 28 -> 29 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.655+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~0 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.655+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~27d size 0x27d (637) 2019-12-07T19:34:06.655+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s4 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac930 blob([0x441000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x27d) SharedBlob(0x558c576ac7e0 sbid 0x0)) need 0x0~27d cache has 0x[0~27d] 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.655+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d = 637 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 29 add_map_inc_bl 29 637 bytes 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 28 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.655+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(29..29 src has 1..29) v4 -- 0x558c57570fc0 con 0x558c54c64000 2019-12-07T19:34:06.655+0000 7f8f06986700 7 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f08189700 10 osd.7 29 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.655+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:06.655+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] publish_stats_to_osd 28:61 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.655+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:29 sent:29 1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) 4->0 ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4)))) e29/29) v1 -- 0x558c576df500 con 0x558c54c64000 2019-12-07T19:34:06.655+0000 7f8f06986700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:06.656858+0000 3 0.000137 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] state: Leaving Peering 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.655+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.655+0000 7f8f06986700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+peering mbc={} ps=[2~1]] exit Started/Primary/Peering 2019-12-07T19:34:06.656876+0000 0 0.000000 2019-12-07T19:34:06.655+0000 7f8f06986700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] enter Started/Primary/Active 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] state: In Active, about to call activate 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.655+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'16 -> 27'16 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 29 send_incremental_map 28 -> 29 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate - purged_snaps [2~1] removed_snaps [2~1,4~1] 2019-12-07T19:34:06.655+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(29..29 src has 1..29) v4 -- 0x558c5730d8c0 con 0x558c54c64400 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:06.655+0000 7f8f07988700 10 osd.7 29 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:06.655+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_notify2(1.2s0 1.2s0 (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e29/29) v1 -- 0x558c576baa00 con 0x558c54c64400 2019-12-07T19:34:06.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 68 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c576e0300 con 0x558c54c5b400 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.092358474s) 2019-12-07T19:34:06.655+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.1(3) 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.655+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.3s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s) 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c55141000 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c54d5f000 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 28 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f0918b700 20 osd.7 29 agent_entry empty queue 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 28 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 28 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_activate_map: 29 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:06.655+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.1(3) is up to date, queueing in pending_activators 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.655+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.655+0000 7f8f06185700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.1(3) 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) uptodate 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.2(4) 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.2(4) is up to date, queueing in pending_activators 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946546939s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.655+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.2(4) 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947682111s) uptodate 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.3(5) 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.655+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.3(5) is up to date, queueing in pending_activators 2019-12-07T19:34:06.655+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.3(5) 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956452115s) uptodate 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.4(2) 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.4(2) is up to date, queueing in pending_activators 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.4(2) 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954806123s) uptodate 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.945840117s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.655+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:29 sent:29 1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) 4->0 ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e29/29) v1 -- 0x558c576b9500 con 0x558c54c64000 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.5(1) 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.5(1) is up to date, queueing in pending_activators 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate peer osd.5(1) 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951438327s) uptodate 2019-12-07T19:34:06.655+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.655+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate setting up missing_loc from shard 1(3) 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.655+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate setting up missing_loc from shard 2(4) 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate setting up missing_loc from shard 3(5) 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.655+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate setting up missing_loc from shard 4(2) 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate setting up missing_loc from shard 5(1) 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 29 send_incremental_map 28 -> 29 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:06.655+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] activate setting up missing_loc from shard 7(0) 2019-12-07T19:34:06.655+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558c54d9b000 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating mbc={} ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:06.655+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(29..29 src has 1..29) v4 -- 0x558c57945b00 con 0x558c54c5bc00 2019-12-07T19:34:06.655+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] release_backoffs [1:30000000::::0,1:40000000::::head) 2019-12-07T19:34:06.655+0000 7f8f05984700 10 osd.7 29 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.655+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.655+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 28 -> 29 2019-12-07T19:34:06.655+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.6s0 1.6s0 (query:29 sent:29 1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) 2->0 ([18,27] all_participants=1(3),2(1),2(4),3(0),4(4),5(5),6(1),7(2) intervals=([18,23] acting 1(3),2(4),3(0),5(5),6(1),7(2)))) e29/29) v1 -- 0x558c57453100 con 0x558c54c5bc00 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.655+0000 7f8f07187700 10 osd.7 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:06.655+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.655+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.655+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.656+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s) 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: got info from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.104938842s int 16s) 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.013703655s -> ruub 46.091235187s 2019-12-07T19:34:06.656+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 28:62 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.091235187s 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.656+0000 7f8f07187700 5 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.657262+0000 5 0.000082 2019-12-07T19:34:06.656+0000 7f8f07187700 5 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.656+0000 7f8f07187700 5 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'12 -> 21'12 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.656+0000 7f8f06986700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] enter Started/Primary/Active/Activating 2019-12-07T19:34:06.656+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 29 send_incremental_map 28 -> 29 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:06.656+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(29..29 src has 1..29) v4 -- 0x558c576b2d80 con 0x558c54c5b400 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 29 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:29 sent:29 1.bs3( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e29/29) v1 -- 0x558c54e98000 con 0x558c54c5b400 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730bb00 seq 10 2019-12-07T19:34:06.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 4(2) notify: (query:29 sent:29 1.cs2( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953923356s) 2->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 4(2), already have info from that osd, ignoring 2019-12-07T19:34:06.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:06.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558c57450e00 con 0x558c54c64000 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730bb00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730bb00 onodes shared_blobs 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 29 send_incremental_map 28 -> 29 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 prepare 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730bb00 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 io_done 2019-12-07T19:34:06.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(29..29 src has 1..29) v4 -- 0x558c57570900 con 0x558c54c19800 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 29 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c54d9b000 2019-12-07T19:34:06.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558c57451500 con 0x558c54c19800 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.656+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558c57452300 con 0x558c54c5bc00 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558c57453800 con 0x558c54c5b400 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 29 send_incremental_map 28 -> 29 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1228 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(29..29 src has 1..29) v4 -- 0x558c57570b40 con 0x558c54c19c00 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 29 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c5513e000 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:06.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558c576e1100 con 0x558c54c19c00 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860332544 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:06.656+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:06.656+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54836f00 seq 15 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998661) bluestore 86 GiB free (0.959953), bluefs_ratio 0.0399508 2019-12-07T19:34:06.656+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:06.656+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.656+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 28 -> 29 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:06.656+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 5585 (1 ios * 4000 + 1585 bytes) 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x77b34~52d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes shared_blobs 2019-12-07T19:34:06.656+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x77b34 0x77b34~52d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 28 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x47b34 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb34 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x61 and padding block with 0xf9f 2019-12-07T19:34:06.656+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:34:06.656+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:34:06.656+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.656+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558c548ab000 2019-12-07T19:34:06.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x78061 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948527770s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_advance_map: 29 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118220s (0.000000090s flush + 0.000118130s kv commit) 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 28 -> 29 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730bb00> 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 kv_submitted 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730bb00 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 finishing 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730bb00 onodes 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730bb00 done 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1429 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 28 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730bb00 [] 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:06.656+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: Active advmap 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1452 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1452 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x78061~5ac to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x78061 0x78061~5ac to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x48061 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x61 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x60d and padding block with 0x9f3 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.656+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:29 sent:29 1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) 4->0 ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4)))) e29/29) v1 -- 0x558c548a2a00 con 0x558c54c5b400 2019-12-07T19:34:06.656+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558c5513e000 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_activate_map: 29 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s) 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] activate_map 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] state: got info from osd.4(0) 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: Active: handling ActMap 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.105685108s int 16s) 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] Active: kicking snap trim 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.013703655s -> ruub 46.091981453s 2019-12-07T19:34:06.656+0000 7f8f0297e700 7 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: handle ActMap primary 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.091981453s 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] prepare_stats_for_publish reporting purged_snaps [2~1,4~1] 2019-12-07T19:34:06.656+0000 7f8f07187700 5 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.657669+0000 6 0.000083 2019-12-07T19:34:06.656+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] publish_stats_to_osd 28:31 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f07187700 5 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7860d 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.656+0000 7f8f07187700 5 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY pruub 44.043602923s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116957s (0.000000090s flush + 0.000116867s kv commit) 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 23'10 -> 23'10 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.656+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5730a600 seq 27 2019-12-07T19:34:06.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a600 onodes shared_blobs 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 prepare 2019-12-07T19:34:06.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a600 2019-12-07T19:34:06.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 io_done 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.656+0000 7f8f07187700 15 osd.7 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558c54d9b000 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7860d~50c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7860d 0x7860d~50c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4860d 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x60d 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb19 and padding block with 0x4e7 2019-12-07T19:34:06.656+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558c57451500 con 0x558c54c5b400 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54836f00> 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 finishing 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x78b19 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.656+0000 7f8f06986700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132777s (0.000000080s flush + 0.000132697s kv commit) 2019-12-07T19:34:06.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a600> 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 kv_submitted 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a600 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 finishing 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a600 onodes 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a600 done 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a600 [] 2019-12-07T19:34:06.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.656+0000 7f8f07187700 15 osd.7 29 enqueue_peering_evt 1.3s4 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: _activate_committed 29 peer_activated now 7(0) last_interval_started 26 last_epoch_started 27 same_interval_since 28 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558c5513e000 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 109 ==== pg_notify2(1.cs0 1.cs0 (query:29 sent:29 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e29/29) v1 ==== 1333+0+0 (crc 0 0 0) 0x558c54e98000 con 0x558c54c64000 2019-12-07T19:34:06.656+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 crt=23'10 unknown pruub 44.043602923s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.656+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.950588616s) e29/29) v1 -- 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:06.656+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 1(3) notify: (query:29 sent:29 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 1(3) notify: (query:29 sent:29 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 1(3) notify: (query:29 sent:29 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 1(3) notify: (query:29 sent:29 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 1(3) notify: (query:29 sent:29 1.cs3( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 3->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 110 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57450e00 con 0x558c54c64000 2019-12-07T19:34:06.656+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 1(3), already have info from that osd, ignoring 2019-12-07T19:34:06.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.656+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.9s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s) 2019-12-07T19:34:06.656+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 33 ==== pg_notify2(1.cs0 1.cs0 (query:29 sent:29 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.951744151s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e29/29) v1 ==== 1344+0+0 (unknown 0 0 0) 0x558c57450e00 con 0x558c54c19c00 2019-12-07T19:34:06.657+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 5(1) notify: (query:29 sent:29 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.951744151s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.657+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 5(1) notify: (query:29 sent:29 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.951744151s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 5(1) notify: (query:29 sent:29 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.951744151s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 5(1) notify: (query:29 sent:29 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.951744151s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.657+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 5(1) notify: (query:29 sent:29 1.cs1( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.951744151s) 1->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.657+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 111 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (crc 0 0 0) 0x558c57570fc0 con 0x558c54c64000 2019-12-07T19:34:06.657+0000 7f8f1199c700 20 osd.7 29 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:06.657+0000 7f8f1199c700 20 osd.7 29 _dispatch 0x558c57570fc0 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.657+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 5(1), already have info from that osd, ignoring 2019-12-07T19:34:06.657+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:06.657+0000 7f8f1199c700 3 osd.7 29 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 no new maps here, dropping 2019-12-07T19:34:06.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.657+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 34 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558c57570b40 con 0x558c54c19c00 2019-12-07T19:34:06.657+0000 7f8f1199c700 20 osd.7 29 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:06.657+0000 7f8f1199c700 20 osd.7 29 _dispatch 0x558c57570b40 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.657+0000 7f8f1199c700 3 osd.7 29 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 no new maps here, dropping 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558c5500a000 2019-12-07T19:34:06.657+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s) 2019-12-07T19:34:06.657+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] state: got info from osd.1(0) 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.657+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.114035089s int 16s) 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.020473142s -> ruub 46.093561947s 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.093561947s 2019-12-07T19:34:06.657+0000 7f8f04181700 5 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.658555+0000 6 0.000096 2019-12-07T19:34:06.657+0000 7f8f04181700 5 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.657+0000 7f8f04181700 5 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.657+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.041350232s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.657+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'8 -> 27'8 2019-12-07T19:34:06.657+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.657+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:06.657+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.657+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c574ffb00 seq 22 2019-12-07T19:34:06.657+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.657+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:06.657+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:06.657+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.657+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.657+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ffb00 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:06.657+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ffb00 onodes shared_blobs 2019-12-07T19:34:06.657+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ffb00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.657+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 prepare 2019-12-07T19:34:06.657+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ffb00 2019-12-07T19:34:06.657+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 io_done 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1387 < min_flush_size 524288 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1410 < min_flush_size 524288 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1410 < min_flush_size 524288 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x78b19~582 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x78b19 0x78b19~582 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x48b19 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb19 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9b and padding block with 0xf65 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7909b 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000137559s (0.000000140s flush + 0.000137419s kv commit) 2019-12-07T19:34:06.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ffb00> 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 kv_submitted 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ffb00 2019-12-07T19:34:06.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 finishing 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ffb00 onodes 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ffb00 done 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ffb00 [] 2019-12-07T19:34:06.657+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.657+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.657+0000 7f8f08189700 15 osd.7 29 enqueue_peering_evt 1.9s4 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.657+0000 7f8f08189700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558c5500a000 2019-12-07T19:34:06.657+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 74 ==== pg_notify2(1.cs0 1.cs0 (query:29 sent:29 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.952428174s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e29/29) v1 ==== 1333+0+0 (unknown 0 0 0) 0x558c576bb100 con 0x558c54c5bc00 2019-12-07T19:34:06.657+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.657+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.041350232s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.657+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:06.657+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 3(5) notify: (query:29 sent:29 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.952428174s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.657+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 3(5) notify: (query:29 sent:29 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.952428174s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.657+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949128765s) e29/29) v1 -- 0x558c574e4700 con 0x558c54c64000 2019-12-07T19:34:06.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.657+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 75 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c57452300 con 0x558c54c5bc00 2019-12-07T19:34:06.657+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.6s2 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115942144s int 16s) 2019-12-07T19:34:06.657+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115942144s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.657+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 76 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558c57945b00 con 0x558c54c5bc00 2019-12-07T19:34:06.657+0000 7f8f1199c700 20 osd.7 29 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:06.657+0000 7f8f1199c700 20 osd.7 29 _dispatch 0x558c57945b00 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.657+0000 7f8f1199c700 3 osd.7 29 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.657+0000 7f8f1199c700 10 osd.7 29 no new maps here, dropping 2019-12-07T19:34:06.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115942144s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115942144s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 3(5) notify: (query:29 sent:29 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.952428174s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115942144s int 16s) 2019-12-07T19:34:06.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 3(5) notify: (query:29 sent:29 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.952428174s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] state: got info from osd.3(0) 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.657+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 3(5) notify: (query:29 sent:29 1.cs5( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.952428174s) 5->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.115942144s int 16s) 2019-12-07T19:34:06.657+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 3(5), already have info from that osd, ignoring 2019-12-07T19:34:06.657+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.021704896s -> ruub 46.094237248s 2019-12-07T19:34:06.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.094237248s 2019-12-07T19:34:06.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.657+0000 7f8f0197c700 5 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.659142+0000 6 0.000092 2019-12-07T19:34:06.657+0000 7f8f0197c700 5 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.657+0000 7f8f0197c700 5 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=26/27 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown NOTIFY pruub 44.040316297s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'20 -> 27'20 2019-12-07T19:34:06.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 35 ==== pg_info2(1.cs0 1.cs0 1.cs1( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.951744151s) e29/29) v1 ==== 1173+0+0 (unknown 0 0 0) 0x558c576e0a00 con 0x558c54c19c00 2019-12-07T19:34:06.658+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.658+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(1) info: 1.cs1( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.951744151s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.658+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(1) info: 1.cs1( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.951744151s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(1) info: 1.cs1( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.951744151s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(1) info: 1.cs1( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.951744151s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0900 seq 42 2019-12-07T19:34:06.658+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:06.658+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(1) info: 1.cs1( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.951744151s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:06.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 5576 (1 ios * 4000 + 1576 bytes) 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes shared_blobs 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.092358474s 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:06.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:34:06.658+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.092358474s) 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:34:06.658+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.5(1) activated and committed 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1443 < min_flush_size 524288 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1443 < min_flush_size 524288 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7909b~5a3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7909b 0x7909b~5a3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4909b 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9b 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x63e and padding block with 0x9c2 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.658+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:63 2019-12-07T19:34:06.658+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7963e 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.658+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0900> 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000136038s (0.000000128s flush + 0.000135910s kv commit) 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:34:06.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:34:06.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:06.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:34:06.658+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.658+0000 7f8f05984700 15 osd.7 29 enqueue_peering_evt 1.6s2 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.658+0000 7f8f05984700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.658+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.658+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 crt=27'20 unknown pruub 44.040316297s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.658+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:06.658+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.6s0 1.6s0 1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.947758137s) e29/29) v1 -- 0x558c54e9aa00 con 0x558c54c5bc00 2019-12-07T19:34:06.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.658+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 69 ==== pg_info2(1.cs0 1.cs0 1.cs2( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953923356s) e29/29) v1 ==== 1162+0+0 (unknown 0 0 0) 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:06.658+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(2) info: 1.cs2( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953923356s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.658+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(2) info: 1.cs2( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953923356s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(2) info: 1.cs2( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953923356s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(2) info: 1.cs2( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953923356s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.658+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(2) info: 1.cs2( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953923356s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.092358474s 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 46.092358474s 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:06.658+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.092358474s) 2019-12-07T19:34:06.659+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.4(2) activated and committed 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.659+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:64 2019-12-07T19:34:06.659+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.659+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 112 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c576e0a00 con 0x558c54c64000 2019-12-07T19:34:06.659+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.7s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s) 2019-12-07T19:34:06.659+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5f000 2019-12-07T19:34:06.659+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 113 ==== pg_info2(1.cs0 1.cs0 1.cs3( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1162+0+0 (crc 0 0 0) 0x558c574e4700 con 0x558c54c64000 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s) 2019-12-07T19:34:06.659+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(3) info: 1.cs3( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.659+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(3) info: 1.cs3( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] state: got info from osd.1(0) 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.116162602s int 16s) 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(3) info: 1.cs3( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(3) info: 1.cs3( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.020473142s -> ruub 46.095689460s 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.095689460s 2019-12-07T19:34:06.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(3) info: 1.cs3( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.659+0000 7f8f0117b700 5 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.660783+0000 6 0.000092 2019-12-07T19:34:06.659+0000 7f8f0117b700 5 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.092358474s 2019-12-07T19:34:06.659+0000 7f8f0117b700 5 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 46.092358474s 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.041897808s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 46.092358474s 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'10 -> 27'10 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.092358474s) 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] rollforward: entry=27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.1(3) activated and committed 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740b200 seq 30 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.659+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b200 cost 5526 (1 ios * 4000 + 1526 bytes) 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b200 onodes shared_blobs 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b200 allocated 0x[] released 0x[] 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 prepare 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b200 2019-12-07T19:34:06.659+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 io_done 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.659+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1382 < min_flush_size 524288 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.659+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1405 < min_flush_size 524288 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1405 < min_flush_size 524288 2019-12-07T19:34:06.659+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:65 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7963e~57d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.659+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7963e 0x7963e~57d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4963e 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x63e 2019-12-07T19:34:06.659+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbbb and padding block with 0x445 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x79bbb 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129323s (0.000000115s flush + 0.000129208s kv commit) 2019-12-07T19:34:06.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.659+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 42 ==== pg_notify2(1.cs0 1.cs0 (query:29 sent:29 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949570549s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) e29/29) v1 ==== 1333+0+0 (crc 0 0 0) 0x558c576dee00 con 0x558c54c19800 2019-12-07T19:34:06.660+0000 7f8f24c58700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 2(4) notify: (query:29 sent:29 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949570549s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.660+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 2(4) notify: (query:29 sent:29 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949570549s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 43 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (crc 0 0 0) 0x558c57570900 con 0x558c54c19800 2019-12-07T19:34:06.660+0000 7f8f1199c700 20 osd.7 29 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.660+0000 7f8f1199c700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:06.660+0000 7f8f1199c700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:06.660+0000 7f8f1199c700 20 osd.7 29 _dispatch 0x558c57570900 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.660+0000 7f8f1199c700 3 osd.7 29 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.660+0000 7f8f1199c700 10 osd.7 29 no new maps here, dropping 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b200> 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 kv_submitted 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b200 2019-12-07T19:34:06.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 finishing 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b200 onodes 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b200 done 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:06.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b200 [] 2019-12-07T19:34:06.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.660+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7f8f05183700 15 osd.7 29 enqueue_peering_evt 1.7s4 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.660+0000 7f8f05183700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558c54d5f000 2019-12-07T19:34:06.660+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.660+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.041897808s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.660+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:06.660+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949379724s) e29/29) v1 -- 0x558c576b8e00 con 0x558c54c64000 2019-12-07T19:34:06.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 2(4) notify: (query:29 sent:29 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949570549s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process 0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 2(4) notify: (query:29 sent:29 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949570549s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.660+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.cs0 from 2(4) notify: (query:29 sent:29 1.cs4( v 27'16 (0'0,27'16] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949570549s) 4->0 ([18,27] all_participants=1(3),2(4),3(5),4(2),5(1),6(0),7(0) intervals=([26,27] acting 1(3),2(4),3(5),4(2),5(1)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.660+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 2(4), already have info from that osd, ignoring 2019-12-07T19:34:06.660+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.660+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 77 ==== pg_info2(1.cs0 1.cs0 1.cs5( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.952428174s) e29/29) v1 ==== 1162+0+0 (unknown 0 0 0) 0x558c54e9aa00 con 0x558c54c5bc00 2019-12-07T19:34:06.660+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.cs5( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.952428174s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.660+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.cs5( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.952428174s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.cs5( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.952428174s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.cs5( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.952428174s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.cs5( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.952428174s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.092358474s 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 46.092358474s 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 46.092358474s 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 46.092358474s 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.092358474s) 2019-12-07T19:34:06.660+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.3(5) activated and committed 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.660+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:66 2019-12-07T19:34:06.660+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:06.660+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.662+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 44 ==== pg_info2(1.cs0 1.cs0 1.cs4( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949570549s) e29/29) v1 ==== 1162+0+0 (crc 0 0 0) 0x558c576dee00 con 0x558c54c19800 2019-12-07T19:34:06.662+0000 7f8f24c58700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(4) info: 1.cs4( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949570549s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.662+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(4) info: 1.cs4( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949570549s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(4) info: 1.cs4( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949570549s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(4) info: 1.cs4( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949570549s) pg_lease_ack(ruub 46.092358474s)) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(4) info: 1.cs4( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.949570549s) pg_lease_ack(ruub 46.092358474s) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.092358474s 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 46.092358474s 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 46.092358474s 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.2 ruub 46.092358474s 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 46.092358474s 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 46.092358474s (sent 46.092358474s) 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.2(4) activated and committed 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.662+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:67 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] state: all_activated_and_committed 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 creating+activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] state: mnow 30.099240285s >= prior_readable_until_ub 0s 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] share_pg_info 2019-12-07T19:34:06.662+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558c576e1800 con 0x558c54c64000 2019-12-07T19:34:06.662+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558c57450700 con 0x558c54c19800 2019-12-07T19:34:06.662+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558c576de700 con 0x558c54c5bc00 2019-12-07T19:34:06.662+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558c576e0300 con 0x558c54c5b400 2019-12-07T19:34:06.662+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558c576df500 con 0x558c54c19c00 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.662+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:68 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] check_local 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] requeue_op 0x558c54e78c60 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] requeue_op 0x558c54e79b80 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] requeue_op 0x558c54e78f20 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] needs_backfill does not need backfill 2019-12-07T19:34:06.662+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] activate all replicas clean, no recovery 2019-12-07T19:34:06.662+0000 7f8f06986700 15 osd.7 29 enqueue_peering_evt 1.cs0 epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.662+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:69 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] agent_stop 2019-12-07T19:34:06.662+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.662+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:06.662+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d20600 seq 16 2019-12-07T19:34:06.662+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:06.662+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:06.662+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:06.662+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 5332 (1 ios * 4000 + 1332 bytes) 2019-12-07T19:34:06.662+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes shared_blobs 2019-12-07T19:34:06.662+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.662+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:06.662+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:06.662+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:06.662+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.662+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) queued 2019-12-07T19:34:06.662+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1163 < min_flush_size 524288 2019-12-07T19:34:06.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:06.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered) prio 255 cost 10 e29) pg 0x558c57676000 2019-12-07T19:34:06.662+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1186 < min_flush_size 524288 2019-12-07T19:34:06.662+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1186 < min_flush_size 524288 2019-12-07T19:34:06.662+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x79bbb~4a2 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x79bbb 0x79bbb~4a2 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x49bbb 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbbb 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5d and padding block with 0xfa3 2019-12-07T19:34:06.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] exit Started/Primary/Active/Activating 2019-12-07T19:34:06.664209+0000 13 0.001598 2019-12-07T19:34:06.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] enter Started/Primary/Active/Recovered 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),7(0) upset 1(3),2(4),3(5),4(2),5(1),7(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),7(0) 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(0) primary objects 5 missing 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(3) objects 5 missing 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 2(4) objects 5 missing 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 5 missing 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(2) objects 5 missing 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(1) objects 5 missing 0 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7a05d 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {1(3),2(4),3(5),4(2),5(1),7(0)=5} 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138251s (0.000000095s flush + 0.000138156s kv commit) 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20600> 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:70 2019-12-07T19:34:06.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] exit Started/Primary/Active/Recovered 2019-12-07T19:34:06.664353+0000 0 0.000000 2019-12-07T19:34:06.663+0000 7f8f0297e700 5 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 pi=[18,28)/1 crt=27'16 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] enter Started/Primary/Active/Clean 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] share_pg_info 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c57451c00 con 0x558c54c64000 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e04000 con 0x558c54c19800 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e04700 con 0x558c54c5bc00 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e04e00 con 0x558c54c5b400 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e05500 con 0x558c54c19c00 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:71 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] exit NotTrimming 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] enter Trimming 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] enter Trimming/WaitReservation 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:72 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] finish_recovery 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] clear_recovery_state 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d20900 seq 17 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20900 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20900 onodes shared_blobs 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 prepare 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20900 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 io_done 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) queued 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) pg 0x558c57676000 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e78f20 prio 63 cost 4 latency 1.006384 osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 29 send_incremental_map 28 -> 29 to 0x558c54c9f000 172.21.15.131:0/224485236 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_map(29..29 src has 1..29) v4 -- 0x558c57302b40 con 0x558c54c9f000 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7a05d~45d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 29 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7a05d 0x7a05d~45d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4a05d 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5d 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4ba and padding block with 0xb46 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] _handle_message: osd_op(client.4540.0:238 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op: op osd_op(client.4540.0:238 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op osd_op(client.4540.0:238 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e28) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_object_context: obc NOT found in cache: 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# _ 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# _ = 0 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7a4ba 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# snapset 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.663+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# snapset = 0 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000133624s (0.000000099s flush + 0.000133525s kv commit) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] populate_obc_watchers 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.663+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20900> 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 kv_submitted 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20900 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 finishing 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20900 onodes 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20900 done 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20900 [] 2019-12-07T19:34:06.663+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:06.663+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# = 0 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_object_context: creating obc from disk: 0x558c57571680 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 0=[]:{} 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:06.663+0000 7f8f0297e700 25 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(read n=1 w=0)) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] execute_ctx 0x558c54e05c00 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b,getxattrs] ov 21'11 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b,getxattrs] 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_osd_op read 0~901120 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_read 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_osd_op checksum in=4b 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_checksum 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_checksum: async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] start_read_op: starting ReadOp(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_read_op: starting read ReadOp(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5793af00 con 0x558c54c64000 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5793b180 con 0x558c54c5b400 2019-12-07T19:34:06.663+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c54d97400 con 0x558c54c19c00 2019-12-07T19:34:06.663+0000 7f8f0297e700 10 osd.7 29 send_incremental_map 21 -> 29 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:34:06.663+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~0 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fb20 blob([0x73000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5757f490 sbid 0x0)) need 0x0~24d cache has 0x[] 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5757fb20 blob([0x73000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5757f490 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~24d]} 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:06.663+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:06.664+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 29 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e9aa00 con 0x558c54c64400 2019-12-07T19:34:06.664+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.2s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.119144108s int 16s) 2019-12-07T19:34:06.664+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.119144108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.119144108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.119144108s int 16s)) prio 255 cost 10 e29) pg 0x558c5731a000 2019-12-07T19:34:06.664+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 30 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (crc 0 0 0) 0x558c5730d8c0 con 0x558c54c64400 2019-12-07T19:34:06.664+0000 7f8f1199c700 20 osd.7 29 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.664+0000 7f8f1199c700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:06.664+0000 7f8f1199c700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:06.664+0000 7f8f1199c700 20 osd.7 29 _dispatch 0x558c5730d8c0 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.664+0000 7f8f1199c700 3 osd.7 29 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.664+0000 7f8f1199c700 10 osd.7 29 no new maps here, dropping 2019-12-07T19:34:06.664+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.119144108s int 16s) 2019-12-07T19:34:06.664+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] state: got info from osd.0(0) 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.664+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.119144108s int 16s) 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.019173437s -> ruub 46.099970671s 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.099970671s 2019-12-07T19:34:06.664+0000 7f8f03980700 5 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.665361+0000 6 0.000097 2019-12-07T19:34:06.664+0000 7f8f03980700 5 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.664+0000 7f8f03980700 5 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.664+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown NOTIFY pruub 44.051723552s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.664+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown pruub 44.051723552s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'12 -> 27'12 2019-12-07T19:34:06.664+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown pruub 44.051723552s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.664+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:06.664+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54d59200 seq 13 2019-12-07T19:34:06.664+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:06.664+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 36 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 ==== 225444+0+0 (unknown 0 0 0) 0x558c5793af00 con 0x558c54c19c00 2019-12-07T19:34:06.664+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:06.664+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000014 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.664+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:06.664+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.664+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:06.664+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:06.664+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5576 (1 ios * 4000 + 1576 bytes) 2019-12-07T19:34:06.664+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:06.664+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:06.664+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:06.664+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:06.664+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:34:06.664+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1443 < min_flush_size 524288 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1443 < min_flush_size 524288 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7a4ba~5a3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7a4ba 0x7a4ba~5a3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4a4ba 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4ba 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa5d and padding block with 0x5a3 2019-12-07T19:34:06.664+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 70 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 ==== 225444+0+0 (unknown 0 0 0) 0x558c54d97400 con 0x558c54c5b400 2019-12-07T19:34:06.664+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740cb00 prio 127 cost 0 latency 0.000010 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.664+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7aa5d 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.664+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116044s (0.000000074s flush + 0.000115970s kv commit) 2019-12-07T19:34:06.664+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:06.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:06.664+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:06.664+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.664+0000 7f8f07988700 15 osd.7 29 enqueue_peering_evt 1.2s4 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.664+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.664+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558c5731a000 2019-12-07T19:34:06.664+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown pruub 44.051723552s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.664+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 crt=27'12 unknown pruub 44.051723552s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.664+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.2s0 1.2s0 1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.959400255s) e29/29) v1 -- 0x558c548a1c00 con 0x558c54c64400 2019-12-07T19:34:06.664+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.665+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 114 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 ==== 225444+0+0 (crc 0 0 0) 0x558c5793b180 con 0x558c54c64000 2019-12-07T19:34:06.665+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c573342c0 prio 127 cost 0 latency 0.000017 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.665+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5757fb20 blob([0x73000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5757f490 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~24d]} 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5757f4a8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7b90 space 0x558c5757f4a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:06.665+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d = 589 2019-12-07T19:34:06.665+0000 7f8f0297e700 10 osd.7 29 add_map_inc_bl 22 589 bytes 2019-12-07T19:34:06.665+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~0 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd2d0 blob([0x14f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c574e0e70 sbid 0x0)) need 0x0~269 cache has 0x[] 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575bd2d0 blob([0x14f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c574e0e70 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~269]} 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575bd2d0 blob([0x14f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c574e0e70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~269]} 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574e0e88 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7c20 space 0x558c574e0e88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:06.665+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 = 617 2019-12-07T19:34:06.665+0000 7f8f0297e700 10 osd.7 29 add_map_inc_bl 23 617 bytes 2019-12-07T19:34:06.665+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(22..29 src has 1..29) v4 -- 0x558c575718c0 con 0x558c53ae5400 2019-12-07T19:34:06.665+0000 7f8f0297e700 10 osd.7 29 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 21 -> 29 (shared) 2019-12-07T19:34:06.665+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57573180 con 0x558c53ae5400 2019-12-07T19:34:06.665+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(22..29 src has 1..29) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c575718c0 con 0x558c53ae5400 2019-12-07T19:34:06.665+0000 7f8f1199c700 20 osd.7 29 OSD::ms_dispatch: osd_map(22..29 src has 1..29) v4 2019-12-07T19:34:06.665+0000 7f8f1199c700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:06.665+0000 7f8f1199c700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:06.665+0000 7f8f1199c700 20 osd.7 29 _dispatch 0x558c575718c0 osd_map(22..29 src has 1..29) v4 2019-12-07T19:34:06.665+0000 7f8f1199c700 3 osd.7 29 handle_osd_map epochs [22,29], i have 29, src has [1,29] 2019-12-07T19:34:06.665+0000 7f8f1199c700 10 osd.7 29 no new maps here, dropping 2019-12-07T19:34:06.665+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c57573180 con 0x558c53ae5400 2019-12-07T19:34:06.665+0000 7f8f1119b700 15 osd.7 29 enqueue_op 0x558c57334580 prio 127 cost 0 latency 0.000016 epoch 29 MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.665+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) 2019-12-07T19:34:06.665+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_read_op: started ReadOp(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),5(1),7(0)) 2019-12-07T19:34:06.665+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e78f20 finish 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.665+0000 7f8f0b990700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:06.665+0000 7f8f0b990700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] SnapTrimmer state: NotTrimming: trimming 4 2019-12-07T19:34:06.665+0000 7f8f0b990700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] exit Trimming/WaitReservation 2019-12-07T19:34:06.665+0000 7f8f0b990700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.665+0000 7f8f0b990700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:73 2019-12-07T19:34:06.665+0000 7f8f0b990700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:06.665+0000 7f8f0b990700 10 osd.7 29 queueing pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] for snaptrim 2019-12-07T19:34:06.665+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=29) prio 5 cost 1048576 e29) 2019-12-07T19:34:06.665+0000 7f8f0b990700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.665+0000 7f8f0b990700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:74 2019-12-07T19:34:06.665+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) pg 0x558c57676000 2019-12-07T19:34:06.665+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c54e79b80 prio 63 cost 4 latency 1.008930 osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.665+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: osd_op(client.4540.0:239 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op: op osd_op(client.4540.0:239 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:06.666+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 78 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c57450000 con 0x558c54c5bc00 2019-12-07T19:34:06.666+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.6s2 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:06.666+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op osd_op(client.4540.0:239 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e28) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(read n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 0=[]:{} 2019-12-07T19:34:06.666+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.666+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:06.666+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.947758137s) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active pruub 44.040316297s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:06.666+0000 7f8f06986700 25 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:06.666+0000 7f8f05984700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active pruub 44.040316297s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(read n=2 w=0)) 2019-12-07T19:34:06.666+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 0x558c54e04700 2019-12-07T19:34:06.666+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:06.666+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f7800 seq 43 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] ov 21'11 2019-12-07T19:34:06.666+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:06.666+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] 2019-12-07T19:34:06.666+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op read 0~901120 2019-12-07T19:34:06.666+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:06.666+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes shared_blobs 2019-12-07T19:34:06.666+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_read 2019-12-07T19:34:06.666+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:34:06.666+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:34:06.666+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op checksum in=4b 2019-12-07T19:34:06.666+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_checksum 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1152 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_checksum: async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1175 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1175 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7aa5d~497 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7aa5d 0x7aa5d~497 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4aa5d 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa5d 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xef4 and padding block with 0x10c 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] start_read_op: starting ReadOp(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_read_op: starting read ReadOp(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:06.666+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57977680 con 0x558c54c64000 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7aef4 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.666+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.666+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57977900 con 0x558c54c5b400 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119233s (0.000000090s flush + 0.000119143s kv commit) 2019-12-07T19:34:06.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.666+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57977b80 con 0x558c54c19c00 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7800> 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:34:06.666+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c54ce6c80 con 0x558c53ae5400 2019-12-07T19:34:06.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 finishing 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:06.666+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:34:06.666+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.666+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.666+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c54ce6c80 con 0x558c53ae5400 2019-12-07T19:34:06.666+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_read_op: started ReadOp(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),5(1),7(0)) 2019-12-07T19:34:06.666+0000 7f8f1119b700 15 osd.7 29 enqueue_op 0x558c573351e0 prio 127 cost 0 latency 0.000019 epoch 29 MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c54e79b80 finish 2019-12-07T19:34:06.666+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.666+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.666+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.002106 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.666+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.666+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=5, attrs_read=0) 2019-12-07T19:34:06.666+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),7(0)) 2019-12-07T19:34:06.666+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740c9a0 finish 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.666+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57334580 prio 127 cost 0 latency 0.000554 MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.666+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.666+0000 7f8f06986700 25 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:06.666+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~37000 size 0x37000 (225280) 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) need {<0x0, 0x1000> : [0x36000:0~1000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x36000: 0x0 reading 0x0~1000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) need {<0x0, 0x2000> : [0x34000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x34000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) need {<0x0, 0x2000> : [0x32000:0~2000]} 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32000: 0x0 reading 0x0~2000 2019-12-07T19:34:06.666+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:06.667+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 71 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 225444+0+0 (unknown 0 0 0) 0x558c57977680 con 0x558c54c5b400 2019-12-07T19:34:06.667+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c57334420 prio 127 cost 0 latency 0.000018 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.667+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.667+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 37 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 225444+0+0 (unknown 0 0 0) 0x558c57977b80 con 0x558c54c19c00 2019-12-07T19:34:06.667+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c57334840 prio 127 cost 0 latency 0.000011 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.667+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.667+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 115 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 225444+0+0 (crc 0 0 0) 0x558c57977900 con 0x558c54c64000 2019-12-07T19:34:06.667+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c57335600 prio 127 cost 0 latency 0.000010 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.667+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.668+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 116 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c57451c00 con 0x558c54c64000 2019-12-07T19:34:06.668+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.9s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.668+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.668+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.668+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558c5500a000 2019-12-07T19:34:06.668+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 117 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1211+0+0 (crc 0 0 0) 0x558c57451c00 con 0x558c54c64000 2019-12-07T19:34:06.668+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949128765s) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.041350232s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.668+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.7s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.668+0000 7f8f08189700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.041350232s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.668+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.668+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:06.668+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.668+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d58f00 seq 23 2019-12-07T19:34:06.668+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.668+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.668+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.668+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:06.668+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes shared_blobs 2019-12-07T19:34:06.668+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.668+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:34:06.668+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:34:06.668+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:34:06.668+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.668+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5f000 2019-12-07T19:34:06.668+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949379724s) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.041897808s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.668+0000 7f8f05183700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.041897808s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.668+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:06.668+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:06.668+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54dde300 seq 31 2019-12-07T19:34:06.668+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:06.668+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:06.668+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:06.668+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.668+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde300 onodes shared_blobs 2019-12-07T19:34:06.668+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.668+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 prepare 2019-12-07T19:34:06.668+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde300 2019-12-07T19:34:06.668+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 io_done 2019-12-07T19:34:06.668+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:06.668+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 31 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c57450000 con 0x558c54c64400 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2164 < min_flush_size 524288 2019-12-07T19:34:06.668+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.2s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:06.668+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2187 < min_flush_size 524288 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2187 < min_flush_size 524288 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7aef4~88b to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7aef4 0x7aef4~88b to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4aef4 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xef4 2019-12-07T19:34:06.668+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) pg 0x558c5731a000 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x77f and padding block with 0x881 2019-12-07T19:34:06.668+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 32 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c57450000 con 0x558c54c64400 2019-12-07T19:34:06.668+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.2s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:06.668+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.668+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active pruub 44.051723552s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:06.668+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active pruub 44.051723552s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.668+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:06.668+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 33 ==== pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 ==== 258+0+0 (crc 0 0 0) 0x558c57944480 con 0x558c54c64400 2019-12-07T19:34:06.668+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7b77f 2019-12-07T19:34:06.668+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54d58000 seq 14 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.668+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d2840 prio 127 cost 0 latency 0.000018 epoch 29 pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.668+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:06.668+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e29) 2019-12-07T19:34:06.668+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000106486s (0.000000099s flush + 0.000106387s kv commit) 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.668+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00,0x558c54dde300> 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.668+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:34:06.668+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:34:06.668+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.668+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:34:06.668+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:34:06.668+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 kv_submitted 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde300 2019-12-07T19:34:06.668+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 finishing 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde300 onodes 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde300 done 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde300 [] 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1152 < min_flush_size 524288 2019-12-07T19:34:06.668+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1175 < min_flush_size 524288 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1175 < min_flush_size 524288 2019-12-07T19:34:06.668+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7b77f~497 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7b77f 0x7b77f~497 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4b77f 2019-12-07T19:34:06.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x77f 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) pg 0x558c5731a000 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc16 and padding block with 0x3ea 2019-12-07T19:34:06.668+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:06.668+0000 7f8f07988700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.668+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7f8f07988700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.668+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:06.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:06.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5740af00 seq 15 2019-12-07T19:34:06.668+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:06.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7bc16 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:06.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:06.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5386 (1 ios * 4000 + 1386 bytes) 2019-12-07T19:34:06.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 2019-12-07T19:34:06.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119373s (0.000000086s flush + 0.000119287s kv commit) 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:06.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:06.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.668+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 118 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1211+0+0 (crc 0 0 0) 0x558c57451c00 con 0x558c54c64000 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58000> 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:34:06.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 kv_queued 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:34:06.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.668+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 79 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c57450000 con 0x558c54c5bc00 2019-12-07T19:34:06.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.668+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.7s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.668+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.669+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.6s2 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:06.669+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1223 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1246 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1246 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7bc16~4de to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7bc16 0x7bc16~4de to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e29) pg 0x558c5731a000 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4bc16 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc16 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf4 and padding block with 0xf0c 2019-12-07T19:34:06.669+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 80 ==== MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=84, reqid=osd.3.0:84, at_version=29'22, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2632+0+0 (unknown 0 0 0) 0x558c54e40800 con 0x558c54c5bc00 2019-12-07T19:34:06.669+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d29a0 prio 127 cost 0 latency 0.000026 epoch 29 MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=84, reqid=osd.3.0:84, at_version=29'22, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.669+0000 7f8f03980700 10 osd.7 29 dequeue_op 0x558c578d2840 prio 127 cost 0 latency 0.000240 pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 pg pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.669+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=84, reqid=osd.3.0:84, at_version=29'22, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.669+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] _handle_message: pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:06.669+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] do_update_log_missing op_trim_to = -- op_roll_forward_to = -- 2019-12-07T19:34:06.669+0000 7f8f03980700 20 update missing, append 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.669+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 lua=27'12 crt=27'12 lcod 27'12 active mbc={} ps=[2~1]] append_log_entries_update_missing trim_to bool = 0 trim_to = 0'0 2019-12-07T19:34:06.669+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.669+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54d59b00 seq 16 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.669+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000029.00000000000000000013' <- 0000000029.00000000000000000013 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._fastinfo' <- _fastinfo 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.669+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:06.669+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 4717 (1 ios * 4000 + 717 bytes) 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea1b00 seq 44 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:06.669+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:06.669+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:06.669+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 5386 (1 ios * 4000 + 1386 bytes) 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes shared_blobs 2019-12-07T19:34:06.669+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 lua=27'12 crt=27'12 lcod 27'12 active mbc={} ps=[2~1]] op_applied version 29'13 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.669+0000 7f8f03980700 10 osd.7 29 dequeue_op 0x558c578d2840 finish 2019-12-07T19:34:06.669+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=84, reqid=osd.3.0:84, at_version=29'22, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=84, reqid=osd.3.0:84, at_version=29'22, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 29 dequeue_op 0x558c578d29a0 prio 127 cost 0 latency 0.000160 MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=84, reqid=osd.3.0:84, at_version=29'22, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=84, reqid=osd.3.0:84, at_version=29'22, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 snap_mapper.remove_oid 1:6c205177:::smithi13140796-8:4 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 snap_mapper._remove_oid 1:6c205177:::smithi13140796-8:4 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:4 4 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 snap_mapper.clear_snaps 1:6c205177:::smithi13140796-8:4 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 27'20 (0'0,27'20] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] append_log log((0'0,27'20], crt=27'20) [29'21 (27'14) delete 1:6c205177:::smithi13140796-8:4 by unknown.0.0:0 2019-12-07T19:34:06.669023+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'22 (27'15) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:06.669023+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'21 (0'0,29'21] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=27'20 crt=27'20 lcod 27'20 active mbc={} ps=[2~1]] add_log_entry 29'21 (27'14) delete 1:6c205177:::smithi13140796-8:4 by unknown.0.0:0 2019-12-07T19:34:06.669023+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'22 (0'0,29'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=27'20 crt=27'20 lcod 27'20 active mbc={} ps=[2~1]] add_log_entry 29'22 (27'15) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:06.669023+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'22 (0'0,29'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=27'20 crt=27'20 lcod 27'20 active mbc={} ps=[2~1]] append_log approx pg log length = 22 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'22 (0'0,29'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=27'20 crt=27'20 lcod 27'20 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.669+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'21, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0c00 seq 45 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:4#15 key 0x8280000000000000016c2051'w!smithi13140796-8!='0x00000000000000040000000000000015'o' 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# -> 2#1:6c205177:::smithi13140796-8:4#15 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:4# -> 2#1:6c205177:::smithi13140796-8:4#15 = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.4.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.2_0000000000000001.6340A8EE.4.smithi13140796-8..' <- SNA_1_0000000000000004_.2_0000000000000001.6340A8EE.4.smithi13140796-8.. 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000029.00000000000000000021' <- 0000000029.00000000000000000021 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000029.00000000000000000022' <- 0000000029.00000000000000000022 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 7506 (1 ios * 4000 + 3506 bytes) 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes 0x558c54ce3680,0x558c575ba280 shared_blobs 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3788) update 2#1:6c205177:::smithi13140796-8:4#15 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54cf8718 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c579821b0 space 0x558c54cf8718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:4#15 is 527 (410 bytes onode + 2 bytes spanning blobs + 115 bytes inline extents) 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54d75358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 680 (484 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57982240 space 0x558c54d75358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57396408 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c579822d0 space 0x558c57396408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57396168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57982360 space 0x558c57396168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x36000:0~1000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c579823f0 space 0x558c574d3748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'22 (0'0,29'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=27'20 crt=27'20 lcod 27'20 active mbc={} ps=[2~1]] op_applied version 29'22 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57982480 space 0x558c574d3908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f0197c700 10 osd.7 29 dequeue_op 0x558c578d29a0 finish 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x34000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c574d3898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbb950 space 0x558c574d3898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c578de638 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbb9e0 space 0x558c578de638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5f000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57819d68 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbba70 space 0x558c57819d68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57880558 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbb00 space 0x558c57880558 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57880638 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbb90 space 0x558c57880638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57880718 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbc20 space 0x558c57880718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c578807f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbcb0 space 0x558c578807f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c578808d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbd40 space 0x558c578808d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740a300 seq 32 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c578809b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbdd0 space 0x558c578809b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbe60 space 0x558c54d75198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes shared_blobs 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57880e88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54dbbef0 space 0x558c57880e88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:34:06.669+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57819f28 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14000 space 0x558c57819f28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c578de0f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7c0f4 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14090 space 0x558c578de0f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c578de2b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14120 space 0x558c578de2b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c578de478 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e141b0 space 0x558c578de478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000629445s (0.000000057s flush + 0.000629388s kv commit) 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54d3df98 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14240 space 0x558c54d3df98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 567 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5794ce18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e142d0 space 0x558c5794ce18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5794c168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14360 space 0x558c5794c168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1790 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 2 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5418 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5794c248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e143f0 space 0x558c5794c248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5794c2b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14480 space 0x558c5794c2b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3680 had 1 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12000 had 1 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5794c478 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14510 space 0x558c5794c478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6512 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x32000:0~2000]} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c574d32e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:06.669+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e145a0 space 0x558c574d32e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6535 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6535 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 = 225280 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7c0f4~1987 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7c0f4 0x7c0f4~1987 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4c0f4 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf4 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa7b and padding block with 0x585 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read read request=225280 r=225280 len=225280 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.669+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00> 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.669+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:06.669+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read: Checking hash of 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [] 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.669+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7da7b 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.669+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 -- 0x558c54ce6a00 con 0x558c53ae5400 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57334580 finish 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000163835s (0.000000089s flush + 0.000163746s kv commit) 2019-12-07T19:34:06.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _finish_recovery 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00,0x558c54ea1b00,0x558c54ea0c00,0x558c5740a300> 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:06.669+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 119 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c57451c00 con 0x558c54c64000 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] purge_strays 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:34:06.669+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.9s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:34:06.669+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 0x558c54ce3680,0x558c575ba280 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.669+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:75 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:34:06.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.669+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 120 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57451c00 con 0x558c54c64000 2019-12-07T19:34:06.669+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 29/28 ECSubWriteReply(tid=84, last_complete=29'22, committed=1, applied=1)) v2 -- 0x558c576b4c00 con 0x558c54c5bc00 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.669+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.669+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.7s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.669+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.669+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558c5500a000 2019-12-07T19:34:06.669+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c573342c0 prio 127 cost 0 latency 0.004597 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.669+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.669+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=5, attrs_read=0) 2019-12-07T19:34:06.669+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.669+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [1(3),225280, 5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=4(2),7(0)) 2019-12-07T19:34:06.669+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c573342c0 finish 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.669+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5730a300 seq 24 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.669+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.669+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:06.669+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c5740cb00 prio 127 cost 0 latency 0.005557 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.669+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.669+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a300 cost 5375 (1 ios * 4000 + 1375 bytes) 2019-12-07T19:34:06.669+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a300 onodes shared_blobs 2019-12-07T19:34:06.669+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.669+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 prepare 2019-12-07T19:34:06.669+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a300 2019-12-07T19:34:06.669+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 io_done 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=5, attrs_read=0) 2019-12-07T19:34:06.669+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c54ce6a00 con 0x558c53ae5400 2019-12-07T19:34:06.669+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7f8f1119b700 15 osd.7 29 enqueue_op 0x558c57334000 prio 127 cost 0 latency 0.000152 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.669+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [1(3),225280, 4(2),225280, 5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=7(0)) 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c5740cb00 finish 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.669+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57334840 prio 127 cost 0 latency 0.002680 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:34:06.669+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),7(0)) 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57334840 finish 2019-12-07T19:34:06.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5f000 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.670+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c573351e0 prio 127 cost 0 latency 0.003735 MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.670+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.670+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:06.670+0000 7f8f06986700 25 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:06.670+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 -- 0x558c54d76c00 con 0x558c54c64400 2019-12-07T19:34:06.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:06.670+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 2019-12-07T19:34:06.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54dde600 seq 33 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~37000 size 0x37000 (225280) 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:06.670+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde600 onodes shared_blobs 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1212 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 prepare 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde600 2019-12-07T19:34:06.670+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 io_done 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1235 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1235 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7da7b~4d3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7da7b 0x7da7b~4d3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4da7b 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa7b 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.670+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:06.670+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 = 225280 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf4e and padding block with 0xb2 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read read request=225280 r=225280 len=225280 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.670+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:06.670+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:06.670+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read: Checking hash of 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7df4e 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.670+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 -- 0x558c575bb180 con 0x558c53ae5400 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c573351e0 finish 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000090239s (0.000000069s flush + 0.000090170s kv commit) 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57335600 prio 127 cost 0 latency 0.002567 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7df4e~46d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7df4e 0x7df4e~46d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4df4e 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf4e 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3bb and padding block with 0xc45 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [1(3),225280, 5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=4(2),7(0)) 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57335600 finish 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a300> 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 kv_submitted 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a300 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 finishing 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a300 onodes 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a300 done 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a300 [] 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.670+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.670+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7e3bb 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129522s (0.000000094s flush + 0.000129428s kv commit) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde600> 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 kv_submitted 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde600 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 finishing 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde600 onodes 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde600 done 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde600 [] 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.670+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c575bb180 con 0x558c53ae5400 2019-12-07T19:34:06.670+0000 7f8f1119b700 15 osd.7 29 enqueue_op 0x558c578d2420 prio 127 cost 0 latency 0.000266 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.670+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.670+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 72 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558c57451c00 con 0x558c54c5b400 2019-12-07T19:34:06.670+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.670+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.670+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 73 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c57451c00 con 0x558c54c5b400 2019-12-07T19:34:06.670+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.3s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.670+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.670+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 74 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558c57451c00 con 0x558c54c5b400 2019-12-07T19:34:06.670+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.670+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.670+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 75 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c57451c00 con 0x558c54c5b400 2019-12-07T19:34:06.670+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.670+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558c54d9b000 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730ac00 seq 11 2019-12-07T19:34:06.670+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730ac00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730ac00 onodes shared_blobs 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 prepare 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730ac00 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 io_done 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558c5513e000 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950588616s) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active pruub 44.043602923s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active pruub 44.043602923s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7e3bb~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7e3bb 0x7e3bb~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4e3bb 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3bb 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7d1 and padding block with 0x82f 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5730af00 seq 28 2019-12-07T19:34:06.670+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730af00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730af00 onodes shared_blobs 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730af00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 prepare 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730af00 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 io_done 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558c54d9b000 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7e7d1 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000081799s (0.000000089s flush + 0.000081710s kv commit) 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730ac00> 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 kv_submitted 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730ac00 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 finishing 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730ac00 onodes 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730ac00 done 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730ac00 [] 2019-12-07T19:34:06.670+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c574fe000 seq 12 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7e7d1~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:06.670+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7e7d1 0x7e7d1~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.670+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4e7d1 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7d1 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbe7 and padding block with 0x419 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe000 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe000 onodes shared_blobs 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 prepare 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe000 2019-12-07T19:34:06.670+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 io_done 2019-12-07T19:34:06.670+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558c54d9b000 2019-12-07T19:34:06.670+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.670+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.670+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:06.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730b200 seq 13 2019-12-07T19:34:06.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:06.670+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:06.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:06.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes shared_blobs 2019-12-07T19:34:06.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[] released 0x[] 2019-12-07T19:34:06.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:06.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57334420 prio 127 cost 0 latency 0.003839 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:06.670+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:34:06.670+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7ebe7 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.670+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [1(3),225280, 4(2),225280, 5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=7(0)) 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.671+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c57334420 finish 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000172148s (0.000000078s flush + 0.000172070s kv commit) 2019-12-07T19:34:06.671+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.671+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) queued 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730af00> 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 kv_submitted 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730af00 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 finishing 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730af00 onodes 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730af00 done 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730af00 [] 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 2 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2204 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2227 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2227 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7ebe7~8b3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7ebe7 0x7ebe7~8b3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4ebe7 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbe7 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c57334000 prio 127 cost 0 latency 0.001254 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x49a and padding block with 0xb66 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=5, attrs_read=0)) v2 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=5, attrs_read=0) 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7f49a 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=5, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [1(3),225280, 4(2),225280, 5(1),225280, 7(0),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=) 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000123543s (0.000000074s flush + 0.000123469s kv commit) 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe000,0x558c5730b200> 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 kv_submitted 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe000 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 finishing 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe000 onodes 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 done 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe000 [] 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [] 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] objects_read_async_cb: got: {1:33f5bf01:::smithi13140796-2:head=0,{0~901120(901120)}} 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:06.671+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 76 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c57451c00 con 0x558c54c5b400 2019-12-07T19:34:06.671+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.3s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.671+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558c5513e000 2019-12-07T19:34:06.671+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.671+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.671+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea0f00 seq 29 2019-12-07T19:34:06.671+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.671+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:06.671+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.671+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:06.671+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] finish_checksum 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 0x558c54e05c00 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7f49a~45d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7f49a 0x7f49a~45d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4f49a 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x49a 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8f7 and padding block with 0x709 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b,getxattrs] ov 21'11 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b,getxattrs] 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op read 0~901120 out=901120b 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op getxattrs 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx alloc reply 0x558c57572f00 result 0 2019-12-07T19:34:06.671+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7f8f7 2019-12-07T19:34:06.671+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] log_op_stats osd_op(client.4540.0:238 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b,getxattrs out=74b] snapc 0=[] ondisk+read+known_if_redirected e28) v8 inb 0 outb 901202 lat 1.014616 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.671+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:76 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000114332s (0.000000067s flush + 0.000114265s kv commit) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.671+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(238 smithi13140796-2 [read 0~901120 out=901120b,checksum out=8b,getxattrs out=74b] v0'0 uv11 ondisk = 0) v8 -- 0x558c57572f00 con 0x558c54c9f000 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:06.671+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.671+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c57334000 finish 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8) prio 63 cost 4 e29) pg 0x558c57676000 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e78c60 prio 63 cost 4 latency 1.014640 osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: osd_op(client.4540.0:240 1.cs0 1.80fdafcc (undecoded) ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op: op osd_op(client.4540.0:240 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e28) v8 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op osd_op(client.4540.0:240 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e28) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(read n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 0=[]:{} 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:06.671+0000 7f8f0297e700 25 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(read n=2 w=0)) 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 0x558c54e05c00 2019-12-07T19:34:06.671+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 77 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c57451c00 con 0x558c54c5b400 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] ov 21'11 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~901120,checksum in=4b] 2019-12-07T19:34:06.671+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.3s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op read 0~901120 2019-12-07T19:34:06.671+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_read 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.671+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op checksum in=4b 2019-12-07T19:34:06.671+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_checksum 2019-12-07T19:34:06.671+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558c5513e000 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_checksum: async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:06.671+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:06.671+0000 7f8f07187700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:34:06.671+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:06.671+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:06.671+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5730a000 seq 30 2019-12-07T19:34:06.671+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] start_read_op: starting ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:06.671+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:06.671+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_read_op: starting read ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:06.671+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:06.671+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a000 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.671+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a000 onodes shared_blobs 2019-12-07T19:34:06.671+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.671+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 prepare 2019-12-07T19:34:06.671+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a000 2019-12-07T19:34:06.671+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 io_done 2019-12-07T19:34:06.671+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57572a00 con 0x558c54c64000 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.671+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57572780 con 0x558c54c5b400 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.671+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57572500 con 0x558c54c19c00 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7f8f7~46d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7f8f7 0x7f8f7~46d to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4f8f7 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8f7 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd64 and padding block with 0x29c 2019-12-07T19:34:06.671+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57572280 con 0x558c53ae5400 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_read_op: started ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, []))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),5(1),7(0)) 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x7fd64 2019-12-07T19:34:06.671+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e78c60 finish 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.671+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=29) prio 5 cost 1048576 e29) queued 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000087230s (0.000000066s flush + 0.000087164s kv commit) 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.672+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c578d2420 prio 127 cost 0 latency 0.001833 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.672+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:34:06.672+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:06.672+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c57572280 con 0x558c53ae5400 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a000> 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.672+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 kv_submitted 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a000 2019-12-07T19:34:06.672+0000 7f8f1119b700 15 osd.7 29 enqueue_op 0x558c578d2000 prio 127 cost 0 latency 0.000124 epoch 29 MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.672+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 finishing 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a000 onodes 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 done 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:06.672+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) 2019-12-07T19:34:06.672+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a000 [] 2019-12-07T19:34:06.672+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.672+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.672+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.672+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 34 ==== MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 1799+0+0 (crc 0 0 0) 0x558c54897000 con 0x558c54c64400 2019-12-07T19:34:06.672+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d2b00 prio 127 cost 0 latency 0.000032 epoch 29 MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.672+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.672+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.672+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558c5731a000 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 29 dequeue_op 0x558c578d2b00 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.2s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.672+0000 7f8f03980700 20 snap_mapper.remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.672+0000 7f8f03980700 20 snap_mapper._remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:06.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.672+0000 7f8f03980700 20 snap_mapper.get_snaps 1:41df217a:::smithi13140796-39:4 4 2019-12-07T19:34:06.672+0000 7f8f03980700 20 snap_mapper.clear_snaps 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.672+0000 7f8f03980700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.672+0000 7f8f03980700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=6, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [1(3),225280, 4(2),225280, 5(1),225280, 7(0),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=) 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] append_log log((0'0,29'13], crt=27'12) [29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'14 (0'0,29'14] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=27'12 lcod 29'13 active mbc={} ps=[2~1]] add_log_entry 29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=27'12 lcod 29'13 active mbc={} ps=[2~1]] add_log_entry 29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=27'12 lcod 29'13 active mbc={} ps=[2~1]] append_log approx pg log length = 15 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=27'12 lcod 29'13 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.672+0000 7f8f03980700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.672+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] objects_read_async_cb: got: {1:33f5bf01:::smithi13140796-2:head=0,{0~901120(901120)}} 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54d21b00 seq 17 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41df217a:::smithi13140796-39:4#e key 0x84800000000000000141df21'z!smithi13140796-39!='0x0000000000000004000000000000000e'o' 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:06.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# -> 4#1:41df217a:::smithi13140796-39:4#e 2019-12-07T19:34:06.672+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 38 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 225444+0+0 (unknown 0 0 0) 0x558c57572a00 con 0x558c54c19c00 2019-12-07T19:34:06.672+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c573358c0 prio 127 cost 0 latency 0.000018 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.2s4_head 4#1:41df217a:::smithi13140796-39:4# -> 4#1:41df217a:::smithi13140796-39:4#e = 0 2019-12-07T19:34:06.672+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) get_onode oid 4#1:41df217a:::smithi13140796-39:head#f key 0x84800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffe000000000000000f'o' 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s4_head 0x558c54d39860) r -2 v.len 0 2019-12-07T19:34:06.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# -> 4#1:41df217a:::smithi13140796-39:head#f 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.2s4_head 4#1:41df217a:::smithi13140796-39:head# -> 4#1:41df217a:::smithi13140796-39:head#f = 0 2019-12-07T19:34:06.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.28BF48E5.4.smithi13140796-39..' <- OBJ_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.4_0000000000000001.28BF48E5.4.smithi13140796-39..' <- SNA_1_0000000000000004_.4_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000029.00000000000000000014' <- 0000000029.00000000000000000014 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.0000000029.00000000000000000015' <- 0000000029.00000000000000000015 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 6674 (1 ios * 4000 + 2674 bytes) 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes 0x558c5793a000,0x558c5793a280 shared_blobs 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore.extentmap(0x558c5793a108) update 4#1:41df217a:::smithi13140796-39:4#e 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:41df217a:::smithi13140796-39:4#e is 958 (412 bytes onode + 2 bytes spanning blobs + 544 bytes inline extents) 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore.extentmap(0x558c5793a388) update 4#1:41df217a:::smithi13140796-39:head#f 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:41df217a:::smithi13140796-39:head#f is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.672+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:06.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:06.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=27'12 lcod 29'13 active mbc={} ps=[2~1]] op_applied version 29'15 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.672+0000 7f8f03980700 10 osd.7 29 dequeue_op 0x558c578d2b00 finish 2019-12-07T19:34:06.672+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 668 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3482 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793a000 had 1 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793a280 had 1 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12c80 had 1 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3505 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3505 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x7fd64~db1 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x7fd64 0x7fd64~db1 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:05.659297+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _allocate len 0xb15 from 0 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _allocate len 0xb15 from 1 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd1b0000 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd1b0000~10000/10000,10000,acd1b0000 2019-12-07T19:34:06.672+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x90000 to include allocated 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 30 (was clean) 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x4fd64 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd64 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb15 and padding block with 0x4eb 2019-12-07T19:34:06.672+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x90000 2019-12-07T19:34:06.673+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] finish_checksum 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x80b15 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (30) on file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]), flushing log 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 30 len 0x49 crc 0xbb5cb35c) 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x1d000~1000 to file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x1d000 0x1d000~1000 to file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x1e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1d000 2019-12-07T19:34:06.673+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 0x558c54e04700 2019-12-07T19:34:06.673+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b] ov 21'11 2019-12-07T19:34:06.673+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.673+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b] 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x1e000 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:06.673+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op read 0~901120 out=901120b 2019-12-07T19:34:06.673+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:06.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000023 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.673+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx alloc reply 0x558c54e13680 result 0 2019-12-07T19:34:06.673+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:06.673+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] log_op_stats osd_op(client.4540.0:239 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e28) v8 inb 0 outb 901128 lat 1.016133 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 30 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.673+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.673+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:77 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000366756s (0.000000132s flush + 0.000366624s kv commit) 2019-12-07T19:34:06.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:34:06.673+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(239 smithi13140796-2 [read 0~901120 out=901120b,checksum out=8b] v0'0 uv11 ondisk = 0) v8 -- 0x558c54e13680 con 0x558c54c9f000 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.673+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:06.673+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 0x558c5793a000,0x558c5793a280 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:06.673+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:06.673+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.673+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.673+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c578d2420 finish 2019-12-07T19:34:06.673+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 -- 0x558c5768b200 con 0x558c54c64400 2019-12-07T19:34:06.673+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.673+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=29) prio 5 cost 1048576 e29) pg 0x558c57676000 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] snap_trimmer posting 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 4 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.673+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:06.673+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 78 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 225444+0+0 (unknown 0 0 0) 0x558c57572780 con 0x558c54c5b400 2019-12-07T19:34:06.673+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c57335a20 prio 127 cost 0 latency 0.000012 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.673+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.673+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c575c86c0) upper_bound after SNA_1_0000000000000004_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.C' 2019-12-07T19:34:06.673+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c575c86c0) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..' 2019-12-07T19:34:06.673+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000004_.0_0000000000000001.C) returns 0 SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239550 in raw 0x558c55239550 len 70 nref 1) ) 2019-12-07T19:34:06.673+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.673+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:06.673+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c575c86c0) upper_bound after SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. key 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..' 2019-12-07T19:34:06.673+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c575c86c0) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47..' 2019-12-07T19:34:06.673+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..) returns 0 SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239550 in raw 0x558c55239550 len 70 nref 1) ) 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: obc NOT found in cache: 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# _ 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# _ = 0 2019-12-07T19:34:06.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# snapset 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# snapset = 0 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] populate_obc_watchers 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# = 0 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: creating obc from disk: 0x558c57557d40 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: 0x558c57557d40 1:3a5a3f5b:::smithi13140796-23:4 rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:4(27'13 client.4540.0:68 dirty|data_digest s 376832 uv 8 dd 11abb9ec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 4=[]:{4=[4]} 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: obc NOT found in cache: 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:06.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# _ 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# _ = 0 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] populate_obc_watchers 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:06.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# = 0 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: creating obc from disk: 0x558c57557b00 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(27'16 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 4=[]:{4=[4]} 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:3a5a3f5b:::smithi13140796-23:4 old_snaps 4 old snapset 4=[]:{4=[4]} 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] simple_opc_create 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:3a5a3f5b:::smithi13140796-23:4 snaps 4 -> ... deleting 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:3a5a3f5b:::smithi13140796-23:4 new snapset 4=[]:{} on 1:3a5a3f5b:::smithi13140796-23:head(27'16 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:3a5a3f5b:::smithi13140796-23:4 writing updated snapset on 1:3a5a3f5b:::smithi13140796-23:head, snapset is 4=[]:{} 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] new_repop rep_tid 8 (no op) 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] new_repop: repgather(0x558c576b6360 0'0 rep_tid=8 committed?=0 r=0) 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] simple_opc_submit 0x558c576b6360 2019-12-07T19:34:06.673+0000 7f8f06986700 7 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] issue_repop rep_tid 8 o 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:4 v=29'18 tt=0'0 tid=8 reqid=osd.7.0:8 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: found on disk, size 94208 2019-12-07T19:34:06.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# hinfo_key 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# hinfo_key = 0 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:4 projected size 0 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:06.673+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:06.673+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 1474560 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:4 v=29'18 tt=0'0 tid=8 reqid=osd.7.0:8 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:4=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:4 v=29'18 tt=0'0 tid=8 reqid=osd.7.0:8 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:4=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:06.673+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:4 v=29'18 tt=0'0 tid=8 reqid=osd.7.0:8 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:4=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.673+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:4, current size is 376832 buffers are {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): new_size start 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_overwrite: {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_append: {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:4 resetting hinfo to logical size 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 1474560 buffers are {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): new_size start 1474560 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_overwrite: {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_append: {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 1474560 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:4={},1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:4 v=29'18 tt=0'0 tid=8 reqid=osd.7.0:8 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:4=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:4=[],1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:4,{} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:34:06.674+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 29/28 ECSubWrite(tid=8, reqid=osd.7.0:8, at_version=29'18, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c575c1000 con 0x558c54c64000 2019-12-07T19:34:06.674+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 29/28 ECSubWrite(tid=8, reqid=osd.7.0:8, at_version=29'18, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c5748f000 con 0x558c54c19800 2019-12-07T19:34:06.674+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 29/28 ECSubWrite(tid=8, reqid=osd.7.0:8, at_version=29'18, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c5748d800 con 0x558c54c5bc00 2019-12-07T19:34:06.674+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 29/28 ECSubWrite(tid=8, reqid=osd.7.0:8, at_version=29'18, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c5748c000 con 0x558c54c5b400 2019-12-07T19:34:06.674+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 29/28 ECSubWrite(tid=8, reqid=osd.7.0:8, at_version=29'18, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c5748f800 con 0x558c54c19c00 2019-12-07T19:34:06.674+0000 7f8f06986700 20 snap_mapper.remove_oid 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.674+0000 7f8f06986700 20 snap_mapper._remove_oid 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:4 4 2019-12-07T19:34:06.674+0000 7f8f06986700 20 snap_mapper.clear_snaps 1:3a5a3f5b:::smithi13140796-23:4 2019-12-07T19:34:06.674+0000 7f8f06986700 20 snap_mapper.clear_snaps rm OBJ_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:06.674+0000 7f8f06986700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:06.674+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 trim 29'17 (27'13) delete 1:3a5a3f5b:::smithi13140796-23:4 by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 trim 29'18 (27'16) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 27'16 (0'0,27'16] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log log((0'0,27'16], crt=27'16) [29'17 (27'13) delete 1:3a5a3f5b:::smithi13140796-23:4 by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'18 (27'16) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'17 (0'0,29'17] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 lua=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] add_log_entry 29'17 (27'13) delete 1:3a5a3f5b:::smithi13140796-23:4 by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 lua=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] add_log_entry 29'18 (27'16) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 lua=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log approx pg log length = 18 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 lua=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.674+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.674+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'17, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea1b00 seq 18 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:4#11 key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0x00000000000000040000000000000011'o' 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# -> 0#1:3a5a3f5b:::smithi13140796-23:4#11 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4# -> 0#1:3a5a3f5b:::smithi13140796-23:4#11 = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23..' <- SNA_1_0000000000000004_.0_0000000000000001.C5A5CFAD.4.smithi13140796-23.. 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000029.00000000000000000017' <- 0000000029.00000000000000000017 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000029.00000000000000000018' <- 0000000029.00000000000000000018 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 7531 (1 ios * 4000 + 3531 bytes) 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes 0x558c574a1180,0x558c5793aa00 shared_blobs 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1288) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1475 (485 bytes onode + 2 bytes spanning blobs + 988 bytes inline extents) 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore.extentmap(0x558c5793ab08) update 0#1:3a5a3f5b:::smithi13140796-23:4#11 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:4#11 is 742 (412 bytes onode + 2 bytes spanning blobs + 328 bytes inline extents) 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 lua=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] op_applied version 29'18 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4662 < min_flush_size 524288 2019-12-07T19:34:06.674+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 121 ==== MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 190744+0+0 (crc 0 0 0) 0x558c5748f800 con 0x558c54c64000 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] eval_repop repgather(0x558c576b6360 29'18 rep_tid=8 committed?=0 r=0) (no op) 2019-12-07T19:34:06.674+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c57335ce0 prio 127 cost 0 latency 0.000026 epoch 29 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1180 had 1 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793aa00 had 1 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57977180 had 1 2019-12-07T19:34:06.674+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4685 < min_flush_size 524288 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] calc_trim_to_aggressive limit = 27'16 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4685 < min_flush_size 524288 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x80b15~124d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x80b15 0x80b15~124d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x50b15 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb15 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd62 and padding block with 0x29e 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] enter Trimming/WaitRepops 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] snap_trimmer complete 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.674+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.674+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558c5500a000 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2000 prio 127 cost 0 latency 0.002707 MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 osd.7 29 dequeue_op 0x558c57335ce0 prio 127 cost 0 latency 0.000107 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpRead(1.cs0 29/28 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,225280,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.674+0000 7f8f06986700 25 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:06.674+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~37000 size 0x37000 (225280) 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] append_log log((0'0,27'8], crt=27'8) [29'9 (21'6) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:244 2019-12-07T19:34:05.664687+0000 0 ObjectCleanRegions clean_offsets: [0~212992,966656~18446744073708584959], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] add_log_entry 29'9 (21'6) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:244 2019-12-07T19:34:05.664687+0000 0 ObjectCleanRegions clean_offsets: [0~212992,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log approx pg log length = 9 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.674+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5730a300 seq 25 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2 keys 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2 keys = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:06.674+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~37000 = 225280 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read read request=225280 r=225280 len=225280 2019-12-07T19:34:06.674+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd000~2e000 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9da56077:::smithi13140796-13:head# 0xd000~2e000 - have 0xd000 (53248) bytes fadvise_flags 0x20 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.674+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0xd000~2e000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:06.674+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:06.674+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730a300 23 blobs 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read: Checking hash of 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 -- 0x558c57977400 con 0x558c53ae5400 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2000 finish 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c573358c0 prio 127 cost 0 latency 0.002237 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x81d62 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:34:06.674+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000415193s (0.000000107s flush + 0.000415086s kv commit) 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1b00> 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 0x558c574a1180,0x558c5793aa00 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:34:06.674+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.674+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2e000/1000,2e000,0 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),7(0)) 2019-12-07T19:34:06.674+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c573358c0 finish 2019-12-07T19:34:06.674+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x444000~2e000/1000,2e000,0 2019-12-07T19:34:06.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x444000~2e000] 2019-12-07T19:34:06.674+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d755e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d755e0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c577f2d20) get_ref 0x0~2000 Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d755e0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c577f2d20) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xd000~2000: 0x0~2000 Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d755f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75b20 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c576ac770) get_ref 0x0~2000 Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75b20 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c576ac770) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf000~2000: 0x0~2000 Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d75b38 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beae00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beae00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c55265f80) get_ref 0x0~2000 Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beae00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c55265f80) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x11000~2000: 0x0~2000 Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beae18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beaee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beaee0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beae70) get_ref 0x0~2000 Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beaee0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beae70) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x13000~2000: 0x0~2000 Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beaef8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beaf50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beafc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beafc0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beaf50) get_ref 0x0~2000 Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beafc0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beaf50) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x15000~2000: 0x0~2000 Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beafd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb0a0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb030) get_ref 0x0~2000 Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb0a0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb030) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x17000~2000: 0x0~2000 Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb0b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb180 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb110) get_ref 0x0~2000 Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb180 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb110) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x19000~2000: 0x0~2000 Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb198 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb260 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb1f0) get_ref 0x0~2000 Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb260 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb278 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb340 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb2d0) get_ref 0x0~2000 Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb340 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb358 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb420 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb3b0) get_ref 0x0~2000 Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb420 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb438 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb500 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb490) get_ref 0x0~2000 Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb500 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb490) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb518 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb5e0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb570) get_ref 0x0~2000 Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb5e0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb570) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb5f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb6c0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb650) get_ref 0x0~2000 Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb6c0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb650) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb6d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb7a0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb730) get_ref 0x0~2000 Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb7a0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb730) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb7b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb880 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb810) get_ref 0x0~2000 Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb880 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb810) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c57335a20 prio 127 cost 0 latency 0.001855 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb898 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beb960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb960 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb8f0) get_ref 0x0~2000 Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beb960 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb978 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57beb9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beba40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beba40 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [4(2),225280, 5(1),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),7(0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb9d0) get_ref 0x0~2000 Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beba40 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57beb9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c57335a20 finish 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beba58 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bebab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bebb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebb20 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebab0) get_ref 0x0~2000 Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebb20 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebab0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebb38 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bebb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bebc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebc00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebb90) get_ref 0x0~2000 Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebc00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebb90) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebc18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bebc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bebce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebce0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebc70) get_ref 0x0~2000 Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebce0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebc70) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebcf8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bebd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bebdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebdc0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebd50) get_ref 0x0~2000 Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebdc0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebd50) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebdd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bebe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bebea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebea0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebe30) get_ref 0x0~2000 Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebea0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebe30) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebeb8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bebf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bebf80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebf80 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebf10) get_ref 0x0~2000 Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bebf80 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.blob(0x558c57bebf10) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebf98 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3b000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2888) dirty_range mark inline shard dirty 2019-12-07T19:34:06.675+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd000~2e000 = 0 2019-12-07T19:34:06.675+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.675+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.675+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000029.00000000000000000009' <- 0000000029.00000000000000000009 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.675+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.675+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a300 cost 287184 (24 ios * 4000 + 191184 bytes) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a300 onodes 0x558c54ce2780 shared_blobs 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2888) update 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2888) update inline shard 571 bytes from 30 extents 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9da56077:::smithi13140796-13:head# is 1062 (485 bytes onode + 2 bytes spanning blobs + 575 bytes inline extents) 2019-12-07T19:34:06.675+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a300 allocated 0x[444000~2e000] released 0x[] 2019-12-07T19:34:06.675+0000 7f8f08189700 10 freelist allocate 0x444000~2e000 2019-12-07T19:34:06.675+0000 7f8f08189700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:06.675+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 prepare 2019-12-07T19:34:06.675+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730a300 2019-12-07T19:34:06.675+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c57977400 con 0x558c53ae5400 2019-12-07T19:34:06.675+0000 7f8f1119b700 15 osd.7 29 enqueue_op 0x558c578d2160 prio 127 cost 0 latency 0.000607 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.675+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.675+0000 7f8f08189700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] op_applied version 29'9 2019-12-07T19:34:06.675+0000 7f8f08189700 10 osd.7 29 dequeue_op 0x558c57335ce0 finish 2019-12-07T19:34:06.675+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.675+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 81 ==== MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=85, reqid=osd.3.0:85, at_version=29'24, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2640+0+0 (unknown 0 0 0) 0x558c57dfe000 con 0x558c54c5bc00 2019-12-07T19:34:06.675+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d2c60 prio 127 cost 0 latency 0.000039 epoch 29 MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=85, reqid=osd.3.0:85, at_version=29'24, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.675+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2160 prio 127 cost 0 latency 0.000746 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.675+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 aio_wait 2019-12-07T19:34:06.675+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=85, reqid=osd.3.0:85, at_version=29'24, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.675+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a300 2019-12-07T19:34:06.675+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.675+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 io_done 2019-12-07T19:34:06.675+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:06.675+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [4(2),225280, 5(1),225280, 7(0),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3)) 2019-12-07T19:34:06.675+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2160 finish 2019-12-07T19:34:06.675+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=85, reqid=osd.3.0:85, at_version=29'24, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.675+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=85, reqid=osd.3.0:85, at_version=29'24, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.675+0000 7f8f05984700 10 osd.7 29 dequeue_op 0x558c578d2c60 prio 127 cost 0 latency 0.000097 MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=85, reqid=osd.3.0:85, at_version=29'24, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.6s2( v 29'22 (0'0,29'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.675+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'22 (0'0,29'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=85, reqid=osd.3.0:85, at_version=29'24, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.675+0000 7f8f05984700 20 snap_mapper.remove_oid 1:6ee4dfe7:::smithi13140796-47:4 2019-12-07T19:34:06.675+0000 7f8f05984700 20 snap_mapper._remove_oid 1:6ee4dfe7:::smithi13140796-47:4 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2992 < min_flush_size 524288 2019-12-07T19:34:06.675+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.675+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.675+0000 7f8f05984700 20 snap_mapper.get_snaps 1:6ee4dfe7:::smithi13140796-47:4 4 2019-12-07T19:34:06.675+0000 7f8f05984700 20 snap_mapper.clear_snaps 1:6ee4dfe7:::smithi13140796-47:4 2019-12-07T19:34:06.675+0000 7f8f05984700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:06.675+0000 7f8f05984700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2780 had 1 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3015 < min_flush_size 524288 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3015 < min_flush_size 524288 2019-12-07T19:34:06.675+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'22 (0'0,29'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] append_log log((0'0,29'22], crt=27'20) [29'23 (27'16) delete 1:6ee4dfe7:::smithi13140796-47:4 by unknown.0.0:0 2019-12-07T19:34:06.670080+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'24 (27'20) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:06.670080+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x81d62~bc7 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x81d62 0x81d62~bc7 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x51d62 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd62 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x929 and padding block with 0x6d7 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x82929 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.675+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.675+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'23 (0'0,29'23] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'22 crt=27'20 lcod 29'22 active mbc={} ps=[2~1]] add_log_entry 29'23 (27'16) delete 1:6ee4dfe7:::smithi13140796-47:4 by unknown.0.0:0 2019-12-07T19:34:06.670080+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000163492s (0.000048144s flush + 0.000115348s kv commit) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.675+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'22 crt=27'20 lcod 29'22 active mbc={} ps=[2~1]] add_log_entry 29'24 (27'20) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:06.670080+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a300> 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.675+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'22 crt=27'20 lcod 29'22 active mbc={} ps=[2~1]] append_log approx pg log length = 24 2019-12-07T19:34:06.675+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 kv_submitted 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a300 2019-12-07T19:34:06.675+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'22 crt=27'20 lcod 29'22 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.675+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.675+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 finishing 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a300 onodes 0x558c54ce2780 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d755f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54c9a1b0 space 0x558c54d755f8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75b38 in 0x558c53a59a40) _finish_write discard buffer(0x558c577ffc20 space 0x558c54d75b38 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beae18 in 0x558c53a59a40) _finish_write discard buffer(0x558c577ffcb0 space 0x558c57beae18 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beaef8 in 0x558c53a59a40) _finish_write discard buffer(0x558c577ffd40 space 0x558c57beaef8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beafd8 in 0x558c53a59a40) _finish_write discard buffer(0x558c577ffdd0 space 0x558c57beafd8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb0b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c577ffe60 space 0x558c57beb0b8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb198 in 0x558c53a59a40) _finish_write discard buffer(0x558c577ffef0 space 0x558c57beb198 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb278 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e14e10 space 0x558c57beb278 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'23, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb358 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e14ea0 space 0x558c57beb358 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb438 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e14f30 space 0x558c57beb438 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb518 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e14fc0 space 0x558c57beb518 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb5f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15050 space 0x558c57beb5f8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb6d8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e150e0 space 0x558c57beb6d8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb7b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15170 space 0x558c57beb7b8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb898 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15200 space 0x558c57beb898 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beb978 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15290 space 0x558c57beb978 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beba58 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15320 space 0x558c57beba58 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bebb38 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e153b0 space 0x558c57bebb38 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bebc18 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15440 space 0x558c57bebc18 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bebcf8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e154d0 space 0x558c57bebcf8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bebdd8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15560 space 0x558c57bebdd8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bebeb8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e155f0 space 0x558c57bebeb8 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bebf98 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15680 space 0x558c57bebf98 0x0~2000 writing nocache) 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a300 done 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.675+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a300 [] 2019-12-07T19:34:06.675+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.675+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=135, last_complete=29'9, committed=1, applied=1)) v2 -- 0x558c5501d600 con 0x558c54c64000 2019-12-07T19:34:06.675+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f7b00 seq 46 2019-12-07T19:34:06.675+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:4#17 key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0x00000000000000040000000000000017'o' 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:06.675+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# -> 2#1:6ee4dfe7:::smithi13140796-47:4#17 2019-12-07T19:34:06.675+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4# -> 2#1:6ee4dfe7:::smithi13140796-47:4#17 = 0 2019-12-07T19:34:06.675+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:06.675+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:06.675+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.675+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.675+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6772BF7E.4.smithi13140796-47..' <- OBJ_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:06.675+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.675+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47..' <- SNA_1_0000000000000004_.2_0000000000000001.6772BF7E.4.smithi13140796-47.. 2019-12-07T19:34:06.675+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.675+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000029.00000000000000000023' <- 0000000029.00000000000000000023 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000029.00000000000000000024' <- 0000000029.00000000000000000024 2019-12-07T19:34:06.675+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.676+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:06.676+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 7515 (1 ios * 4000 + 3515 bytes) 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes 0x558c54ce3400,0x558c575ba500 shared_blobs 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3508) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1142 (485 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba608) update 2#1:6ee4dfe7:::smithi13140796-47:4#17 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:4#17 is 1069 (412 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.676+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:06.676+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:06.676+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.676+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4640 < min_flush_size 524288 2019-12-07T19:34:06.676+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'22 crt=27'20 lcod 29'22 active mbc={} ps=[2~1]] op_applied version 29'24 2019-12-07T19:34:06.676+0000 7f8f05984700 10 osd.7 29 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:06.676+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3400 had 1 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba500 had 1 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2000 had 1 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4663 < min_flush_size 524288 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4663 < min_flush_size 524288 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x82929~1237 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x82929 0x82929~1237 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x52929 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x929 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb60 and padding block with 0x4a0 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x83b60 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.676+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112301s (0.000000070s flush + 0.000112231s kv commit) 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7b00> 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:06.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 0x558c54ce3400,0x558c575ba500 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:06.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:06.676+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.676+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.676+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 29/28 ECSubWriteReply(tid=85, last_complete=29'24, committed=1, applied=1)) v2 -- 0x558c576b5e00 con 0x558c54c5bc00 2019-12-07T19:34:06.676+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.676+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 122 ==== MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 225444+0+0 (crc 0 0 0) 0x558c54e13680 con 0x558c54c64000 2019-12-07T19:34:06.676+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740d340 prio 127 cost 0 latency 0.000014 epoch 29 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.676+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740d340 prio 127 cost 0 latency 0.000047 MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 29/28 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,901120,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 901120, [1(3),225280, 4(2),225280, 5(1),225280, 7(0),225280]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,7(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=) 2019-12-07T19:34:06.676+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 79 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c1e600 con 0x558c54c5b400 2019-12-07T19:34:06.676+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740d4a0 prio 196 cost 0 latency 0.000011 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.676+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.676+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] objects_read_async_cb: got: {1:33f5bf01:::smithi13140796-2:head=0,{0~901120(901120)}} 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:06.676+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 123 ==== MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 96536+0+0 (crc 0 0 0) 0x558c575c1000 con 0x558c54c64000 2019-12-07T19:34:06.676+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740c160 prio 127 cost 0 latency 0.000020 epoch 29 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.676+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.676+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558c5500a000 2019-12-07T19:34:06.676+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c5740c160 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.676+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.676+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] append_log log((0'0,29'9], crt=27'8) [29'10 (29'9) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:245 2019-12-07T19:34:05.667234+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.676+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 29'9 active mbc={} ps=[2~1]] add_log_entry 29'10 (29'9) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:245 2019-12-07T19:34:05.667234+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.676+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 29'9 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:06.676+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 29'9 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.676+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.676+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5730b800 seq 26 2019-12-07T19:34:06.676+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2 keys 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2 keys = 0 2019-12-07T19:34:06.676+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:06.676+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b000~17000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9da56077:::smithi13140796-13:head# 0x3b000~17000 - have 0x3b000 (241664) bytes fadvise_flags 0x20 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x3b000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730b800 12 blobs 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.676+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0x17000/1000,17000,0 2019-12-07T19:34:06.676+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x472000~17000/1000,17000,0 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x472000~17000] 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2d90 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f2e70) get_ref 0x0~2000 Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2d90 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 39 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54cea800 con 0x558c54c19c00 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f2e70) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c577f2da8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740d080 prio 196 cost 0 latency 0.000012 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.676+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2e00 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f2ee0) get_ref 0x0~2000 Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2e00 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f2ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] finish_checksum 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c577f2e18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f3030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3030 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f2fc0) get_ref 0x0~2000 Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3030 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f2fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c577f3048 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f30a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3110 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f30a0) get_ref 0x0~2000 Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3110 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f30a0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c577f3128 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f31f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f31f0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3180) get_ref 0x0~2000 Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f31f0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3180) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c577f3208 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f32d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f32d0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3260) get_ref 0x0~2000 Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f32d0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3260) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c577f32e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f33b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f33b0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3340) get_ref 0x0~2000 Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f33b0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3340) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c577f33c8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57880fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880fc0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3420) get_ref 0x0~2000 Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880fc0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c577f3420) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57880fd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcacb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcad90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcad90 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcacb0) get_ref 0x0~2000 Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcad90 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcacb0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcada8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcae00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcae70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcae70 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcae00) get_ref 0x0~2000 Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcae70 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcae00) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcae88 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcaee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcaf50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 0x558c54e05c00 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcaf50 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcaee0) get_ref 0x0~2000 Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcaf50 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcaee0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcaf68 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcafc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb030 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcafc0) get_ref 0x1000~1000 Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb030 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.blob(0x558c57dcafc0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x51000~1000: 0x1000~1000 Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcb048 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b] ov 21'11 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x52000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) dirty_range mark inline shard dirty 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b000~17000 = 0 2019-12-07T19:34:06.676+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b] 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.676+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op read 0~901120 out=901120b 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000029.00000000000000000010' <- 0000000029.00000000000000000010 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b800 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b800 onodes 0x558c54ce2780 shared_blobs 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) update 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) update inline shard 812 bytes from 42 extents 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9da56077:::smithi13140796-13:head# is 1303 (485 bytes onode + 2 bytes spanning blobs + 816 bytes inline extents) 2019-12-07T19:34:06.676+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b800 allocated 0x[472000~17000] released 0x[] 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] execute_ctx alloc reply 0x558c54e13900 result 0 2019-12-07T19:34:06.676+0000 7f8f04181700 10 freelist allocate 0x472000~17000 2019-12-07T19:34:06.676+0000 7f8f04181700 20 freelist _xor first_key 0x400000 last_key 0x480000 2019-12-07T19:34:06.676+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 prepare 2019-12-07T19:34:06.676+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730b800 2019-12-07T19:34:06.676+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] log_op_stats osd_op(client.4540.0:240 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~901120 out=901120b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e28) v8 inb 0 outb 901128 lat 1.019838 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.676+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:78 2019-12-07T19:34:06.676+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(240 smithi13140796-2 [read 0~901120 out=901120b,checksum out=8b] v0'0 uv11 ondisk = 0) v8 -- 0x558c54e13900 con 0x558c54c9f000 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740d340 finish 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.676+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740d4a0 prio 196 cost 0 latency 0.000511 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.677+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.677+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740d4a0 finish 2019-12-07T19:34:06.677+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 29'9 active mbc={} ps=[2~1]] op_applied version 29'10 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c5740c160 finish 2019-12-07T19:34:06.677+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c5740d080 prio 196 cost 0 latency 0.000305 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.677+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.677+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:06.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 aio_wait 2019-12-07T19:34:06.677+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b800 2019-12-07T19:34:06.677+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 io_done 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2472 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2780 had 1 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2495 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2495 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x83b60~9bf to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x83b60 0x83b60~9bf to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x53b60 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb60 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x51f and padding block with 0xae1 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8451f 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000162142s (0.000052600s flush + 0.000109542s kv commit) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b800> 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.677+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 kv_submitted 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b800 2019-12-07T19:34:06.677+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 finishing 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b800 onodes 0x558c54ce2780 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2da8 in 0x558c53a59a40) _finish_write discard buffer(0x558c577ffb90 space 0x558c577f2da8 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2e18 in 0x558c53a59a40) _finish_write discard buffer(0x558c5796fef0 space 0x558c577f2e18 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3048 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15c20 space 0x558c577f3048 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3128 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15cb0 space 0x558c577f3128 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3208 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15d40 space 0x558c577f3208 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f32e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15dd0 space 0x558c577f32e8 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f33c8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15e60 space 0x558c577f33c8 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880fd8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15ef0 space 0x558c57880fd8 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcada8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28000 space 0x558c57dcada8 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcae88 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28090 space 0x558c57dcae88 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcaf68 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28120 space 0x558c57dcaf68 0x0~2000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcb048 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e281b0 space 0x558c57dcb048 0x1000~1000 writing nocache) 2019-12-07T19:34:06.677+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=136, last_complete=29'10, committed=1, applied=1)) v2 -- 0x558c5501de00 con 0x558c54c64000 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 done 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.677+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b800 [] 2019-12-07T19:34:06.677+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.677+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.677+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 124 ==== MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2294+0+0 (crc 0 0 0) 0x558c54d86000 con 0x558c54c64000 2019-12-07T19:34:06.677+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000030 epoch 29 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.677+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.677+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.677+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558c5500a000 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000103 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] append_log log((0'0,29'10], crt=27'8) [29'11 (29'10) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:246 2019-12-07T19:34:05.667270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 29'10 active mbc={} ps=[2~1]] add_log_entry 29'11 (29'10) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:246 2019-12-07T19:34:05.667270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 29'10 active mbc={} ps=[2~1]] append_log approx pg log length = 11 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 29'10 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.677+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.677+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.677+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5730bb00 seq 27 2019-12-07T19:34:06.677+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 3 keys 2019-12-07T19:34:06.677+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 3 keys = 0 2019-12-07T19:34:06.677+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.677+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.677+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000029.00000000000000000011' <- 0000000029.00000000000000000011 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.677+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.677+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730bb00 cost 5944 (1 ios * 4000 + 1944 bytes) 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730bb00 onodes 0x558c54ce2780 shared_blobs 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2888) update 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9da56077:::smithi13140796-13:head# is 1303 (485 bytes onode + 2 bytes spanning blobs + 816 bytes inline extents) 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.677+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 prepare 2019-12-07T19:34:06.677+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730bb00 2019-12-07T19:34:06.677+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 io_done 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 29'10 active mbc={} ps=[2~1]] op_applied version 29'11 2019-12-07T19:34:06.677+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2403 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2780 had 1 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2426 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2426 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8451f~97a to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8451f 0x8451f~97a to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5451f 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x51f 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe99 and padding block with 0x167 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x84e99 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.677+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.677+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000100855s (0.000000103s flush + 0.000100752s kv commit) 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730bb00> 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.678+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.678+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 kv_submitted 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730bb00 2019-12-07T19:34:06.678+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 finishing 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730bb00 onodes 0x558c54ce2780 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730bb00 done 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.678+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730bb00 [] 2019-12-07T19:34:06.678+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.678+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.678+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.678+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=137, last_complete=29'11, committed=1, applied=1)) v2 -- 0x558c54c1fe00 con 0x558c54c64000 2019-12-07T19:34:06.678+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 45 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57569c00 con 0x558c54c19800 2019-12-07T19:34:06.678+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.678+0000 7f8f24c58700 15 osd.7 29 enqueue_op 0x558c54e78dc0 prio 196 cost 0 latency 0.000014 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.678+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.678+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e78dc0 prio 196 cost 0 latency 0.000101 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.678+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.678+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:34:06.678+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.678+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 82 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c576b5e00 con 0x558c54c5bc00 2019-12-07T19:34:06.678+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d2dc0 prio 196 cost 0 latency 0.000014 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.678+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.678+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2dc0 prio 196 cost 0 latency 0.000135 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.678+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.678+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2dc0 finish 2019-12-07T19:34:06.678+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.680+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 439464 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.681+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 125 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c1fe00 con 0x558c54c64000 2019-12-07T19:34:06.681+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c578d3600 prio 196 cost 0 latency 0.000017 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.681+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c578d3600 prio 196 cost 0 latency 0.000078 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=8, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:4 v=29'18 tt=0'0 tid=8 reqid=osd.7.0:8 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:4=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=27'16 crt=27'16 lcod 27'16 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] repop_all_committed: repop tid 8 all committed 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] eval_repop repgather(0x558c576b6360 29'18 rep_tid=8 committed?=1 r=0) (no op) 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] commit: repgather(0x558c576b6360 29'18 rep_tid=8 committed?=1 r=0) 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.681+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:79 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] removing repgather(0x558c576b6360 29'18 rep_tid=8 committed?=1 r=0) 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] q front is repgather(0x558c576b6360 29'18 rep_tid=8 committed?=1 r=0) 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] exit Trimming/WaitRepops 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 29 queueing pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] for snaptrim 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=29) prio 5 cost 1048576 e29) 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.681+0000 7f8f0297e700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:80 2019-12-07T19:34:06.681+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] remove_repop repgather(0x558c576b6360 29'18 rep_tid=8 committed?=1 r=0) 2019-12-07T19:34:06.681+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=29) prio 5 cost 1048576 e29) queued 2019-12-07T19:34:06.681+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:4 v=29'18 tt=0'0 tid=8 reqid=osd.7.0:8 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:4=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:06.681+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:4 v=0'0 tt=0'0 tid=9 reqid=osd.7.0:8 roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:4 v=0'0 tt=0'0 tid=9 reqid=osd.7.0:8 roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:06.682+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 29/28 ECSubWrite(tid=9, reqid=osd.7.0:8, at_version=0'0, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c575c0800 con 0x558c54c64000 2019-12-07T19:34:06.682+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 29/28 ECSubWrite(tid=9, reqid=osd.7.0:8, at_version=0'0, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c551dd800 con 0x558c54c19800 2019-12-07T19:34:06.682+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 29/28 ECSubWrite(tid=9, reqid=osd.7.0:8, at_version=0'0, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c574a4800 con 0x558c54c5bc00 2019-12-07T19:34:06.682+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 29/28 ECSubWrite(tid=9, reqid=osd.7.0:8, at_version=0'0, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c574a2800 con 0x558c54c5b400 2019-12-07T19:34:06.682+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 29/28 ECSubWrite(tid=9, reqid=osd.7.0:8, at_version=0'0, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c574a3000 con 0x558c54c19c00 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=27'16 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log log((0'0,29'18], crt=27'16) [] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] rollforward: entry=29'17 (27'13) delete 1:3a5a3f5b:::smithi13140796-23:4 by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] rollforward: entry=29'18 (27'16) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:06.674760+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log approx pg log length = 18 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.682+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c548f6300 seq 19 2019-12-07T19:34:06.682+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4#11 onode 0x558c5793aa00 txc 0x558c548f6300 2019-12-07T19:34:06.682+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4#11 0x0 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c574f1ea0) put_ref 0x0~2000 Blob(0x558c574f1ea0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c57396700) put_ref 0x0~2000 Blob(0x558c57396700 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c573968c0) put_ref 0x0~2000 Blob(0x558c573968c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c57396310) put_ref 0x0~2000 Blob(0x558c57396310 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cfc0) put_ref 0x0~2000 Blob(0x558c5794cfc0 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d0a0) put_ref 0x0~2000 Blob(0x558c5794d0a0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d110) put_ref 0x0~2000 Blob(0x558c5794d110 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d180) put_ref 0x0~2000 Blob(0x558c5794d180 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d1f0) put_ref 0x0~2000 Blob(0x558c5794d1f0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d260) put_ref 0x0~2000 Blob(0x558c5794d260 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d2d0) put_ref 0x0~2000 Blob(0x558c5794d2d0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d340) put_ref 0x0~1000 Blob(0x558c5794d340 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore.extentmap(0x558c5793ab08) dirty_range mark inline shard dirty 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574f1ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x339000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57396700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33b000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c573968c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33d000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57396310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33f000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c5794cfc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x341000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c5794d0a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x343000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c5794d110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x345000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c5794d180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x347000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5794d1f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x349000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c5794d260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34b000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c5794d2d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34d000~2000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~1000: 0x0~1000 Blob(0x558c5794d340 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34f000~1000] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d75e30,0x558c57396230,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:4#11 = 0 2019-12-07T19:34:06.682+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes shared_blobs 0x558c54d75e30,0x558c57396230,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2851 is 11 SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284a is 11 SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2853 is 11 SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2852 is 11 SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284b is 11 SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2854 is 11 SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2855 is 11 SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284c is 11 SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284d is 11 SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284e is 11 SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284f is 11 SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2850 is 11 SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1))) 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:06.682+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=29) prio 5 cost 1048576 e29) pg 0x558c57676000 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1509 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] snap_trimmer posting 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 4 2019-12-07T19:34:06.682+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.682+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793aa00 had 1 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x84e99~5fc to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x84e99 0x84e99~5fc to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54681b40) upper_bound after SNA_1_0000000000000004_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.C' 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x54e99 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe99 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x495 and padding block with 0xb6b 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54681b40) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3..' 2019-12-07T19:34:06.682+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000004_.0_0000000000000001.C) returns 0 SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239550 in raw 0x558c55239550 len 69 nref 1) ) 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[2~1]] SnapTrimmer state: adding snap 4 to purged_snaps 2019-12-07T19:34:06.682+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x85495 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[2~1,4~1]] SnapTrimmer state: purged_snaps now [2~1,4~1], snap_trimq now [] 2019-12-07T19:34:06.682+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea0c00 seq 20 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123191s (0.000000101s flush + 0.000123090s kv commit) 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.682+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:06.682+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:06.682+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes shared_blobs 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.682+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:06.682+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:06.682+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[2~1,4~1]] share_pg_info 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e04000 con 0x558c54c64000 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x85495~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x85495 0x85495~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x55495 2019-12-07T19:34:06.682+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e06a00 con 0x558c54c19800 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x495 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x902 and padding block with 0x6fe 2019-12-07T19:34:06.682+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e07100 con 0x558c54c5bc00 2019-12-07T19:34:06.682+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c54e07800 con 0x558c54c5b400 2019-12-07T19:34:06.682+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558c57453100 con 0x558c54c19c00 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[2~1,4~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[2~1,4~1]] exit Trimming 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] prepare_stats_for_publish reporting purged_snaps [2~1,4~1] 2019-12-07T19:34:06.682+0000 7f8f06986700 15 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] publish_stats_to_osd 29:81 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] enter NotTrimming 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:06.682+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] snap_trimmer complete 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x85902 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000125500s (0.000000084s flush + 0.000125416s kv commit) 2019-12-07T19:34:06.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300,0x558c54ea0c00> 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:06.682+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:06.682+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.682+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.683+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 126 ==== MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2632+0+0 (crc 0 0 0) 0x558c574a2800 con 0x558c54c64000 2019-12-07T19:34:06.683+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740db80 prio 127 cost 0 latency 0.000039 epoch 29 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.683+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.683+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.683+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558c5500a000 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c5740db80 prio 127 cost 0 latency 0.000073 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.683+0000 7f8f04181700 20 snap_mapper.remove_oid 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:06.683+0000 7f8f04181700 20 snap_mapper._remove_oid 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:06.683+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.683+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:4 4 2019-12-07T19:34:06.683+0000 7f8f04181700 20 snap_mapper.clear_snaps 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:06.683+0000 7f8f04181700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.683+0000 7f8f04181700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'11 (0'0,29'11] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] append_log log((0'0,29'11], crt=27'8) [29'12 (27'7) delete 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'13 (27'8) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 29'11 active mbc={} ps=[2~1]] add_log_entry 29'12 (27'7) delete 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 29'11 active mbc={} ps=[2~1]] add_log_entry 29'13 (27'8) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 29'11 active mbc={} ps=[2~1]] append_log approx pg log length = 13 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 29'11 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.683+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.683+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54836f00 seq 28 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9694a0e5:::smithi13140796-3:4#c key 0x8480000000000000019694a0e521736d'ithi13140796-3!='0x0000000000000004000000000000000c'o' 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:06.683+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# -> 4#1:9694a0e5:::smithi13140796-3:4#c 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4# -> 4#1:9694a0e5:::smithi13140796-3:4#c = 0 2019-12-07T19:34:06.683+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:06.683+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.683+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9692507A.4.smithi13140796-3..' <- OBJ_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.683+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3..' <- SNA_1_0000000000000004_.4_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.683+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000029.00000000000000000012' <- 0000000029.00000000000000000012 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000029.00000000000000000013' <- 0000000029.00000000000000000013 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 7506 (1 ios * 4000 + 3506 bytes) 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes 0x558c54ce2a00,0x558c54d97680 shared_blobs 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 835 (484 bytes onode + 2 bytes spanning blobs + 349 bytes inline extents) 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97788) update 4#1:9694a0e5:::smithi13140796-3:4#c 2019-12-07T19:34:06.683+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 40 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57569200 con 0x558c54c19c00 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:4#c is 469 (409 bytes onode + 2 bytes spanning blobs + 58 bytes inline extents) 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:34:06.683+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:34:06.683+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:34:06.683+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c5740d1e0 prio 196 cost 0 latency 0.000280 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.683+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 29'11 active mbc={} ps=[2~1]] op_applied version 29'13 2019-12-07T19:34:06.683+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c5740db80 finish 2019-12-07T19:34:06.683+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.683+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3725 < min_flush_size 524288 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97680 had 1 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96f00 had 1 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.683+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 80 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5501de00 con 0x558c54c5b400 2019-12-07T19:34:06.683+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c54e79340 prio 196 cost 0 latency 0.000010 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.683+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3748 < min_flush_size 524288 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3748 < min_flush_size 524288 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x85902~ea4 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x85902 0x85902~ea4 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x55902 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x902 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7a6 and padding block with 0x85a 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x867a6 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.683+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000121464s (0.000000074s flush + 0.000121390s kv commit) 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.683+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54836f00> 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.683+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:34:06.683+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 finishing 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 0x558c54ce2a00,0x558c54d97680 2019-12-07T19:34:06.683+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.683+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:34:06.683+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.683+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.683+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=139, last_complete=29'13, committed=1, applied=1)) v2 -- 0x558c54dfa000 con 0x558c54c64000 2019-12-07T19:34:06.683+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.683+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740d1e0 prio 196 cost 0 latency 0.000655 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.683+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.683+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c5740d1e0 finish 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.683+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e79340 prio 196 cost 0 latency 0.000285 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.683+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.683+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e79340 finish 2019-12-07T19:34:06.683+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.683+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 46 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54d77e00 con 0x558c54c19800 2019-12-07T19:34:06.683+0000 7f8f24c58700 15 osd.7 29 enqueue_op 0x558c54e798c0 prio 196 cost 0 latency 0.000022 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.683+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.683+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.683+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.684+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c54e798c0 prio 196 cost 0 latency 0.000045 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.684+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.684+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c54e798c0 finish 2019-12-07T19:34:06.684+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.685+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 127 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57569e00 con 0x558c54c64000 2019-12-07T19:34:06.685+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c54e79e40 prio 196 cost 0 latency 0.000016 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.685+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.685+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.685+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.685+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.685+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.685+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e79e40 prio 196 cost 0 latency 0.000043 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.685+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.685+0000 7f8f0297e700 10 osd.7 29 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:34:06.685+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.685+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 83 ==== MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5768b200 con 0x558c54c5bc00 2019-12-07T19:34:06.685+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d2f20 prio 196 cost 0 latency 0.000015 epoch 29 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.685+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.685+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.685+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.685+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.685+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558c57676000 2019-12-07T19:34:06.685+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2f20 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.685+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 29/28 ECSubWriteReply(tid=9, last_complete=29'18, committed=1, applied=1)) v2 2019-12-07T19:34:06.685+0000 7f8f06986700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:4 v=0'0 tt=0'0 tid=9 reqid=osd.7.0:8 roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:06.685+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:06.685+0000 7f8f06986700 20 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:06.685+0000 7f8f06986700 10 osd.7 29 dequeue_op 0x558c578d2f20 finish 2019-12-07T19:34:06.685+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.687+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 84 ==== MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=87, reqid=osd.3.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=29'24)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c574a4800 con 0x558c54c5bc00 2019-12-07T19:34:06.687+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d3e40 prio 127 cost 0 latency 0.000022 epoch 29 MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=87, reqid=osd.3.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:06.687+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=87, reqid=osd.3.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=87, reqid=osd.3.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=87, reqid=osd.3.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 osd.7 29 dequeue_op 0x558c578d3e40 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=87, reqid=osd.3.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=29'24)) v2 pg pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s2 29/28 ECSubWrite(tid=87, reqid=osd.3.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=27'20 active mbc={} ps=[2~1]] append_log log((0'0,29'24], crt=27'20) [] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] rollforward: entry=29'21 (27'14) delete 1:6c205177:::smithi13140796-8:4 by unknown.0.0:0 2019-12-07T19:34:06.669023+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] rollforward: entry=29'22 (27'15) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:06.669023+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] rollforward: entry=29'23 (27'16) delete 1:6ee4dfe7:::smithi13140796-47:4 by unknown.0.0:0 2019-12-07T19:34:06.670080+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] rollforward: entry=29'24 (27'20) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:06.670080+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] append_log approx pg log length = 24 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.687+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c574fe000 seq 47 2019-12-07T19:34:06.687+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:4#15 onode 0x558c54ce3680 txc 0x558c574fe000 2019-12-07T19:34:06.687+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:4#15 0x0 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75260) put_ref 0x0~2000 Blob(0x558c54d75260 blob([0x4e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c552656c0) put_ref 0x0~2000 Blob(0x558c552656c0 blob([0x50000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c57314620) put_ref 0x0~2000 Blob(0x558c57314620 blob([0x52000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) put_ref 0x0~2000 Blob(0x558c575bd6c0 blob([0x54000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3788) dirty_range mark inline shard dirty 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d75260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map(0x4e000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4e000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x4e000~2000] from SharedBlob(0x558c54d75ab0 loaded (sbid 0x2818 ref_map())) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x4e000~2000 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c552656c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map(0x50000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x50000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x50000~2000] from SharedBlob(0x558c54d753b0 loaded (sbid 0x2819 ref_map())) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x50000~2000 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57314620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map(0x52000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x52000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x52000~2000] from SharedBlob(0x558c54cf8000 loaded (sbid 0x281a ref_map())) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x52000~2000 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map(0x54000~2000=1)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x54000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x54000~2000] from SharedBlob(0x558c54cf80e0 loaded (sbid 0x281b ref_map())) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x54000~2000 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf8000,0x558c54cf80e0,0x558c54d753b0,0x558c54d75ab0 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:4#15 = 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4#17 onode 0x558c575ba500 txc 0x558c574fe000 2019-12-07T19:34:06.687+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4#17 0x0 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) put_ref 0x0~2000 Blob(0x558c574d3030 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) put_ref 0x0~2000 Blob(0x558c574d30a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) put_ref 0x0~2000 Blob(0x558c57566e00 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2fc0) put_ref 0x0~2000 Blob(0x558c574d2fc0 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2f50) put_ref 0x0~2000 Blob(0x558c574d2f50 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2e70) put_ref 0x0~2000 Blob(0x558c574d2e70 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2e00) put_ref 0x0~2000 Blob(0x558c574d2e00 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2d90) put_ref 0x0~2000 Blob(0x558c574d2d90 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2d20) put_ref 0x0~2000 Blob(0x558c574d2d20 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2cb0) put_ref 0x0~2000 Blob(0x558c574d2cb0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2c40) put_ref 0x0~2000 Blob(0x558c574d2c40 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2bd0) put_ref 0x0~2000 Blob(0x558c574d2bd0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2b60) put_ref 0x0~2000 Blob(0x558c574d2b60 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2af0) put_ref 0x0~2000 Blob(0x558c574d2af0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2a80) put_ref 0x0~2000 Blob(0x558c574d2a80 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2a10) put_ref 0x0~2000 Blob(0x558c574d2a10 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d29a0) put_ref 0x0~2000 Blob(0x558c574d29a0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2930) put_ref 0x0~2000 Blob(0x558c574d2930 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2700) put_ref 0x0~2000 Blob(0x558c574d2700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) put_ref 0x0~2000 Blob(0x558c574d2770 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) put_ref 0x0~2000 Blob(0x558c574d27e0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) put_ref 0x0~2000 Blob(0x558c574d28c0 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75810) put_ref 0x0~2000 Blob(0x558c54d75810 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75f10) put_ref 0x0~2000 Blob(0x558c54d75f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba608) dirty_range mark inline shard dirty 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574d3030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22b000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22d000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57566e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22f000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x231000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c574d2f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x233000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c574d2e70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x235000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c574d2e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x237000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c574d2d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x239000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c574d2d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23b000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c574d2cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23d000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c574d2c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23f000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c574d2bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x241000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c574d2b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x243000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c574d2af0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x245000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c574d2a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x247000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c574d2a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x249000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c574d29a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24b000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c574d2930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24d000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c574d2700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24f000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c574d2770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x251000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x253000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c574d28c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x255000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c54d75810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x257000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c54d75f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x259000~2000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:4#17 = 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe000 cost 5586 (1 ios * 4000 + 1586 bytes) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe000 onodes shared_blobs 0x558c54cf8000,0x558c54cf80e0,0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54d753b0,0x558c54d75ab0,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281a is empty 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281b is empty 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2819 is empty 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2818 is empty 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1))) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe000 allocated 0x[] released 0x[4e000~8000] 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x4e000~8000 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 freelist release 0x4e000~8000 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 prepare 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe000 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 io_done 2019-12-07T19:34:06.687+0000 7f8f0197c700 10 osd.7 29 dequeue_op 0x558c578d3e40 finish 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1927 < min_flush_size 524288 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3680 had 1 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba500 had 1 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1950 < min_flush_size 524288 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1950 < min_flush_size 524288 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x867a6~79e to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x867a6 0x867a6~79e to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x567a6 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7a6 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf44 and padding block with 0xbc 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x86f44 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.687+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000098407s (0.000000092s flush + 0.000098315s kv commit) 2019-12-07T19:34:06.687+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe000> 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.687+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 kv_submitted 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe000 2019-12-07T19:34:06.687+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 finishing 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe000 onodes 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 done 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:06.687+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe000 [4e000~8000] 2019-12-07T19:34:06.687+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x4e000~8000 2019-12-07T19:34:06.687+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d75ab0) put 0x558c54d75ab0 removing self from set 0x558c54d38c10 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75ac8 in 0x558c53a58fc0) _clear 2019-12-07T19:34:06.687+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 29/28 ECSubWriteReply(tid=87, last_complete=29'24, committed=1, applied=1)) v2 -- 0x558c575b9800 con 0x558c54c5bc00 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 _rm buffer(0x558c54c9a870 space 0x558c54d75ac8 0x0~1000 clean) 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 _rm buffer(0x558c551a8b40 space 0x558c54d75ac8 0x1000~1000 clean) 2019-12-07T19:34:06.687+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d753b0) put 0x558c54d753b0 removing self from set 0x558c54d38c10 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d753c8 in 0x558c53a58fc0) _clear 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 _rm buffer(0x558c551a8990 space 0x558c54d753c8 0x0~2000 clean) 2019-12-07T19:34:06.687+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54cf80e0) put 0x558c54cf80e0 removing self from set 0x558c54d38c10 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf80f8 in 0x558c53a58fc0) _clear 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 _rm buffer(0x558c551a9cb0 space 0x558c54cf80f8 0x0~2000 clean) 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54cf8000) put 0x558c54cf8000 removing self from set 0x558c54d38c10 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8018 in 0x558c53a58fc0) _clear 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 _rm buffer(0x558c551a8ab0 space 0x558c54cf8018 0x0~2000 clean) 2019-12-07T19:34:06.687+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.688+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 35 ==== MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748d800 con 0x558c54c64400 2019-12-07T19:34:06.688+0000 7f8f23c56700 15 osd.7 29 enqueue_op 0x558c578d3ce0 prio 127 cost 0 latency 0.000023 epoch 29 MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 2019-12-07T19:34:06.688+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2) prio 127 cost 0 e29) pg 0x558c5731a000 2019-12-07T19:34:06.688+0000 7f8f03980700 10 osd.7 29 dequeue_op 0x558c578d3ce0 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 pg pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.688+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 2019-12-07T19:34:06.688+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=27'12 active mbc={} ps=[2~1]] append_log log((0'0,29'15], crt=27'12) [] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1]] rollforward: entry=29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1]] rollforward: entry=29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1]] rollforward: entry=29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.688+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1]] append_log approx pg log length = 15 2019-12-07T19:34:06.688+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.688+0000 7f8f03980700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.688+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5740af00 seq 18 2019-12-07T19:34:06.688+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41df217a:::smithi13140796-39:4#e onode 0x558c5793a000 txc 0x558c5740af00 2019-12-07T19:34:06.688+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.2s4_head 4#1:41df217a:::smithi13140796-39:4#e 0x0 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c574e1ce0) put_ref 0x0~2000 Blob(0x558c574e1ce0 blob([0x2e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c574e0930) put_ref 0x0~2000 Blob(0x558c574e0930 blob([0x2e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57880bd0) put_ref 0x0~2000 Blob(0x558c57880bd0 blob([0x2e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c574e0e00) put_ref 0x0~2000 Blob(0x558c574e0e00 blob([0x2e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c5523a070) put_ref 0x0~2000 Blob(0x558c5523a070 blob([0x2ea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c55279110) put_ref 0x0~2000 Blob(0x558c55279110 blob([0x2ec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c55278ee0) put_ref 0x0~2000 Blob(0x558c55278ee0 blob([0x2ee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c54d3c0e0) put_ref 0x0~2000 Blob(0x558c54d3c0e0 blob([0x2f0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c5742bb20) put_ref 0x0~2000 Blob(0x558c5742bb20 blob([0x2f2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c578813b0) put_ref 0x0~2000 Blob(0x558c578813b0 blob([0x2f4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57881420) put_ref 0x0~2000 Blob(0x558c57881420 blob([0x2f6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57881490) put_ref 0x0~2000 Blob(0x558c57881490 blob([0x2f8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57881500) put_ref 0x0~2000 Blob(0x558c57881500 blob([0x2fa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57881570) put_ref 0x0~2000 Blob(0x558c57881570 blob([0x2fc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c578815e0) put_ref 0x0~2000 Blob(0x558c578815e0 blob([0x2fe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57881650) put_ref 0x0~2000 Blob(0x558c57881650 blob([0x300000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c578816c0) put_ref 0x0~2000 Blob(0x558c578816c0 blob([0x302000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57881730) put_ref 0x0~2000 Blob(0x558c57881730 blob([0x304000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c578817a0) put_ref 0x0~2000 Blob(0x558c578817a0 blob([0x306000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.blob(0x558c57881810) put_ref 0x0~1000 Blob(0x558c57881810 blob([0x308000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore.extentmap(0x558c5793a108) dirty_range mark inline shard dirty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574e1ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map(0x2e2000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e2000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2e2000~2000] from SharedBlob(0x558c5757f0a0 loaded (sbid 0x2836 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2e2000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c574e0930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map(0x2e4000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e4000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2e4000~2000] from SharedBlob(0x558c574d3f80 loaded (sbid 0x2837 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2e4000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57880bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map(0x2e6000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e6000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2e6000~2000] from SharedBlob(0x558c57819dc0 loaded (sbid 0x2838 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2e6000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574e0e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map(0x2e8000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e8000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2e8000~2000] from SharedBlob(0x558c57819ea0 loaded (sbid 0x2839 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2e8000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c5523a070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map(0x2ea000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ea000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2ea000~2000] from SharedBlob(0x558c57819f80 loaded (sbid 0x283a ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ea000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c55279110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map(0x2ec000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ec000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2ec000~2000] from SharedBlob(0x558c578de070 loaded (sbid 0x283b ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ec000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c55278ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map(0x2ee000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ee000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2ee000~2000] from SharedBlob(0x558c578de150 loaded (sbid 0x283c ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ee000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c54d3c0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map(0x2f0000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f0000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2f0000~2000] from SharedBlob(0x558c578de230 loaded (sbid 0x283d ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2f0000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5742bb20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map(0x2f2000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f2000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2f2000~2000] from SharedBlob(0x558c578de310 loaded (sbid 0x283e ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2f2000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c578813b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map(0x2f4000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f4000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2f4000~2000] from SharedBlob(0x558c578de3f0 loaded (sbid 0x283f ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2f4000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57881420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map(0x2f6000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f6000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2f6000~2000] from SharedBlob(0x558c578de4d0 loaded (sbid 0x2840 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2f6000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57881490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map(0x2f8000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f8000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2f8000~2000] from SharedBlob(0x558c578de5b0 loaded (sbid 0x2841 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2f8000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57881500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map(0x2fa000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fa000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2fa000~2000] from SharedBlob(0x558c578de690 loaded (sbid 0x2842 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2fa000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57881570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map(0x2fc000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fc000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2fc000~2000] from SharedBlob(0x558c578de770 loaded (sbid 0x2843 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2fc000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c578815e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map(0x2fe000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fe000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2fe000~2000] from SharedBlob(0x558c578de850 loaded (sbid 0x2844 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2fe000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c57881650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map(0x300000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x300000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x300000~2000] from SharedBlob(0x558c578de930 loaded (sbid 0x2845 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x300000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c578816c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map(0x302000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x302000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x302000~2000] from SharedBlob(0x558c578dea10 loaded (sbid 0x2846 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x302000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57881730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map(0x304000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x304000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x304000~2000] from SharedBlob(0x558c578deaf0 loaded (sbid 0x2847 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x304000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c578817a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map(0x306000~2000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x306000~2000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x306000~2000] from SharedBlob(0x558c578debd0 loaded (sbid 0x2848 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x306000~2000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~1000: 0x0~1000 Blob(0x558c57881810 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map(0x308000~1000=1)))) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x308000~1000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x308000~1000] from SharedBlob(0x558c578decb0 loaded (sbid 0x2849 ref_map())) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x308000~1000 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c574d3f80,0x558c5757f0a0,0x558c57819dc0,0x558c57819ea0,0x558c57819f80,0x558c578de070,0x558c578de150,0x558c578de230,0x558c578de310,0x558c578de3f0,0x558c578de4d0,0x558c578de5b0,0x558c578de690,0x558c578de770,0x558c578de850,0x558c578de930,0x558c578dea10,0x558c578deaf0,0x558c578debd0,0x558c578decb0 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41df217a:::smithi13140796-39:4#e = 0 2019-12-07T19:34:06.688+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41df217a:::smithi13140796-39:head#f onode 0x558c5793a280 txc 0x558c5740af00 2019-12-07T19:34:06.688+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.2s4_head 4#1:41df217a:::smithi13140796-39:head#f 0x0 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41df217a:::smithi13140796-39:head#f = 0 2019-12-07T19:34:06.688+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5587 (1 ios * 4000 + 1587 bytes) 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 0x558c574d3f80,0x558c5757f0a0,0x558c57819dc0,0x558c57819ea0,0x558c57819f80,0x558c578de070,0x558c578de150,0x558c578de230,0x558c578de310,0x558c578de3f0,0x558c578de4d0,0x558c578de5b0,0x558c578de690,0x558c578de770,0x558c578de850,0x558c578de930,0x558c578dea10,0x558c578deaf0,0x558c578debd0,0x558c578decb0 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2837 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2836 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2838 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2839 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283a is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283b is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283c is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283d is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283e is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x283f is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2840 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2841 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2842 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2843 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2844 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2845 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2846 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2847 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2848 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2849 is empty 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[2e2000~27000] 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x2e2000~27000 2019-12-07T19:34:06.688+0000 7f8f03980700 10 freelist release 0x2e2000~27000 2019-12-07T19:34:06.688+0000 7f8f03980700 20 freelist _xor first_key 0x280000 last_key 0x300000 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:06.688+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:06.688+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:06.688+0000 7f8f03980700 10 osd.7 29 dequeue_op 0x558c578d3ce0 finish 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.688+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1573 < min_flush_size 524288 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793a000 had 1 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793a280 had 1 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1596 < min_flush_size 524288 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1596 < min_flush_size 524288 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x86f44~63c to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x86f44 0x86f44~63c to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x56f44 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf44 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x580 and padding block with 0xa80 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x87580 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.688+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000096654s (0.000000104s flush + 0.000096550s kv commit) 2019-12-07T19:34:06.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.688+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 36 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e07100 con 0x558c54c64400 2019-12-07T19:34:06.688+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.2s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:06.688+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.689+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.689+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.689+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.689+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e29) pg 0x558c5731a000 2019-12-07T19:34:06.689+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:06.689+0000 7f8f07988700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.689+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:06.689+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:06.689+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5740a000 seq 19 2019-12-07T19:34:06.689+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:06.689+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:06.689+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:06.689+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:06.689+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.689+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes shared_blobs 2019-12-07T19:34:06.689+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.689+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:34:06.689+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:34:06.689+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:34:06.689+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00> 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 kv_queued 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [2e2000~27000] 2019-12-07T19:34:06.689+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x2e2000~27000 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578decb0) put 0x558c578decb0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578decc8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578debd0) put 0x558c578debd0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578debe8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578deaf0) put 0x558c578deaf0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578deb08 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578dea10) put 0x558c578dea10 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dea28 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de930) put 0x558c578de930 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 -- 0x558c551aec00 con 0x558c54c64400 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de948 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de850) put 0x558c578de850 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de868 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de770) put 0x558c578de770 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de788 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de690) put 0x558c578de690 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de6a8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de5b0) put 0x558c578de5b0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de5c8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de4d0) put 0x558c578de4d0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de4e8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de3f0) put 0x558c578de3f0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de408 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de310) put 0x558c578de310 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de328 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de230) put 0x558c578de230 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de248 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de150) put 0x558c578de150 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de168 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de070) put 0x558c578de070 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de088 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57819f80) put 0x558c57819f80 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819f98 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57819ea0) put 0x558c57819ea0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819eb8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57819dc0) put 0x558c57819dc0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819dd8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5757f0a0) put 0x558c5757f0a0 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f0b8 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574d3f80) put 0x558c574d3f80 removing self from set 0x558c54d39930 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3f98 in 0x558c53a595e0) _clear 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x87580~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x87580 0x87580~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x57580 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x580 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9ed and padding block with 0x613 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x879ed 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.689+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000090562s (0.000000092s flush + 0.000090470s kv commit) 2019-12-07T19:34:06.689+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000> 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:34:06.689+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.689+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.690+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 85 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c54e07100 con 0x558c54c5bc00 2019-12-07T19:34:06.690+0000 7f8f23c56700 15 osd.7 29 enqueue_peering_evt 1.6s2 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:06.690+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.690+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.690+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.690+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.690+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e29) pg 0x558c54d5c000 2019-12-07T19:34:06.690+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:06.690+0000 7f8f05984700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.690+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:06.690+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:06.690+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f6900 seq 48 2019-12-07T19:34:06.690+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:06.690+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:06.690+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.690+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:06.690+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6900 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.690+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6900 onodes shared_blobs 2019-12-07T19:34:06.690+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.690+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 prepare 2019-12-07T19:34:06.690+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6900 2019-12-07T19:34:06.690+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 io_done 2019-12-07T19:34:06.690+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x879ed~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x879ed 0x879ed~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x579ed 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9ed 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe5a and padding block with 0x1a6 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x87e5a 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.690+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000125536s (0.000000075s flush + 0.000125461s kv commit) 2019-12-07T19:34:06.690+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6900> 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.690+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 kv_submitted 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6900 2019-12-07T19:34:06.690+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 finishing 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6900 onodes 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 done 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:06.690+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6900 [] 2019-12-07T19:34:06.690+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.690+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.690+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.690+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.693+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 128 ==== MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c575c0800 con 0x558c54c64000 2019-12-07T19:34:06.693+0000 7f8f24457700 15 osd.7 29 enqueue_op 0x558c54e79760 prio 127 cost 0 latency 0.000033 epoch 29 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:06.693+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e29) pg 0x558c5500a000 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c54e79760 prio 127 cost 0 latency 0.000084 MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 pg pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s4 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] append_log log((0'0,29'13], crt=27'8) [] 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] rollforward: entry=29'9 (21'6) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:244 2019-12-07T19:34:05.664687+0000 0 ObjectCleanRegions clean_offsets: [0~212992,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] rollforward: entry=29'10 (29'9) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:245 2019-12-07T19:34:05.667234+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] rollforward: entry=29'11 (29'10) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:246 2019-12-07T19:34:05.667270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] rollforward: entry=29'12 (27'7) delete 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] rollforward: entry=29'13 (27'8) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] append_log approx pg log length = 13 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.693+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.693+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c548f7500 seq 29 2019-12-07T19:34:06.693+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4#c onode 0x558c54d97680 txc 0x558c548f7500 2019-12-07T19:34:06.693+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4#c 0x0 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8770) put_ref 0x0~2000 Blob(0x558c54cf8770 blob([0x3b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore.blob(0x558c54d75d50) put_ref 0x0~1000 Blob(0x558c54d75d50 blob([0x3d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97788) dirty_range mark inline shard dirty 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54cf8770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map(0x3b000~2000=1)))) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b000~2000] 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3b000~2000] from SharedBlob(0x558c54d3de30 loaded (sbid 0x281c ref_map())) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3b000~2000 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~1000: 0x0~1000 Blob(0x558c54d75d50 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map(0x3d000~1000=1)))) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3d000~1000] 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3d000~1000] from SharedBlob(0x558c54d3c620 loaded (sbid 0x281d ref_map())) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3d000~1000 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3c620,0x558c54d3de30 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:4#c = 0 2019-12-07T19:34:06.693+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7500 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7500 onodes shared_blobs 0x558c54d3c620,0x558c54d3de30 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281d is empty 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281c is empty 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7500 allocated 0x[] released 0x[3b000~3000] 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x3b000~3000 2019-12-07T19:34:06.693+0000 7f8f04181700 10 freelist release 0x3b000~3000 2019-12-07T19:34:06.693+0000 7f8f04181700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 prepare 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7500 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 io_done 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 29 dequeue_op 0x558c54e79760 finish 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 422 < min_flush_size 524288 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1284 < min_flush_size 524288 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97680 had 1 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1307 < min_flush_size 524288 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1307 < min_flush_size 524288 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x87e5a~51b to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x87e5a 0x87e5a~51b to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x57e5a 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe5a 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x375 and padding block with 0xc8b 2019-12-07T19:34:06.693+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 129 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e04000 con 0x558c54c64000 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x88375 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.693+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.693+0000 7f8f24457700 15 osd.7 29 enqueue_peering_evt 1.9s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7500> 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.693+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 kv_submitted 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7500 2019-12-07T19:34:06.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7500 finishing 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7500 onodes 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7500 done 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.693+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7500 [3b000~3000] 2019-12-07T19:34:06.693+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x3b000~3000 2019-12-07T19:34:06.693+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3de30) put 0x558c54d3de30 removing self from set 0x558c54d0efd0 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3de48 in 0x558c53a59a40) _clear 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 _rm buffer(0x558c5796fd40 space 0x558c54d3de48 0x0~1000 clean) 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 _rm buffer(0x558c5796fdd0 space 0x558c54d3de48 0x1000~1000 clean) 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3c620) put 0x558c54d3c620 removing self from set 0x558c54d0efd0 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c638 in 0x558c53a59a40) _clear 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 _rm buffer(0x558c5796fe60 space 0x558c54d3c638 0x0~1000 clean) 2019-12-07T19:34:06.693+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000097323s (0.000000096s flush + 0.000097227s kv commit) 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558c5500a000 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.693+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.693+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d58c00 seq 30 2019-12-07T19:34:06.693+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:34:06.693+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:34:06.693+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:06.693+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.693+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.694+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=144, last_complete=29'13, committed=1, applied=1)) v2 -- 0x558c54dfa200 con 0x558c54c64000 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x88375~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x88375 0x88375~46d to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:06.694+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x58375 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x375 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7e2 and padding block with 0x81e 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x887e2 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:06.694+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000121965s (0.000000092s flush + 0.000121873s kv commit) 2019-12-07T19:34:06.694+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.694+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:34:06.694+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:06.694+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.694+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:34:06.694+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:06.694+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:06.694+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.730+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.781+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.831+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.859+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.309829762s delta_ub 0.014020976s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c65400 2019-12-07T19:34:06.859+0000 7f8f24c58700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013486851s,0.014020976s]) 2019-12-07T19:34:06.859+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.296342911s delta_ub 18446744073.696064765s) v5 -- 0x558c54c59800 con 0x558c54c65400 2019-12-07T19:34:06.859+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.309829762s delta_ub 0.014020976s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58600 con 0x558c54c65000 2019-12-07T19:34:06.859+0000 7f8f24c58700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013442347s,0.014020976s]) 2019-12-07T19:34:06.859+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.296387415s delta_ub 18446744073.696109269s) v5 -- 0x558c54c59800 con 0x558c54c65000 2019-12-07T19:34:06.881+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.931+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.981+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.031+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 7 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.511023139s delta_ub 0.021771485s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c19000 2019-12-07T19:34:07.053+0000 7f8f23c56700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021327143s,0.021771485s]) 2019-12-07T19:34:07.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.489695996s delta_ub 18446744073.688224473s) v5 -- 0x558c575b9800 con 0x558c54c19000 2019-12-07T19:34:07.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 7 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.511023139s delta_ub 0.021771485s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c18800 2019-12-07T19:34:07.053+0000 7f8f23c56700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021110239s,0.021771485s]) 2019-12-07T19:34:07.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.489912900s delta_ub 18446744073.688441377s) v5 -- 0x558c575b9800 con 0x558c54c18800 2019-12-07T19:34:07.081+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.131+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.181+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.231+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.281+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.332+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.346+0000 7f8f21451700 10 osd.7 29 tick 2019-12-07T19:34:07.346+0000 7f8f21451700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:07.346+0000 7f8f21451700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:07.346+0000 7f8f21451700 20 osd.7 29 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:07.382+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.432+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.442+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.909818262s delta_ub 0.031035657s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c5ac00 2019-12-07T19:34:07.442+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.909818262s delta_ub 0.031035657s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c5a400 2019-12-07T19:34:07.442+0000 7f8f23c56700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030702578s,0.031035657s]) 2019-12-07T19:34:07.442+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.879115684s delta_ub 18446744073.678849038s) v5 -- 0x558c575b9800 con 0x558c54c5ac00 2019-12-07T19:34:07.442+0000 7f8f24c58700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030683814s,0.031035657s]) 2019-12-07T19:34:07.442+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.879134448s delta_ub 18446744073.678867802s) v5 -- 0x558c54c59800 con 0x558c54c5a400 2019-12-07T19:34:07.482+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.532+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.553+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 8 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 0.019741035s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c18c00 2019-12-07T19:34:07.553+0000 7f8f23c56700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018870568s,0.019741035s]) 2019-12-07T19:34:07.553+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 30.990100066s delta_ub 18446744073.690681048s) v5 -- 0x558c575b9800 con 0x558c54c18c00 2019-12-07T19:34:07.553+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 8 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 0.019741035s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c18400 2019-12-07T19:34:07.553+0000 7f8f23c56700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018831108s,0.019741035s]) 2019-12-07T19:34:07.553+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 30.990139526s delta_ub 18446744073.690720508s) v5 -- 0x558c575b9800 con 0x558c54c18400 2019-12-07T19:34:07.582+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 10 osd.7 29 tick_without_osd_lock 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fba9000/0x40000000/0x1680000000, data 0x425802/0x455000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 25 osd.7 29 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 25 osd.7 29 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:03.373892+0000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 25 osd.7 29 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 25 osd.7 29 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373810+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 25 osd.7 29 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 25 osd.7 29 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 20 osd.7 29 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 20 osd.7 29 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 20 osd.7 29 scrub_load_below_threshold loadavg per cpu 0.0575 < max 5 = yes 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 20 osd.7 29 sched_scrub load_is_low=1 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 10 osd.7 29 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:07.590076+0000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 20 osd.7 29 sched_scrub done 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 10 osd.7 29 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 20 osd.7 29 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:07.589+0000 7f8f1fbbe700 10 osd.7 29 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:07.633+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.682+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.732+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.783+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.833+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.883+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.885+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57be8800 0x558c54d04100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:07.885+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57be8800 0x558c54d04100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:07.885+0000 7f8f24457700 10 osd.7 29 ms_handle_authentication new session 0x558c57bfa000 con 0x558c57be8800 entity client.admin addr 2019-12-07T19:34:07.885+0000 7f8f24457700 10 osd.7 29 ms_handle_authentication session 0x558c57bfa000 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:07.886+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3259582720 conn(0x558c57be8800 0x558c54d04100 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4385 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:07.886+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4385 172.21.15.131:0/3259582720 1 ==== command(tid 14: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c54d9f0e0 con 0x558c57be8800 2019-12-07T19:34:07.886+0000 7f8f164bb700 20 osd.7 29 OSD::ms_dispatch: command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:07.886+0000 7f8f164bb700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:07.886+0000 7f8f164bb700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:07.886+0000 7f8f164bb700 20 osd.7 29 _dispatch 0x558c54d9f0e0 command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:07.886+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3259582720 -- command_reply(tid 14: 0 ) v1 -- 0x558c54ce1340 con 0x558c57be8800 2019-12-07T19:34:07.908+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4385 172.21.15.131:0/3259582720 2 ==== command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c54d9ef00 con 0x558c57be8800 2019-12-07T19:34:07.908+0000 7f8f164bb700 20 osd.7 29 OSD::ms_dispatch: command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:07.908+0000 7f8f164bb700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:07.908+0000 7f8f164bb700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:07.908+0000 7f8f164bb700 20 osd.7 29 _dispatch 0x558c54d9ef00 command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:07.909+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3259582720 -- command_reply(tid 15: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce1500 con 0x558c57be8800 2019-12-07T19:34:07.911+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3259582720 conn(0x558c57be8800 msgr2=0x558c54d04100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 64 2019-12-07T19:34:07.912+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3259582720 conn(0x558c57be8800 msgr2=0x558c54d04100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:07.912+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3259582720 conn(0x558c57be8800 0x558c54d04100 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:07.912+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3259582720 conn(0x558c57be8800 0x558c54d04100 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:07.912+0000 7f8f164bb700 2 osd.7 29 ms_handle_reset con 0x558c57be8800 session 0x558c57bfa000 2019-12-07T19:34:07.933+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 414391 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.942+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e30 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.410096593s delta_ub 0.030982002s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c5ac00 2019-12-07T19:34:07.942+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e30 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.410096593s delta_ub 0.030982002s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58600 con 0x558c54c5a400 2019-12-07T19:34:07.942+0000 7f8f23c56700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030655834s,0.030982002s]) 2019-12-07T19:34:07.942+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.379440759s delta_ub 18446744073.678895782s) v5 -- 0x558c575b9800 con 0x558c54c5ac00 2019-12-07T19:34:07.942+0000 7f8f23c56700 10 osd.7 29 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 29 -> 30 (as per caller) 2019-12-07T19:34:07.942+0000 7f8f24c58700 20 osd.7 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030630129s,0.030982002s]) 2019-12-07T19:34:07.942+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.379466464s delta_ub 18446744073.678921487s) v5 -- 0x558c54c59800 con 0x558c54c5a400 2019-12-07T19:34:07.943+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 41 ==== osd_map(30..30 src has 1..30) v4 ==== 633+0+0 (unknown 0 0 0) 0x558c57e03b00 con 0x558c54c19c00 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 osd.7 29 OSD::ms_dispatch: osd_map(30..30 src has 1..30) v4 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 osd.7 29 do_waiters -- start 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 osd.7 29 do_waiters -- finish 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 osd.7 29 _dispatch 0x558c57e03b00 osd_map(30..30 src has 1..30) v4 2019-12-07T19:34:07.943+0000 7f8f1199c700 3 osd.7 29 handle_osd_map epochs [30,30], i have 29, src has [1,30] 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 osd.7 29 handle_osd_map got inc map for epoch 30 2019-12-07T19:34:07.943+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3bd8e826:::osdmap.29:0# 0x0~0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1438 size 0x1438 (5176) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576aca80 blob([0x442000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574f1c00 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0700 blob([!~1000,0x443000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c574e09a0 sbid 0x0)) need 0x1000~438 cache has 0x[1000~438] 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3bd8e826:::osdmap.29:0# 0x0~1438 = 5176 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 osd.7 29 add_map_bl 29 5176 bytes 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 osd.7 29 got_full_map 30, nothing requested 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 osd.7 29 handle_osd_map pg_num_history pg_num_history(e30 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {30={}} 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 osd.7 29 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e30 [1,30] lci=[13,30]) 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54dde300 seq 21 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c4cb972f:::inc_osdmap.30:0# key 0x7f7fffffffffffffffc4cb97'/!inc_osdmap.30!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c4cb972f:::inc_osdmap.30:0# 0x0~24d 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1141 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c4cb972f:::inc_osdmap.30:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 1 blobs 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3b000~1000/1000,1000,0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3b000~1000] 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576acee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576aca10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576acee0 blob([0x3b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576aca10 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c576acee0) get_ref 0x0~24d Blob(0x558c576acee0 blob([0x3b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576aca10 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c576acee0) get_ref init 0x1000, 1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c576acee0 blob([0x3b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c576aca10 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c576aca28 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e888) dirty_range mark inline shard dirty 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c4cb972f:::inc_osdmap.30:0# 0x0~24d = 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3c18e826:::osdmap.30:0# key 0x7f7fffffffffffffff3c18e8'&!osdmap.30!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3c18e826:::osdmap.30:0# 0x0~1438 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1142 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3c18e826:::osdmap.30:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~438 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 2 blobs 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3c000~2000/1000,2000,0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3c000~2000] 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57567ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57567ea0 blob([0x3c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567d50 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c57567ea0) get_ref 0x0~1000 Blob(0x558c57567ea0 blob([0x3c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567d50 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c57567ea0) get_ref init 0x1000, 1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57567ea0 blob([0x3c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57567d50 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57567d68 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57567dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57567dc0 blob([!~1000,0x3d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567500 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c57567dc0) get_ref 0x1000~438 Blob(0x558c57567dc0 blob([!~1000,0x3d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57567500 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c57567dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558c57567dc0 blob([!~1000,0x3d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c57567500 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57567518 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1438 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756eb08) dirty_range mark inline shard dirty 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3c18e826:::osdmap.30:0# 0x0~1438 = 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 0 blobs 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 0 blobs 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde300 cost 23246 (4 ios * 4000 + 7246 bytes) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde300 onodes 0x558c53a91900,0x558c55276c80,0x558c5756e780,0x558c5756ea00 shared_blobs 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e888) update #-1:c4cb972f:::inc_osdmap.30:0# 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756e888) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c4cb972f:::inc_osdmap.30:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756eb08) update #-1:3c18e826:::osdmap.30:0# 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756eb08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3c18e826:::osdmap.30:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde300 allocated 0x[3b000~3000] released 0x[] 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 freelist allocate 0x3b000~3000 2019-12-07T19:34:07.943+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 prepare 2019-12-07T19:34:07.943+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54dde300 2019-12-07T19:34:07.943+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 aio_wait 2019-12-07T19:34:07.943+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde300 2019-12-07T19:34:07.943+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 io_done 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756e780 had 1 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756ea00 had 1 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860267008 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998644) bluestore 86 GiB free (0.959952), bluefs_ratio 0.0399502 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x887e2~226c to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x887e2 0x887e2~226c to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x587e2 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7e2 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa4e and padding block with 0x5b2 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8aa4e 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:07.943+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000159655s (0.000004879s flush + 0.000154776s kv commit) 2019-12-07T19:34:07.943+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde300> 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:07.943+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 kv_submitted 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde300 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54dde300 osr 0x558c54ea4b40 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) _discard drop 18 0x1b000~1000 2019-12-07T19:34:07.943+0000 7f8f141a1700 10 osd.7 29 _committed_osd_maps 30..30 2019-12-07T19:34:07.943+0000 7f8f141a1700 10 osd.7 29 advance to epoch 30 (<= last 30 <= newest_map 30) 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 19 0x1b000~1000 crc f8683679 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) _discard drop 18 0x2000~1000 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 19 0x2000~1000 crc b2b726a5 2019-12-07T19:34:07.943+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:07.943+0000 7f8f141a1700 7 osd.7 30 consume_map version 30 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.0s3 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.9s4 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.2s4 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.bs3 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.3s4 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.cs0 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.4s0 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.ds2 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.6s2 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 identify_splits_and_merges 1.7s4 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 30 prune_pg_created 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:0.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:1.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:2.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:3.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:4.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:5.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:6.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7:7.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:07.944+0000 7f8f141a1700 10 osd.7 30 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.0s3 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.9s4 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.2s4 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.bs3 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.3s4 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c5500d000 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c5500a000 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.cs0 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.4s0 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c5731a000 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.ds2 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.6s2 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f141a1700 15 osd.7 30 enqueue_peering_evt 1.7s4 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c54d9b000 2019-12-07T19:34:07.944+0000 7f8f141a1700 7 osd.7 30 activate_map version 30 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f0918b700 20 osd.7 30 agent_entry empty queue 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c54d5f000 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 29 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 29 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 29 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 29 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:07.944+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:07.944+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.944+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f05183700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f04181700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c54d5c000 2019-12-07T19:34:07.944+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.944+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:07.944+0000 7f8f03980700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c55141000 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c57676000 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c5513e000 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 29 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 29 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:07.944+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 29 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] state: Active advmap 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f04982700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] state: Active: handling ActMap 2019-12-07T19:34:07.944+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] Active: kicking snap trim 2019-12-07T19:34:07.944+0000 7f8f0297e700 7 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] state: handle ActMap primary 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] prepare_stats_for_publish reporting purged_snaps [2~1,4~1] 2019-12-07T19:34:07.944+0000 7f8f0297e700 15 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] publish_stats_to_osd 29:82 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558c548ab000 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 29 -> 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 29 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: Active advmap 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.944+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: Active: handling ActMap 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] Active: kicking snap trim 2019-12-07T19:34:07.944+0000 7f8f0297e700 7 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: handle ActMap primary 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] prepare_stats_for_publish reporting purged_snaps [2~1,4~1] 2019-12-07T19:34:07.944+0000 7f8f0297e700 15 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] publish_stats_to_osd 28:32 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 30 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:07.944+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:07.983+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.033+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.083+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.133+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.183+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.205+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:08.234+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.284+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.330+0000 7f8f21451700 10 osd.7 30 tick 2019-12-07T19:34:08.330+0000 7f8f21451700 10 osd.7 30 do_waiters -- start 2019-12-07T19:34:08.330+0000 7f8f21451700 10 osd.7 30 do_waiters -- finish 2019-12-07T19:34:08.330+0000 7f8f21451700 20 osd.7 30 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:08.334+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.384+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.434+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.484+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.534+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.584+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 417627 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.601+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 37 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e07100 con 0x558c54c64400 2019-12-07T19:34:08.601+0000 7f8f23c56700 15 osd.7 30 enqueue_peering_evt 1.2s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:08.601+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.601+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) pg 0x558c5731a000 2019-12-07T19:34:08.601+0000 7f8f07988700 20 osd.7 op_wq _add_slot_waiter 1.2s4 peering, item epoch is 31, will wait on OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.602+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 38 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e07100 con 0x558c54c64400 2019-12-07T19:34:08.602+0000 7f8f23c56700 15 osd.7 30 enqueue_peering_evt 1.2s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:08.602+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.602+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {31=} 2019-12-07T19:34:08.602+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {31=} 2019-12-07T19:34:08.602+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) pg 0x558c5731a000 2019-12-07T19:34:08.602+0000 7f8f03980700 20 osd.7 op_wq _add_slot_waiter 1.2s4 peering, item epoch is 31, will wait on OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.602+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.605+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 47 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e06a00 con 0x558c54c19800 2019-12-07T19:34:08.606+0000 7f8f24c58700 15 osd.7 30 enqueue_peering_evt 1.ds2 epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.606+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) pg 0x558c55141000 2019-12-07T19:34:08.606+0000 7f8f06185700 20 osd.7 op_wq _add_slot_waiter 1.ds2 peering, item epoch is 31, will wait on OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.606+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.606+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 48 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e06a00 con 0x558c54c19800 2019-12-07T19:34:08.606+0000 7f8f24c58700 15 osd.7 30 enqueue_peering_evt 1.ds2 epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.606+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.606+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {31=} 2019-12-07T19:34:08.606+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.606+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {31=} 2019-12-07T19:34:08.606+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) pg 0x558c55141000 2019-12-07T19:34:08.606+0000 7f8f0217d700 20 osd.7 op_wq _add_slot_waiter 1.ds2 peering, item epoch is 31, will wait on OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.606+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 81 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c54e04000 con 0x558c54c5b400 2019-12-07T19:34:08.608+0000 7f8f24457700 15 osd.7 30 enqueue_peering_evt 1.bs3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.608+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 82 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c54e04000 con 0x558c54c5b400 2019-12-07T19:34:08.608+0000 7f8f24457700 15 osd.7 30 enqueue_peering_evt 1.3s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.608+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.608+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 83 ==== osd_map(30..31 src has 1..31) v4 ==== 1286+0+0 (unknown 0 0 0) 0x558c57e03d40 con 0x558c54c5b400 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 osd.7 30 OSD::ms_dispatch: osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 osd.7 30 do_waiters -- start 2019-12-07T19:34:08.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 84 ==== MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57573400 con 0x558c54c5b400 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 osd.7 30 do_waiters -- finish 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 osd.7 30 _dispatch 0x558c57e03d40 osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f24457700 15 osd.7 30 enqueue_op 0x558c54e79ce0 prio 127 cost 0 latency 0.000023 epoch 31 MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.609+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) pg 0x558c54d9b000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 31, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) pg 0x558c5513e000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 31, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {31=} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {31=} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) pg 0x558c54d9b000 2019-12-07T19:34:08.609+0000 7f8f07187700 10 osd.7 30 dequeue_op 0x558c54e79ce0 prio 127 cost 0 latency 0.000073 MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] 2019-12-07T19:34:08.609+0000 7f8f07187700 10 osd.7 30 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:08.609+0000 7f8f1199c700 3 osd.7 30 handle_osd_map epochs [30,31], i have 30, src has [1,31] 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 osd.7 30 handle_osd_map got inc map for epoch 31 2019-12-07T19:34:08.609+0000 7f8f07187700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] _handle_message: MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.609+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3c18e826:::osdmap.30:0# 0x0~0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1438 size 0x1438 (5176) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:08.609+0000 7f8f07187700 25 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:08.609+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567ea0 blob([0x3c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57567d50 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~32000 size 0x32000 (204800) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567dc0 blob([!~1000,0x3d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c57567500 sbid 0x0)) need 0x1000~438 cache has 0x[1000~438] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3c18e826:::osdmap.30:0# 0x0~1438 = 5176 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 osd.7 30 add_map_bl 30 5176 bytes 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 osd.7 30 got_full_map 31, nothing requested 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 osd.7 30 handle_osd_map pg_num_history pg_num_history(e31 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {31={1=[2~1]}} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c552dbf40) lower_bound to PSN__1_0000000000000001 key 0x00000000000000000000000000000403'.PSN__1_0000000000000001' 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c552dbf40) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 snap_mapper._lookup_purged_snap pool 1 snap 1 key 'PSN__1_0000000000000001' lower_bound not found 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c552dbf40) lower_bound to PSN__1_0000000000000003 key 0x00000000000000000000000000000403'.PSN__1_0000000000000003' 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c552dbf40) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 snap_mapper._lookup_purged_snap pool 1 snap 3 key 'PSN__1_0000000000000003' lower_bound not found 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 1 keys 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 osd.7 30 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e31 [1,31] lci=[13,31]) 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54dde600 seq 22 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cf0b972f:::inc_osdmap.31:0# key 0x7f7fffffffffffffffcf0b97'/!inc_osdmap.31!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:08.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 85 ==== MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57572f00 con 0x558c54c5b400 2019-12-07T19:34:08.609+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cf0b972f:::inc_osdmap.31:0# 0x0~285 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1143 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cf0b972f:::inc_osdmap.31:0# 0x0~285 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~285 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd7b on front/back, now 0x0~1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde600 1 blobs 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4e000~1000/1000,1000,0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4e000~1000] 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ace00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bc1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:08.609+0000 7f8f24457700 15 osd.7 30 enqueue_op 0x558c54e798c0 prio 127 cost 0 latency 0.000016 epoch 31 MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ace00 blob([0x4e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc1c0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c576ace00) get_ref 0x0~285 Blob(0x558c576ace00 blob([0x4e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc1c0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c576ace00) get_ref init 0x1000, 1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~285: 0x0~285 Blob(0x558c576ace00 blob([0x4e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x558c575bc1c0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c575bc1d8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:08.609+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x285 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756ed88) dirty_range mark inline shard dirty 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cf0b972f:::inc_osdmap.31:0# 0x0~285 = 0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3398e826:::osdmap.31:0# key 0x7f7fffffffffffffff3398e8'&!osdmap.31!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:08.609+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3398e826:::osdmap.31:0# 0x0~1470 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1144 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3398e826:::osdmap.31:0# 0x0~1470 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~470 2019-12-07T19:34:08.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {31=} 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb90 on front/back, now 0x0~1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde600 2 blobs 2019-12-07T19:34:08.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4f000~2000/1000,2000,0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4f000~2000] 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bde30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552657a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bde30 blob([0x4f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552657a0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c575bde30) get_ref 0x0~1000 Blob(0x558c575bde30 blob([0x4f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552657a0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c575bde30) get_ref init 0x1000, 1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c575bde30 blob([0x4f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c552657a0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c552657b8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57396a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265880 blob([!~1000,0x50000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396a80 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c55265880) get_ref 0x1000~470 Blob(0x558c55265880 blob([!~1000,0x50000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57396a80 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c55265880) get_ref init 0x2000, 1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~470: 0x1000~470 Blob(0x558c55265880 blob([!~1000,0x50000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,470]) SharedBlob(0x558c57396a80 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57396a98 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1470 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756f008) dirty_range mark inline shard dirty 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3398e826:::osdmap.31:0# 0x0~1470 = 0 2019-12-07T19:34:08.609+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde600 0 blobs 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:08.609+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:08.609+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000002' <- PSN__1_0000000000000002 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:08.609+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde600 0 blobs 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde600 cost 23413 (4 ios * 4000 + 7413 bytes) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde600 onodes 0x558c53a91900,0x558c55276c80,0x558c5756ec80,0x558c5756ef00 shared_blobs 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756ed88) update #-1:cf0b972f:::inc_osdmap.31:0# 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756ed88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cf0b972f:::inc_osdmap.31:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756f008) update #-1:3398e826:::osdmap.31:0# 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5756f008) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3398e826:::osdmap.31:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde600 allocated 0x[4e000~3000] released 0x[] 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 freelist allocate 0x4e000~3000 2019-12-07T19:34:08.609+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 prepare 2019-12-07T19:34:08.609+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54dde600 2019-12-07T19:34:08.609+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 aio_wait 2019-12-07T19:34:08.609+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde600 2019-12-07T19:34:08.609+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 io_done 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8858 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756ec80 had 1 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756ef00 had 1 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8881 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8881 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8aa4e~22b1 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8aa4e 0x8aa4e~22b1 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5aa4e 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa4e 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcff and padding block with 0x301 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8ccff 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000149043s (0.000005170s flush + 0.000143873s kv commit) 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde600> 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 kv_submitted 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde600 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54dde600 osr 0x558c54ea4b40 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) _discard drop 19 0x1b000~1000 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7 30 _committed_osd_maps 31..31 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7 30 advance to epoch 31 (<= last 31 <= newest_map 31) 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 20 0x1b000~1000 crc e2b79a84 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) _discard drop 19 0x2000~1000 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 20 0x2000~1000 crc 98c3298 2019-12-07T19:34:08.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.609+0000 7f8f141a1700 7 osd.7 31 consume_map version 31 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.0s3 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.9s4 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.2s4 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad7b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.bs3 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.3s4 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c54dba000 space 0x558c576ad7b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.cs0 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.4s0 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.ds2 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.6s2 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 identify_splits_and_merges 1.7s4 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c573965c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 31 prune_pg_created 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c54dba090 space 0x558c573965c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:0.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:1.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:2.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac478 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 pending_peering first epoch 31 <= 31, requeueing 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57982990 space 0x558c576ac478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.2s4 to_process <> waiting <> waiting_peering {31=} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac248 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5796fb90 space 0x558c576ac248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:3.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 pending_peering first epoch 31 <= 31, requeueing 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5796fc20 space 0x558c576ac328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process waiting <> waiting_peering {31=} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57396638 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28ab0 space 0x558c57396638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 31 <= 31, requeueing 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {31=} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:4.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28b40 space 0x558c57970868 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:5.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) pg 0x558c5731a000 2019-12-07T19:34:08.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 86 ==== MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57572500 con 0x558c54c5b400 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 pending_peering first epoch 31 <= 31, requeueing 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.ds2 to_process <> waiting <> waiting_peering {31=} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28bd0 space 0x558c57970948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f24457700 15 osd.7 31 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000024 epoch 31 MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:6.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7:7.consume_map 31 (was 30) 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28c60 space 0x558c57970a28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f141a1700 10 osd.7 31 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28cf0 space 0x558c57970b78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.0s3 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28d80 space 0x558c57970c58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) pg 0x558c55141000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.9s4 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07988700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28e10 space 0x558c57970d38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s4 30 -> 31 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.2s4 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28ea0 space 0x558c57970e18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07988700 10 osd.7 pg_epoch: 30 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:08.609+0000 7f8f06185700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 30 -> 31 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.bs3 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28f30 space 0x558c57970ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.3s4 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f06185700 10 osd.7 pg_epoch: 30 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e28fc0 space 0x558c57970fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.cs0 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579710b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e29050 space 0x558c579710b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.609+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.4s0 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.609+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e290e0 space 0x558c57971198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.609+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.609+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e29170 space 0x558c57970a98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e29200 space 0x558c579b2328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c57676000 2019-12-07T19:34:08.609+0000 7f8f06185700 20 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2408 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07988700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.609+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e29290 space 0x558c579b2408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.609+0000 7f8f06185700 20 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.609+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b24e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.609+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e29320 space 0x558c579b24e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.ds2 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:08.610+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b25c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.6s2 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e293b0 space 0x558c579b25c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c548ab000 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b26a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7f8f06986700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e29440 space 0x558c579b26a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 31 2019-12-07T19:34:08.610+0000 7f8f141a1700 15 osd.7 31 enqueue_peering_evt 1.7s4 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:08.610+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:08.610+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e294d0 space 0x558c579b2788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:08.610+0000 7f8f141a1700 7 osd.7 31 activate_map version 31 2019-12-07T19:34:08.610+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.610+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57e29560 space 0x558c579b2868 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 204800 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c574fe300 seq 18 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c54d5c000 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:08.610+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5740b800 seq 20 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_sub_read read request=204800 r=204800 len=204800 2019-12-07T19:34:08.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe300 onodes shared_blobs 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b800 onodes shared_blobs 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b800 allocated 0x[] released 0x[] 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 prepare 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 30 -> 31 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c5500a000 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c5500d000 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 30 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 30 -> 31 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 30 -> 31 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 30 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.610+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 30/28 ECSubReadReply(tid=56, attrs_read=0)) v2 -- 0x558c57976280 con 0x558c54c5b400 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 30 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 31 dequeue_op 0x558c54e79ce0 finish 2019-12-07T19:34:08.610+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.610+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) pg 0x558c5513e000 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.610+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 30 -> 31 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 31 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.610+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 30 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe300 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 io_done 2019-12-07T19:34:08.610+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.610+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) pg 0x558c55141000 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c54d5f000 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:08.610+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 30 -> 31 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c574fe600 seq 19 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8ccff~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 30 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:08.610+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8ccff 0x8ccff~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:08.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 87 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c54e04000 con 0x558c54c5b400 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 prepare 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5ccff 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcff 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b800 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x174 and padding block with 0xe8c 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 io_done 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe600 onodes shared_blobs 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe600 allocated 0x[] released 0x[] 2019-12-07T19:34:08.610+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.bs3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 prepare 2019-12-07T19:34:08.610+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe600 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.610+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 io_done 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e31) pg 0x558c5731a000 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c55141000 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.610+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.610+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54ddf500 seq 21 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddf500 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddf500 onodes shared_blobs 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddf500 allocated 0x[] released 0x[] 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf500 prepare 2019-12-07T19:34:08.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddf500 2019-12-07T19:34:08.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf500 io_done 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8d174 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c5731a000 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000137575s (0.000000097s flush + 0.000137478s kv commit) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:08.610+0000 7f8f07988700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 2 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 10 osd.7 31 tick_without_osd_lock 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2228 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3338 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3361 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3361 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fba3000/0x40000000/0x1680000000, data 0x42857c/0x45b000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8d174~d21 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:03.373892+0000 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8d174 0x8d174~d21 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5d174 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x174 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe95 and padding block with 0x16b 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373810+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:08.610+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:03.373299+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe300> 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe300 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 nothing queued 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 finishing 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe300 onodes 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe300 done 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe600 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe300 [] 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) pg 0x558c54d9b000 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5730a900 seq 31 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 30 -> 31 2019-12-07T19:34:08.610+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 30 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a900 onodes shared_blobs 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a900 allocated 0x[] released 0x[] 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 prepare 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a900 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 io_done 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8de95 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000183178s (0.000000099s flush + 0.000183079s kv commit) 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b800,0x558c574fe600,0x558c54ddf500> 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b800 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 finishing 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b800 onodes 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b800 done 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddf500 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b800 [] 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe600 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 finishing 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe600 onodes 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe600 done 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe600 [] 2019-12-07T19:34:08.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8de95~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8de95 0x8de95~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5de95 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe95 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x30a and padding block with 0xcf6 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf500 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddf500 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf500 finishing 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddf500 onodes 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddf500 done 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddf500 [] 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c54ea0300 seq 14 2019-12-07T19:34:08.610+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes shared_blobs 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c54d9b000 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f0918b700 20 osd.7 31 agent_entry empty queue 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558c5513e000 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) pg 0x558c54d9b000 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730b500 seq 15 2019-12-07T19:34:08.610+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b500 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b500 onodes shared_blobs 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b500 allocated 0x[] released 0x[] 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 prepare 2019-12-07T19:34:08.610+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b500 2019-12-07T19:34:08.610+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 io_done 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8e30a 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000233951s (0.000000077s flush + 0.000233874s kv commit) 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a900> 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a900 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) pg 0x558c54d9b000 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 finishing 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a900 onodes 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a900 done 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a900 [] 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 2 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 31 dequeue_op 0x558c54e798c0 prio 127 cost 0 latency 0.001424 MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2228 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2251 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2251 < min_flush_size 524288 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] _handle_message: MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8e30a~8cb to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8e30a 0x8e30a~8cb to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f0317f700 25 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:08.610+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5e30a 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x30a 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~32000 size 0x32000 (204800) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbd5 and padding block with 0x42b 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 204800 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read read request=204800 r=204800 len=204800 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:08.610+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.610+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 31/28 ECSubReadReply(tid=57, attrs_read=0)) v2 -- 0x558c57454500 con 0x558c54c5b400 2019-12-07T19:34:08.610+0000 7f8f0317f700 10 osd.7 31 dequeue_op 0x558c54e798c0 finish 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8ebd5 2019-12-07T19:34:08.610+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 30 -> 31 2019-12-07T19:34:08.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) pg 0x558c54d9b000 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 31 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.001089 MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] 2019-12-07T19:34:08.610+0000 7f8f06986700 10 osd.7 pg_epoch: 30 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000204615s (0.000000064s flush + 0.000204551s kv commit) 2019-12-07T19:34:08.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.610+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] _handle_message: MOSDECSubOpRead(1.bs3 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.610+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[2~1,4~1]] state: Active advmap 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0300,0x558c5730b500> 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:34:08.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:34:08.610+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[2~1,4~1]] on_active_advmap new removed_snaps [7~1], snap_trimq now [7~1] 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:34:08.610+0000 7f8f07187700 25 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 kv_submitted 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.611+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] on_active_advmap new purged_snaps [2~1], now [4~1] 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 kv_submitted 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~32000 size 0x32000 (204800) 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b500 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 finishing 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b500 onodes 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 done 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] share_pg_info 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b500 [] 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 30 -> 31 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c57452300 con 0x558c54c64000 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 20 osd.7 31 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 20 osd.7 31 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 30 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c57450700 con 0x558c54c19800 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[2~1,4~1]] state: Active advmap 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[2~1,4~1]] on_active_advmap new removed_snaps [7~1], snap_trimq now [7~1] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] on_active_advmap new purged_snaps [2~1], now [4~1] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c576e1100 con 0x558c54c5bc00 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] share_pg_info 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c54e05c00 con 0x558c54c64400 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c576e0300 con 0x558c54c19c00 2019-12-07T19:34:08.611+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 204800 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c54e04700 con 0x558c54c64000 2019-12-07T19:34:08.611+0000 7f8f07187700 20 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read read request=204800 r=204800 len=204800 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c54e06300 con 0x558c54c19800 2019-12-07T19:34:08.611+0000 7f8f06986700 15 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] publish_stats_to_osd 29:83 2019-12-07T19:34:08.611+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] state: Started advmap 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c57450e00 con 0x558c54c5b400 2019-12-07T19:34:08.611+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 20 osd.7 31 scrub_load_below_threshold loadavg per cpu 0.0575 < max 5 = yes 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 20 osd.7 31 sched_scrub load_is_low=1 2019-12-07T19:34:08.611+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c57453800 con 0x558c54c19c00 2019-12-07T19:34:08.611+0000 7f8f07187700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 10 osd.7 31 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:08.612239+0000 2019-12-07T19:34:08.611+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 20 osd.7 31 sched_scrub done 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] plpg_on_pool_change 2019-12-07T19:34:08.611+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 10 osd.7 31 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 20 osd.7 31 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:08.611+0000 7f8f1fbbe700 10 osd.7 31 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] hit_set_clear 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f07187700 20 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] agent_stop 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.611+0000 7f8f0297e700 15 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] publish_stats_to_osd 29:33 2019-12-07T19:34:08.611+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 31/28 ECSubReadReply(tid=58, attrs_read=0)) v2 -- 0x558c57573400 con 0x558c54c5b400 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] handle_activate_map: 31 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] state: Started advmap 2019-12-07T19:34:08.611+0000 7f8f07187700 10 osd.7 31 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] activate_map 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.611+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] state: Active: handling ActMap 2019-12-07T19:34:08.611+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:08.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] Active: kicking snap trim 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] plpg_on_pool_change 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] hit_set_clear 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] agent_stop 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] exit NotTrimming 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] handle_activate_map: 31 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] enter Trimming 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] activate_map 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] enter Trimming/WaitReservation 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] state: Active: handling ActMap 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] Active: kicking snap trim 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:08.611+0000 7f8f06986700 15 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] publish_stats_to_osd 30:84 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] exit NotTrimming 2019-12-07T19:34:08.611+0000 7f8f06986700 7 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] state: handle ActMap primary 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] enter Trimming 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] enter Trimming/WaitReservation 2019-12-07T19:34:08.611+0000 7f8f06986700 15 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:85 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f0297e700 15 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] publish_stats_to_osd 30:34 2019-12-07T19:34:08.611+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7f8f0297e700 7 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] state: handle ActMap primary 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.611+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:08.611+0000 7f8f0297e700 15 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:35 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d59200 seq 21 2019-12-07T19:34:08.611+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 31 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d58300 seq 12 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:08.611+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes shared_blobs 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8ebd5~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8ebd5 0x8ebd5~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5ebd5 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbd5 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4a and padding block with 0xfb6 2019-12-07T19:34:08.611+0000 7f8f0b990700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:08.611+0000 7f8f0b990700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] SnapTrimmer state: NotTrimming: trimming 7 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] exit Trimming/WaitReservation 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f0b990700 15 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:86 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:08.611+0000 7f8f0b990700 10 osd.7 31 queueing pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[7~1] ps=[4~1]] for snaptrim 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=31) prio 5 cost 1048576 e31) 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=31) prio 5 cost 1048576 e31) queued 2019-12-07T19:34:08.611+0000 7f8f0b990700 15 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:87 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=31) prio 5 cost 1048576 e31) pg 0x558c57676000 2019-12-07T19:34:08.611+0000 7f8f0b990700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[7~1] ps=[4~1]] snap_trimmer posting 2019-12-07T19:34:08.611+0000 7f8f0b990700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] SnapTrimmer state: NotTrimming: trimming 7 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[7~1] ps=[4~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 7 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] exit Trimming/WaitReservation 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f0b990700 15 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:36 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:08.611+0000 7f8f0b990700 10 osd.7 31 queueing pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[7~1] ps=[4~1]] for snaptrim 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=31) prio 5 cost 1048576 e31) 2019-12-07T19:34:08.611+0000 7f8f0b990700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5516fec0) upper_bound after SNA_1_0000000000000007_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000007_.0_0000000000000001.C' 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7f8f0b990700 15 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:37 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=31) prio 5 cost 1048576 e31) queued 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=31) prio 5 cost 1048576 e31) pg 0x558c548ab000 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8f04a 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5516fec0) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.611+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000007_.0_0000000000000001.C) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000251363s (0.000000082s flush + 0.000251281s kv commit) 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[7~1] ps=[4~1]] snap_trimmer posting 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[7~1] ps=[4~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[7~1] ps=[4~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 7 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1]] SnapTrimmer state: adding snap 7 to purged_snaps 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8f04a~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8f04a 0x8f04a~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,7~1]] SnapTrimmer state: purged_snaps now [4~1,7~1], snap_trimq now [] 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5f04a 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4a 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d58f00 seq 22 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4bf and padding block with 0xb41 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c575c9180) upper_bound after SNA_1_0000000000000007_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000007_.0_0000000000000001.4' 2019-12-07T19:34:08.611+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c575c9180) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000007_.0_0000000000000001.4) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes shared_blobs 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:34:08.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[7~1] ps=[4~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 kv_queued 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1]] SnapTrimmer state: adding snap 7 to purged_snaps 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,7~1]] share_pg_info 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c576df500 con 0x558c54c64000 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,7~1]] SnapTrimmer state: purged_snaps now [4~1,7~1], snap_trimq now [] 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d58600 seq 13 2019-12-07T19:34:08.611+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c576e1800 con 0x558c54c19800 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58600 onodes shared_blobs 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c576b9500 con 0x558c54c5bc00 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58600 allocated 0x[] released 0x[] 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8f4bf 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 prepare 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58600 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 io_done 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c576b8e00 con 0x558c54c5b400 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,7~1]] share_pg_info 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000121255s (0.000000077s flush + 0.000121178s kv commit) 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558c574e4700 con 0x558c54c19c00 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c57451500 con 0x558c54c64400 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,7~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2220 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 88 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c54e04700 con 0x558c54c5b400 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c57452a00 con 0x558c54c64000 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,7~1]] exit Trimming 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2243 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2243 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:08.611+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.3s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8f4bf~8c3 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c57451c00 con 0x558c54c19800 2019-12-07T19:34:08.611+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8f4bf 0x8f4bf~8c3 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f06986700 15 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] publish_stats_to_osd 31:88 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5f4bf 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4bf 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd82 and padding block with 0x27e 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c54e04e00 con 0x558c54c5b400 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] enter NotTrimming 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:08.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 -- 0x558c54e05500 con 0x558c54c19c00 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,7~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:08.611+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] snap_trimmer complete 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300> 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,7~1]] exit Trimming 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 kv_submitted 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.611+0000 7f8f0297e700 15 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] publish_stats_to_osd 31:38 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] enter NotTrimming 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:08.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] snap_trimmer complete 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x8fd82 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000128841s (0.000000074s flush + 0.000128767s kv commit) 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) pg 0x558c5513e000 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00,0x558c54d58600> 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 kv_submitted 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58600 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 finishing 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58600 onodes 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 done 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58600 [] 2019-12-07T19:34:08.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea1800 seq 32 2019-12-07T19:34:08.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:08.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1800 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1800 onodes shared_blobs 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1800 allocated 0x[] released 0x[] 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 prepare 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1800 2019-12-07T19:34:08.611+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 io_done 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.611+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 638 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1121 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1144 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1144 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x8fd82~478 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x8fd82 0x8fd82~478 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:06.674086+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _allocate len 0x1fa from 0 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _allocate len 0x1fa from 1 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd1c0000 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd1c0000~10000/10000,10000,acd1c0000 2019-12-07T19:34:08.611+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0xa0000 to include allocated 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 31 (was clean) 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x5fd82 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd82 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1fa and padding block with 0xe06 2019-12-07T19:34:08.611+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0xa0000 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x901fa 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (31) on file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]), flushing log 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 31 len 0x49 crc 0xf9c61c64) 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x1e000~1000 to file(ino 1 size 0x1e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x1e000 0x1e000~1000 to file(ino 1 size 0x1e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x1f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1e000 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x1f000 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:08.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000037 2019-12-07T19:34:08.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.613+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 31 2019-12-07T19:34:08.613+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001605309s (0.000000079s flush + 0.001605230s kv commit) 2019-12-07T19:34:08.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1800> 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 kv_submitted 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1800 2019-12-07T19:34:08.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 finishing 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1800 onodes 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1800 done 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:08.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1800 [] 2019-12-07T19:34:08.613+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 130 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:08.616+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.0s3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:08.616+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 131 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:08.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.616+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.7s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e31) pg 0x558c5500d000 2019-12-07T19:34:08.616+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.616+0000 7f8f04982700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:08.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) pg 0x558c54d5f000 2019-12-07T19:34:08.616+0000 7f8f04982700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.616+0000 7f8f05183700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.616+0000 7f8f05183700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.616+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:08.616+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:08.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:08.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54ea0000 seq 21 2019-12-07T19:34:08.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:08.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54dde900 seq 34 2019-12-07T19:34:08.616+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:08.616+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:08.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:08.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:08.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:08.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:08.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:08.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:08.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:08.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde900 onodes shared_blobs 2019-12-07T19:34:08.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde900 allocated 0x[] released 0x[] 2019-12-07T19:34:08.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:08.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes shared_blobs 2019-12-07T19:34:08.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[] released 0x[] 2019-12-07T19:34:08.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde900 prepare 2019-12-07T19:34:08.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde900 2019-12-07T19:34:08.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde900 io_done 2019-12-07T19:34:08.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:34:08.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:34:08.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:34:08.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x901fa~8d3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x901fa 0x901fa~8d3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x601fa 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1fa 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xacd and padding block with 0x533 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x90acd 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000110715s (0.000000146s flush + 0.000110569s kv commit) 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde900,0x558c54ea0000> 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde900 kv_submitted 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde900 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde900 finishing 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde900 onodes 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde900 done 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde900 [] 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:34:08.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:34:08.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 132 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:08.619+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.9s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.619+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) pg 0x558c5500a000 2019-12-07T19:34:08.619+0000 7f8f04181700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.619+0000 7f8f04181700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:08.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:08.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740a600 seq 31 2019-12-07T19:34:08.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:08.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:08.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:08.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:08.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:08.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes shared_blobs 2019-12-07T19:34:08.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:08.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:08.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:08.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x90acd~475 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x90acd 0x90acd~475 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x60acd 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xacd 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf42 and padding block with 0xbe 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x90f42 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000103081s (0.000000083s flush + 0.000102998s kv commit) 2019-12-07T19:34:08.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a600> 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:08.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:08.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:08.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 133 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:08.625+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.0s3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:08.625+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 134 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e31) pg 0x558c5500d000 2019-12-07T19:34:08.625+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.7s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.625+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) pg 0x558c54d5f000 2019-12-07T19:34:08.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 135 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.625+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.9s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.625+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c548f6c00 seq 22 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.625+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1b00 seq 35 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes shared_blobs 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.625+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:34:08.625+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes shared_blobs 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:34:08.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:34:08.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) pg 0x558c5500a000 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.625+0000 7f8f08189700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:08.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:08.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5730b200 seq 32 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2220 < min_flush_size 524288 2019-12-07T19:34:08.625+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:08.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:08.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2243 < min_flush_size 524288 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2243 < min_flush_size 524288 2019-12-07T19:34:08.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:08.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes shared_blobs 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[] released 0x[] 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x90f42~8c3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x90f42 0x90f42~8c3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:08.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x60f42 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf42 2019-12-07T19:34:08.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x805 and padding block with 0x7fb 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x91805 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000106513s (0.000000176s flush + 0.000106337s kv commit) 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6c00,0x558c54ea1b00> 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 finishing 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x91805~46d to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:34:08.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x91805 0x91805~46d to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x61805 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x805 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc72 and padding block with 0x38e 2019-12-07T19:34:08.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x91c72 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000108541s (0.000000098s flush + 0.000108443s kv commit) 2019-12-07T19:34:08.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b200> 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [] 2019-12-07T19:34:08.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 86 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c576b9500 con 0x558c54c5bc00 2019-12-07T19:34:08.627+0000 7f8f23c56700 15 osd.7 31 enqueue_peering_evt 1.6s2 epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:08.627+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e31) pg 0x558c54d5c000 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c574ffb00 seq 49 2019-12-07T19:34:08.627+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ffb00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ffb00 onodes shared_blobs 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ffb00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 prepare 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ffb00 2019-12-07T19:34:08.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 io_done 2019-12-07T19:34:08.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x91c72~475 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x91c72 0x91c72~475 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x61c72 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc72 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe7 and padding block with 0xf19 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x920e7 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123172s (0.000000087s flush + 0.000123085s kv commit) 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ffb00> 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 kv_submitted 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ffb00 2019-12-07T19:34:08.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 finishing 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ffb00 onodes 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ffb00 done 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:08.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ffb00 [] 2019-12-07T19:34:08.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 87 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c576b9500 con 0x558c54c5bc00 2019-12-07T19:34:08.628+0000 7f8f23c56700 15 osd.7 31 enqueue_peering_evt 1.6s2 epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:08.628+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e31) pg 0x558c54d5c000 2019-12-07T19:34:08.628+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:08.628+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.628+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:08.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:08.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d58c00 seq 50 2019-12-07T19:34:08.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:08.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:08.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:08.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:08.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 2019-12-07T19:34:08.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:34:08.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:34:08.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:34:08.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x920e7~46d to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x920e7 0x920e7~46d to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x620e7 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe7 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x554 and padding block with 0xaac 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x92554 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:08.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118713s (0.000000122s flush + 0.000118591s kv commit) 2019-12-07T19:34:08.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:34:08.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:08.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:34:08.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:08.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:08.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.634+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.653+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 9 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 0.019443140s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c18c00 2019-12-07T19:34:08.653+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019030755s,0.019443140s]) 2019-12-07T19:34:08.653+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.090374596s delta_ub 18446744073.690520861s) v5 -- 0x558c575b9800 con 0x558c54c18c00 2019-12-07T19:34:08.653+0000 7f8f23c56700 10 osd.7 31 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:08.653+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 9 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 0.019443140s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c18400 2019-12-07T19:34:08.653+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.018967925s,0.019443140s]) 2019-12-07T19:34:08.653+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.090437426s delta_ub 18446744073.690583691s) v5 -- 0x558c54d77000 con 0x558c54c18400 2019-12-07T19:34:08.685+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.735+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.785+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.835+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.885+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.935+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.986+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.036+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.086+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.114+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:09.136+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.186+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.236+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.272+0000 7f8f0097a700 5 osd.7 31 heartbeat osd_stat(store_statfs(0x163fba3000/0x40000000/0x1680000000, data 0x42857c/0x45b000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5] op hist []) 2019-12-07T19:34:09.272+0000 7f8f0097a700 20 osd.7 31 check_full_status cur ratio 4.73446e-05, physical ratio 4.73446e-05, new state none 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.690583691s) v5 -- 0x558c5768b400 con 0x558c5476d400 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.690583691s) v5 -- 0x558c5768b600 con 0x558c551b6800 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.688441377s) v5 -- 0x558c5768b800 con 0x558c54c9e000 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.688441377s) v5 -- 0x558c5768ba00 con 0x558c54c9e400 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.673534223s) v5 -- 0x558c5768bc00 con 0x558c551b7800 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.673534223s) v5 -- 0x558c5768be00 con 0x558c551b7c00 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.687846720s) v5 -- 0x558c55251400 con 0x558c552c2000 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.687846720s) v5 -- 0x558c57519e00 con 0x558c552c2400 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.696109269s) v5 -- 0x558c55270e00 con 0x558c552c2800 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.696109269s) v5 -- 0x558c55271800 con 0x558c552c2c00 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.678921487s) v5 -- 0x558c55271600 con 0x558c54c65800 2019-12-07T19:34:09.272+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.678921487s) v5 -- 0x558c55271400 con 0x558c54c65c00 2019-12-07T19:34:09.272+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.723069853s delta_ub 0.013979962s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c552c2800 2019-12-07T19:34:09.272+0000 7f8f24c58700 25 osd.7 31 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:03.373595+0000 -> 2019-12-07T19:34:09.273917+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 2019-12-07T19:34:09.272+0000 7f8f24c58700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013713437s,0.013979962s]) 2019-12-07T19:34:09.272+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 5 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.740173432s delta_ub 0.031083541s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c65800 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:03.373556+0000 -> 2019-12-07T19:34:09.273936+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 2019-12-07T19:34:09.272+0000 7f8f23c56700 10 osd.7 31 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 30 -> 31 (as per caller) 2019-12-07T19:34:09.272+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030797280s,0.031083541s]) 2019-12-07T19:34:09.272+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.723127281s delta_ub 0.014037390s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55271e00 con 0x558c552c2c00 2019-12-07T19:34:09.272+0000 7f8f24457700 25 osd.7 31 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.273917+0000 last_rx_front 2019-12-07T19:34:03.373616+0000 -> 2019-12-07T19:34:09.273950+0000 2019-12-07T19:34:09.272+0000 7f8f24457700 25 osd.7 31 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:09.273650+0000) and older pending ping(s) 2019-12-07T19:34:09.272+0000 7f8f24457700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013737117s,0.014037390s]) 2019-12-07T19:34:09.272+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 5 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.740198955s delta_ub 0.031109064s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c65c00 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.273936+0000 last_rx_front 2019-12-07T19:34:03.373582+0000 -> 2019-12-07T19:34:09.273966+0000 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:09.273650+0000) and older pending ping(s) 2019-12-07T19:34:09.272+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030793431s,0.031109064s]) 2019-12-07T19:34:09.272+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.731562030s delta_ub 0.022472139s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55271e00 con 0x558c552c2000 2019-12-07T19:34:09.272+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.745767822s delta_ub 0.036677931s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c551b7800 2019-12-07T19:34:09.272+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 5 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.730971591s delta_ub 0.021881700s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c9e400 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:03.373741+0000 -> 2019-12-07T19:34:09.274078+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 2019-12-07T19:34:09.272+0000 7f8f23c56700 10 osd.7 31 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:09.272+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036249961s,0.036677931s]) 2019-12-07T19:34:09.272+0000 7f8f24457700 25 osd.7 31 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:03.373810+0000 -> 2019-12-07T19:34:09.274091+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 2019-12-07T19:34:09.272+0000 7f8f24457700 10 osd.7 31 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:09.272+0000 7f8f24457700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022031139s,0.022472139s]) 2019-12-07T19:34:09.272+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.728737829s delta_ub 0.019647938s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c5476d400 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:03.373801+0000 -> 2019-12-07T19:34:09.274103+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 2019-12-07T19:34:09.272+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019194867s,0.019647938s]) 2019-12-07T19:34:09.272+0000 7f8f24c58700 25 osd.7 31 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 last_rx_front 2019-12-07T19:34:03.373892+0000 -> 2019-12-07T19:34:09.274111+0000 2019-12-07T19:34:09.272+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 5 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.731015302s delta_ub 0.021925411s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c9e000 2019-12-07T19:34:09.272+0000 7f8f24c58700 10 osd.7 31 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:09.272+0000 7f8f24c58700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021420701s,0.021881700s]) 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:03.373784+0000 -> 2019-12-07T19:34:09.274127+0000 last_rx_front 2019-12-07T19:34:09.274111+0000 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:09.273650+0000) and older pending ping(s) 2019-12-07T19:34:09.272+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021448798s,0.021925411s]) 2019-12-07T19:34:09.272+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.745792626s delta_ub 0.036702735s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c551b7c00 2019-12-07T19:34:09.272+0000 7f8f24c58700 25 osd.7 31 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274078+0000 last_rx_front 2019-12-07T19:34:03.373832+0000 -> 2019-12-07T19:34:09.274140+0000 2019-12-07T19:34:09.272+0000 7f8f24c58700 25 osd.7 31 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:09.273650+0000) and older pending ping(s) 2019-12-07T19:34:09.272+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.731614645s delta_ub 0.022524754s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c552c2400 2019-12-07T19:34:09.272+0000 7f8f24c58700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036212713s,0.036702735s]) 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274091+0000 last_rx_front 2019-12-07T19:34:03.373756+0000 -> 2019-12-07T19:34:09.274157+0000 2019-12-07T19:34:09.272+0000 7f8f23c56700 25 osd.7 31 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:09.273650+0000) and older pending ping(s) 2019-12-07T19:34:09.272+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022017687s,0.022524754s]) 2019-12-07T19:34:09.272+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.728785685s delta_ub 0.019695794s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c551b6800 2019-12-07T19:34:09.272+0000 7f8f24c58700 25 osd.7 31 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274103+0000 last_rx_front 2019-12-07T19:34:03.373866+0000 -> 2019-12-07T19:34:09.274169+0000 2019-12-07T19:34:09.272+0000 7f8f24c58700 25 osd.7 31 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:34:09.273650+0000) and older pending ping(s) 2019-12-07T19:34:09.272+0000 7f8f24c58700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019177024s,0.019695794s]) 2019-12-07T19:34:09.286+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.302+0000 7f8f21451700 10 osd.7 31 tick 2019-12-07T19:34:09.302+0000 7f8f21451700 10 osd.7 31 do_waiters -- start 2019-12-07T19:34:09.302+0000 7f8f21451700 10 osd.7 31 do_waiters -- finish 2019-12-07T19:34:09.302+0000 7f8f21451700 20 osd.7 31 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:09.336+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.353+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 8 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.811320781s delta_ub 0.021925411s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c18800 2019-12-07T19:34:09.353+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021208568s,0.021925411s]) 2019-12-07T19:34:09.353+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.790112213s delta_ub 18446744073.688343048s) v5 -- 0x558c55271400 con 0x558c54c18800 2019-12-07T19:34:09.353+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 8 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.811320781s delta_ub 0.021925411s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c19000 2019-12-07T19:34:09.353+0000 7f8f23c56700 20 osd.7 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021058329s,0.021925411s]) 2019-12-07T19:34:09.353+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.790262452s delta_ub 18446744073.688493287s) v5 -- 0x558c55271400 con 0x558c54c19000 2019-12-07T19:34:09.386+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.437+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.486+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.537+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.587+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 425463 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 10 osd.7 31 tick_without_osd_lock 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fba3000/0x40000000/0x1680000000, data 0x42857c/0x45b000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274103+0000 last_rx_front 2019-12-07T19:34:09.274169+0000 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274127+0000 last_rx_front 2019-12-07T19:34:09.274111+0000 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274078+0000 last_rx_front 2019-12-07T19:34:09.274140+0000 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274091+0000 last_rx_front 2019-12-07T19:34:09.274157+0000 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.273917+0000 last_rx_front 2019-12-07T19:34:09.273950+0000 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 25 osd.7 31 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.273936+0000 last_rx_front 2019-12-07T19:34:09.273966+0000 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 20 osd.7 31 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 10 osd.7 31 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 20 osd.7 31 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:09.597+0000 7f8f1fbbe700 10 osd.7 31 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:09.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 88 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c576b9500 con 0x558c54c5bc00 2019-12-07T19:34:09.606+0000 7f8f23c56700 15 osd.7 31 enqueue_peering_evt 1.6s2 epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:09.606+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e32) pg 0x558c54d5c000 2019-12-07T19:34:09.606+0000 7f8f0197c700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 32, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.607+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 49 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57451c00 con 0x558c54c19800 2019-12-07T19:34:09.607+0000 7f8f24c58700 15 osd.7 31 enqueue_peering_evt 1.ds2 epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:09.607+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) pg 0x558c55141000 2019-12-07T19:34:09.607+0000 7f8f0217d700 20 osd.7 op_wq _add_slot_waiter 1.ds2 peering, item epoch is 32, will wait on OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 7 ==== osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 ==== 302+0+4 (crc 0 0 0) 0x558c552302c0 con 0x558c54c9f000 2019-12-07T19:34:09.608+0000 7f8f24457700 15 osd.7 31 enqueue_op 0x558c578d2c60 prio 63 cost 4 latency 0.000019 epoch 32 osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.608+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 8 ==== osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 ==== 264+0+4 (crc 0 0 0) 0x558c55230840 con 0x558c54c9f000 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) queued 2019-12-07T19:34:09.608+0000 7f8f24457700 15 osd.7 31 enqueue_op 0x558c578d2160 prio 63 cost 4 latency 0.000008 epoch 32 osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.608+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) pg 0x558c57676000 2019-12-07T19:34:09.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 9 ==== osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 ==== 264+0+4 (crc 0 0 0) 0x558c55230580 con 0x558c54c9f000 2019-12-07T19:34:09.608+0000 7f8f06986700 10 osd.7 31 dequeue_op 0x558c578d2c60 prio 63 cost 4 latency 0.000049 osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] 2019-12-07T19:34:09.608+0000 7f8f24457700 15 osd.7 31 enqueue_op 0x558c578d29a0 prio 63 cost 4 latency 0.000009 epoch 32 osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.608+0000 7f8f06986700 10 osd.7 31 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 29 -> 32 (as per caller) 2019-12-07T19:34:09.608+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] do_request min 32, queue on waiting_for_map client.4540 2019-12-07T19:34:09.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) queued 2019-12-07T19:34:09.608+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=32}) v3 -- 0x558c57518e00 con 0x558c5476d000 2019-12-07T19:34:09.608+0000 7f8f06986700 10 osd.7 31 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) queued 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) pg 0x558c57676000 2019-12-07T19:34:09.608+0000 7f8f06986700 10 osd.7 31 dequeue_op 0x558c578d2160 prio 63 cost 4 latency 0.000090 osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:34:09.608+0000 7f8f06986700 10 osd.7 31 dequeue_op 0x558c578d2160 finish 2019-12-07T19:34:09.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) pg 0x558c57676000 2019-12-07T19:34:09.608+0000 7f8f0297e700 10 osd.7 31 dequeue_op 0x558c578d29a0 prio 63 cost 4 latency 0.000294 osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] 2019-12-07T19:34:09.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:34:09.608+0000 7f8f0297e700 10 osd.7 31 dequeue_op 0x558c578d29a0 finish 2019-12-07T19:34:09.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.608+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 19 ==== osd_map(32..32 src has 1..32) v4 ==== 661+0+0 (crc 0 0 0) 0x558c57944d80 con 0x558c5476d000 2019-12-07T19:34:09.608+0000 7f8f164bb700 20 osd.7 31 OSD::ms_dispatch: osd_map(32..32 src has 1..32) v4 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 osd.7 31 do_waiters -- start 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 osd.7 31 do_waiters -- finish 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 osd.7 31 _dispatch 0x558c57944d80 osd_map(32..32 src has 1..32) v4 2019-12-07T19:34:09.609+0000 7f8f164bb700 3 osd.7 31 handle_osd_map epochs [32,32], i have 31, src has [1,32] 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 osd.7 31 handle_osd_map got inc map for epoch 32 2019-12-07T19:34:09.609+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3398e826:::osdmap.31:0# 0x0~0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1470 size 0x1470 (5232) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bde30 blob([0x4f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c552657a0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:09.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 89 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c54e05500 con 0x558c54c5b400 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265880 blob([!~1000,0x50000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,470]) SharedBlob(0x558c57396a80 sbid 0x0)) need 0x1000~470 cache has 0x[1000~470] 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3398e826:::osdmap.31:0# 0x0~1470 = 5232 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 osd.7 31 add_map_bl 31 5232 bytes 2019-12-07T19:34:09.609+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.bs3 epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:09.609+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 90 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c54e05500 con 0x558c54c5b400 2019-12-07T19:34:09.609+0000 7f8f24457700 15 osd.7 31 enqueue_peering_evt 1.3s4 epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:09.609+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 osd.7 31 got_full_map 32, nothing requested 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) pg 0x558c54d9b000 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 32, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) pg 0x558c5513e000 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 32, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 osd.7 31 handle_osd_map pg_num_history pg_num_history(e32 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {32={}} 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 osd.7 31 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e32 [1,32] lci=[13,32]) 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c577fc000 seq 23 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c18b972f:::inc_osdmap.32:0# key 0x7f7fffffffffffffffc18b97'/!inc_osdmap.32!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c18b972f:::inc_osdmap.32:0# 0x0~269 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1145 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c18b972f:::inc_osdmap.32:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc000 1 blobs 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x51000~1000/1000,1000,0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x51000~1000] 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa850 blob([0x51000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa690 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c574fa850) get_ref 0x0~269 Blob(0x558c574fa850 blob([0x51000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa690 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c574fa850) get_ref init 0x1000, 1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c574fa850 blob([0x51000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c574fa690 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c574fa6a8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54e44388) dirty_range mark inline shard dirty 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c18b972f:::inc_osdmap.32:0# 0x0~269 = 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3978e826:::osdmap.32:0# key 0x7f7fffffffffffffff3978e8'&!osdmap.32!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3978e826:::osdmap.32:0# 0x0~1454 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1146 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3978e826:::osdmap.32:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~454 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc000 2 blobs 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x52000~2000/1000,2000,0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x52000~2000] 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac700 blob([0x52000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881110 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c576ac700) get_ref 0x0~1000 Blob(0x558c576ac700 blob([0x52000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881110 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c576ac700) get_ref init 0x1000, 1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c576ac700 blob([0x52000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57881110 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57881128 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd730 blob([!~1000,0x53000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f180 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c575bd730) get_ref 0x1000~454 Blob(0x558c575bd730 blob([!~1000,0x53000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c5757f180 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c575bd730) get_ref init 0x2000, 1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558c575bd730 blob([!~1000,0x53000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c5757f180 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5757f198 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1454 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54e45508) dirty_range mark inline shard dirty 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3978e826:::osdmap.32:0# 0x0~1454 = 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc000 0 blobs 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc000 0 blobs 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc000 cost 23302 (4 ios * 4000 + 7302 bytes) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc000 onodes 0x558c53a91900,0x558c54e44280,0x558c54e45400,0x558c55276c80 shared_blobs 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54e44388) update #-1:c18b972f:::inc_osdmap.32:0# 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54e44388) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c18b972f:::inc_osdmap.32:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54e45508) update #-1:3978e826:::osdmap.32:0# 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54e45508) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3978e826:::osdmap.32:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc000 allocated 0x[51000~3000] released 0x[] 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 freelist allocate 0x51000~3000 2019-12-07T19:34:09.609+0000 7f8f164bb700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 prepare 2019-12-07T19:34:09.609+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fc000 2019-12-07T19:34:09.609+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 aio_wait 2019-12-07T19:34:09.609+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc000 2019-12-07T19:34:09.609+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 io_done 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44280 had 1 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45400 had 1 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860201472 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998627) bluestore 86 GiB free (0.959952), bluefs_ratio 0.0399496 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x92554~226c to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x92554 0x92554~226c to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x62554 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x554 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7c0 and padding block with 0x840 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x947c0 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000128408s (0.000004328s flush + 0.000124080s kv commit) 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc000> 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 kv_submitted 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc000 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c577fc000 osr 0x558c54ea4b40 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) _discard drop 20 0x1b000~1000 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 21 0x1b000~1000 crc 8d83060d 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) _discard drop 20 0x2000~1000 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7 31 _committed_osd_maps 32..32 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c54de8780) prepare_write seq 21 0x2000~1000 crc 5f5a0ec9 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7 31 advance to epoch 32 (<= last 32 <= newest_map 32) 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.609+0000 7f8f141a1700 7 osd.7 32 consume_map version 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.0s3 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.9s4 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.2s4 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.bs3 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.3s4 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.cs0 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.4s0 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.ds2 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.6s2 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 identify_splits_and_merges 1.7s4 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 32 prune_pg_created 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:0.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:1.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:2.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:3.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:4.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:5.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.ds2 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) pg 0x558c5513e000 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:6.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7:7.consume_map 32 (was 31) 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) pg 0x558c55141000 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e32) pg 0x558c54d5c000 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 10 osd.7 32 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.0s3 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 31 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 31 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.9s4 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.2s4 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c5500d000 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:09.609+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.bs3 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.3s4 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c5731a000 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.cs0 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.4s0 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.ds2 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 31 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.6s2 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f141a1700 15 osd.7 32 enqueue_peering_evt 1.7s4 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c5500a000 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 31 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:09.609+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f141a1700 7 osd.7 32 activate_map version 32 2019-12-07T19:34:09.609+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:09.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730a300 seq 51 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.609+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c54d5f000 2019-12-07T19:34:09.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 31 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:09.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a300 onodes shared_blobs 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:09.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a300 allocated 0x[] released 0x[] 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea0600 seq 33 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 prepare 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a300 2019-12-07T19:34:09.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 io_done 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:09.609+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 31 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:09.609+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0600 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c54d5c000 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0600 onodes shared_blobs 2019-12-07T19:34:09.609+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0600 allocated 0x[] released 0x[] 2019-12-07T19:34:09.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:09.609+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 prepare 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f05984700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0600 2019-12-07T19:34:09.609+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 io_done 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x947c0~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x947c0 0x947c0~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x647c0 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7c0 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc45 and padding block with 0x3bb 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) pg 0x558c54d9b000 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:09.609+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 31 -> 32 2019-12-07T19:34:09.609+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c574fe900 seq 20 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 31 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:09.609+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.609+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:09.609+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.609+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:09.609+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.609+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:09.609+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe900 onodes shared_blobs 2019-12-07T19:34:09.609+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe900 allocated 0x[] released 0x[] 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.609+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.609+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 prepare 2019-12-07T19:34:09.609+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe900 2019-12-07T19:34:09.609+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 io_done 2019-12-07T19:34:09.609+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x94c45 2019-12-07T19:34:09.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c55141000 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.609+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107327s (0.000000120s flush + 0.000107207s kv commit) 2019-12-07T19:34:09.609+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:09.609+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a300> 2019-12-07T19:34:09.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 kv_submitted 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a300 2019-12-07T19:34:09.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 finishing 2019-12-07T19:34:09.609+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a300 onodes 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a300 done 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c54ea0900 seq 16 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:09.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a300 [] 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:09.609+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:09.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.609+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:09.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:09.609+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:09.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x94c45~8f3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x94c45 0x94c45~8f3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes shared_blobs 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[] released 0x[] 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x64c45 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc45 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x538 and padding block with 0xac8 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c54d9b000 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c5513e000 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.610+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x95538 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000115869s (0.000000070s flush + 0.000115799s kv commit) 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0600,0x558c574fe900> 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 kv_submitted 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0600 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 finishing 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0600 onodes 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0600 done 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0600 [] 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 kv_submitted 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe900 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 finishing 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe900 onodes 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x95538~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe900 done 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe900 [] 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x95538 0x95538~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x65538 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x538 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9bd and padding block with 0x643 2019-12-07T19:34:09.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x959bd 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000093023s (0.000000048s flush + 0.000092975s kv commit) 2019-12-07T19:34:09.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0900> 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:34:09.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.610+0000 7f8f0918b700 20 osd.7 32 agent_entry empty queue 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c57676000 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 31 -> 32 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 31 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,7~1]] state: Active advmap 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] on_active_advmap new removed_snaps [6~1], snap_trimq now [6~1] 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.610+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] state: Active: handling ActMap 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] Active: kicking snap trim 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] exit NotTrimming 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming/WaitReservation 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f06986700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 31:89 2019-12-07T19:34:09.610+0000 7f8f06986700 7 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] state: handle ActMap primary 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f06986700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 31:90 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.610+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] requeue_map_waiters client.4540 0x558c578d2c60,0x558c578d2160,0x558c578d29a0 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.610+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558c548ab000 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) queued 2019-12-07T19:34:09.610+0000 7f8f0b990700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:09.610+0000 7f8f0b990700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: NotTrimming: trimming 6 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] exit Trimming/WaitReservation 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 31 -> 32 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 31 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:09.610+0000 7f8f0b990700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 32:91 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,7~1]] state: Active advmap 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] on_active_advmap new removed_snaps [6~1], snap_trimq now [6~1] 2019-12-07T19:34:09.610+0000 7f8f0b990700 10 osd.7 32 queueing pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[6~1] ps=[4~1,7~1]] for snaptrim 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=32) prio 5 cost 1048576 e32) 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.610+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:09.610+0000 7f8f0b990700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 32:92 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) pg 0x558c57676000 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d2c60 prio 63 cost 4 latency 0.001874 osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] _handle_message: osd_op(client.4540.0:266 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] state: Active: handling ActMap 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] Active: kicking snap trim 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op: op osd_op(client.4540.0:266 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] exit NotTrimming 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op osd_op(client.4540.0:266 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e32) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming/WaitReservation 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 4=[]:{} 2019-12-07T19:34:09.610+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 31:39 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:09.610+0000 7f8f0297e700 7 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] state: handle ActMap primary 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f06986700 25 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:09.610+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 31:40 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(read n=1 w=0)) 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] execute_ctx 0x558c574e4000 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.610+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b,getxattrs] ov 29'18 2019-12-07T19:34:09.610+0000 7f8f0b990700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:09.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) queued 2019-12-07T19:34:09.610+0000 7f8f0b990700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: NotTrimming: trimming 6 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] exit Trimming/WaitReservation 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b,getxattrs] 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op read 0~1474560 2019-12-07T19:34:09.610+0000 7f8f0b990700 15 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 32:41 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] async_read noted for 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.610+0000 7f8f0b990700 10 osd.7 32 queueing pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[6~1] ps=[4~1,7~1]] for snaptrim 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=32) prio 5 cost 1048576 e32) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op checksum in=4b 2019-12-07T19:34:09.610+0000 7f8f0b990700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_checksum 2019-12-07T19:34:09.610+0000 7f8f0b990700 15 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 32:42 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_checksum: async_read noted for 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] start_read_op: starting ReadOp(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read_op: starting read ReadOp(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 send_incremental_map 31 -> 32 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:09.610+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c18b972f:::inc_osdmap.32:0# 0x0~0 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa850 blob([0x51000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c574fa690 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:09.610+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c18b972f:::inc_osdmap.32:0# 0x0~269 = 617 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 add_map_inc_bl 32 617 bytes 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(32..32 src has 1..32) v4 -- 0x558c54dd1b00 con 0x558c54c64000 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 31 -> 32 (shared) 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57454500 con 0x558c54c64000 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 send_incremental_map 31 -> 32 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(32..32 src has 1..32) v4 -- 0x558c54dd1d40 con 0x558c54c5b400 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 31 -> 32 (shared) 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738c000 con 0x558c54c5b400 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 send_incremental_map 31 -> 32 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(32..32 src has 1..32) v4 -- 0x558c5738bd40 con 0x558c54c19c00 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 31 -> 32 (shared) 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738c280 con 0x558c54c19c00 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 send_incremental_map 29 -> 32 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:34:09.610+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c4cb972f:::inc_osdmap.30:0# 0x0~0 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576acee0 blob([0x3b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c576aca10 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:09.610+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c4cb972f:::inc_osdmap.30:0# 0x0~24d = 589 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 add_map_inc_bl 30 589 bytes 2019-12-07T19:34:09.610+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cf0b972f:::inc_osdmap.31:0# 0x0~0 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~285 size 0x285 (645) 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:09.610+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ace00 blob([0x4e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x558c575bc1c0 sbid 0x0)) need 0x0~285 cache has 0x[0~285] 2019-12-07T19:34:09.610+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cf0b972f:::inc_osdmap.31:0# 0x0~285 = 645 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 add_map_inc_bl 31 645 bytes 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(30..32 src has 1..32) v4 -- 0x558c5738e000 con 0x558c53ae5400 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 29 -> 32 (shared) 2019-12-07T19:34:09.610+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738c500 con 0x558c53ae5400 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read_op: started ReadOp(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2),5(1),7(0)) 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) queued 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) pg 0x558c57676000 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d2160 prio 63 cost 4 latency 0.002308 osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] _handle_message: osd_op(client.4540.0:267 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op: op osd_op(client.4540.0:267 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.610+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op osd_op(client.4540.0:267 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e32) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:09.610+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(read n=1 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 4=[]:{} 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:09.611+0000 7f8f06986700 25 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(read n=2 w=0)) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] execute_ctx 0x558c548a2a00 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] ov 29'18 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op read 0~1474560 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] async_read noted for 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op checksum in=4b 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_checksum 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_checksum: async_read noted for 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] start_read_op: starting ReadOp(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read_op: starting read ReadOp(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738c780 con 0x558c54c64000 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738ca00 con 0x558c54c5b400 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738cc80 con 0x558c54c19c00 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738cf00 con 0x558c53ae5400 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read_op: started ReadOp(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2),5(1),7(0)) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d2160 finish 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=32) prio 5 cost 1048576 e32) queued 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8) prio 63 cost 4 e32) pg 0x558c57676000 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d29a0 prio 63 cost 4 latency 0.002684 osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] _handle_message: osd_op(client.4540.0:268 1.cs0 1.dafc5a5c (undecoded) ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op: op osd_op(client.4540.0:268 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e32) v8 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op osd_op(client.4540.0:268 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e32) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(read n=2 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 4=[]:{} 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:09.611+0000 7f8f06986700 25 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(read n=3 w=0)) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] execute_ctx 0x558c57450000 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] ov 29'18 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560,checksum in=4b] 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op read 0~1474560 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] async_read noted for 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_op checksum in=4b 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_checksum 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_checksum: async_read noted for 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] start_read_op: starting ReadOp(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read_op: starting read ReadOp(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738d180 con 0x558c54c64000 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738d400 con 0x558c54c5b400 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738d680 con 0x558c54c19c00 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738d900 con 0x558c53ae5400 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] do_read_op: started ReadOp(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, []))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2),5(1),7(0)) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d29a0 finish 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=32) prio 5 cost 1048576 e32) queued 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=32) prio 5 cost 1048576 e32) pg 0x558c548ab000 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=32) prio 5 cost 1048576 e32) pg 0x558c57676000 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] snap_trimmer posting 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 6 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] snap_trimmer posting 2019-12-07T19:34:09.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 6 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:09.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c575c9e00) upper_bound after SNA_1_0000000000000006_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000006_.0_0000000000000001.4' 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c575c9e00) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54dccb40) upper_bound after SNA_1_0000000000000006_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000006_.0_0000000000000001.C' 2019-12-07T19:34:09.611+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000006_.0_0000000000000001.4) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54dccb40) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000006_.0_0000000000000001.C) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,7~1]] SnapTrimmer state: adding snap 6 to purged_snaps 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,7~1]] SnapTrimmer state: adding snap 6 to purged_snaps 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:09.611+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2]] SnapTrimmer state: purged_snaps now [4~1,6~2], snap_trimq now [] 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2]] SnapTrimmer state: purged_snaps now [4~1,6~2], snap_trimq now [] 2019-12-07T19:34:09.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d58000 seq 14 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c577fcc00 seq 23 2019-12-07T19:34:09.611+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:09.611+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:09.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcc00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcc00 onodes shared_blobs 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcc00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 prepare 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcc00 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 io_done 2019-12-07T19:34:09.611+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c5738c500 con 0x558c53ae5400 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2]] share_pg_info 2019-12-07T19:34:09.611+0000 7f8f1119b700 15 osd.7 32 enqueue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000901 epoch 32 MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:09.611+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) 2019-12-07T19:34:09.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558c54e07100 con 0x558c54c64000 2019-12-07T19:34:09.611+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c5738cf00 con 0x558c53ae5400 2019-12-07T19:34:09.611+0000 7f8f1119b700 15 osd.7 32 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000516 epoch 32 MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:09.611+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) 2019-12-07T19:34:09.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558c54e05c00 con 0x558c54c19800 2019-12-07T19:34:09.611+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c5738d900 con 0x558c53ae5400 2019-12-07T19:34:09.611+0000 7f8f1119b700 15 osd.7 32 enqueue_op 0x558c54e79080 prio 127 cost 0 latency 0.000220 epoch 32 MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:09.611+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) 2019-12-07T19:34:09.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558c54e04700 con 0x558c54c5bc00 2019-12-07T19:34:09.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558c57450e00 con 0x558c54c5b400 2019-12-07T19:34:09.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558c57390000 con 0x558c54c19c00 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2]] exit Trimming 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:09.611+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:93 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] enter NotTrimming 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] snap_trimmer complete 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d34a0 prio 127 cost 0 latency 0.001063 MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:09.611+0000 7f8f0297e700 25 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:09.611+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~5a000 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~5a000 size 0x5a000 (368640) 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:09.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:34:09.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:34:09.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2]] share_pg_info 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 -- 0x558c57452300 con 0x558c54c64400 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(30..32 src has 1..32) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c5738e000 con 0x558c53ae5400 2019-12-07T19:34:09.612+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 -- 0x558c576dee00 con 0x558c54c64000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f1199c700 20 osd.7 32 OSD::ms_dispatch: osd_map(30..32 src has 1..32) v4 2019-12-07T19:34:09.612+0000 7f8f1199c700 10 osd.7 32 do_waiters -- start 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f1199c700 10 osd.7 32 do_waiters -- finish 2019-12-07T19:34:09.612+0000 7f8f1199c700 20 osd.7 32 _dispatch 0x558c5738e000 osd_map(30..32 src has 1..32) v4 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 -- 0x558c576de700 con 0x558c54c19800 2019-12-07T19:34:09.612+0000 7f8f1199c700 3 osd.7 32 handle_osd_map epochs [30,32], i have 32, src has [1,32] 2019-12-07T19:34:09.612+0000 7f8f1199c700 10 osd.7 32 no new maps here, dropping 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:09.612+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 -- 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 -- 0x558c54e98e00 con 0x558c54c19c00 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x959bd~8f3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x959bd 0x959bd~8f3 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2]] exit Trimming 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x659bd 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9bd 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2b0 and padding block with 0xd50 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 15 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:43 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] enter NotTrimming 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) need 0x1000~1000 cache has 0x[] 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] snap_trimmer complete 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) need {<0x0, 0x2000> : [0x45000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x45000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) need {<0x0, 0x1000> : [0x16000:0~1000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~1000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x962b0 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000136856s (0.000000086s flush + 0.000136770s kv commit) 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcc00,0x558c54d58000> 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 kv_submitted 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcc00 2019-12-07T19:34:09.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 finishing 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcc00 onodes 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcc00 done 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcc00 [] 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:34:09.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) need {<0x0, 0x2000> : [0x33000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x33000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) need {<0x0, 0x2000> : [0x35000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x35000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) need {<0x0, 0x2000> : [0x37000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x37000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) need {<0x0, 0x2000> : [0x39000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x39000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) need {<0x0, 0x2000> : [0x3b000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3b000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) need {<0x0, 0x2000> : [0x3d000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3d000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) need {<0x0, 0x2000> : [0x3f000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3f000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) need {<0x0, 0x2000> : [0x41000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x41000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) need {<0x0, 0x2000> : [0x47000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x47000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) need {<0x0, 0x2000> : [0x49000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x49000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) need {<0x0, 0x2000> : [0x4b000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4b000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) need {<0x0, 0x2000> : [0x4d000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4d000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) need {<0x0, 0x2000> : [0x4f000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4f000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) need {<0x0, 0x2000> : [0x51000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x51000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) need {<0x0, 0x2000> : [0x53000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x53000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) need {<0x0, 0x2000> : [0x55000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x55000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) need {<0x0, 0x2000> : [0x57000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x57000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) need {<0x1000, 0x1000> : [0x59000:1000~1000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x59000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) need {<0x0, 0x2000> : [0x43000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x43000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) need {<0x0, 0x2000> : [0x17000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) need {<0x0, 0x2000> : [0x19000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) need {<0x0, 0x2000> : [0x1b000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) need {<0x0, 0x2000> : [0x1d000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) need {<0x0, 0x2000> : [0x1f000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) need {<0x0, 0x2000> : [0x21000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) need {<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) need {<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x25000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) need {<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x27000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) need {<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x29000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) need {<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2b000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) need {<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2d000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) need {<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2f000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) need {<0x0, 0x2000> : [0x31000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x31000: 0x0 reading 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c575bd9e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb73b0 space 0x558c575bd9e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d2868 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb6870 space 0x558c574d2868 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57396248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54e14750 space 0x558c57396248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7a70 space 0x558c5757e8d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x45000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c573967f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7b00 space 0x558c573967f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) need 0x{<0x0, 0x1000> : [0x16000:0~1000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578dfeb8 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb79e0 space 0x558c578dfeb8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1908 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57982120 space 0x558c578f1908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5742bc18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7cb0 space 0x558c5742bc18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f19e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7d40 space 0x558c578f19e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.612+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1ac8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7dd0 space 0x558c578f1ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1ba8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7e60 space 0x558c578f1ba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c578f1c88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54cb7ef0 space 0x558c578f1c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 42 ==== MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 ==== 368805+0+0 (unknown 0 0 0) 0x558c5738d680 con 0x558c54c19c00 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54d75e48 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6000 space 0x558c54d75e48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x33000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c088 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6090 space 0x558c5792c088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x35000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c5740db80 prio 127 cost 0 latency 0.000021 epoch 31 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6120 space 0x558c5792c168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x37000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a61b0 space 0x558c5792c248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x39000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c328 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6240 space 0x558c5792c328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3b000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c408 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a62d0 space 0x558c5792c408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3d000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c4e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6360 space 0x558c5792c4e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3f000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c5c8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a63f0 space 0x558c5792c5c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x41000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792c6a8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6480 space 0x558c5792c6a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x47000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d198 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6510 space 0x558c5792d198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x49000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d278 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a65a0 space 0x558c5792d278 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4b000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6630 space 0x558c5792d358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4d000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d438 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a66c0 space 0x558c5792d438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4f000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d518 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6750 space 0x558c5792d518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x51000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d5f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a67e0 space 0x558c5792d5f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x53000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d6d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6870 space 0x558c5792d6d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x55000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d7b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6900 space 0x558c5792d7b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x57000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6990 space 0x558c5792d898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x59000:1000~1000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5792d978 in 0x558c53a59ce0) _discard 0x1000~1000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6a20 space 0x558c5792d978 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x43000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c546777b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6ab0 space 0x558c546777b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x17000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d438 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6b40 space 0x558c5794d438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x19000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d518 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6bd0 space 0x558c5794d518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1b000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d5f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6c60 space 0x558c5794d5f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1d000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d6d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6cf0 space 0x558c5794d6d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1f000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d7b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6d80 space 0x558c5794d7b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x21000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6e10 space 0x558c5794d898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d978 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6ea0 space 0x558c5794d978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794da58 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6f30 space 0x558c5794da58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794db38 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a6fc0 space 0x558c5794db38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794dc18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a7050 space 0x558c5794dc18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794dcf8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a70e0 space 0x558c5794dcf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794ddd8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a7170 space 0x558c5794ddd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794deb8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a7200 space 0x558c5794deb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x31000:0~2000]} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794df98 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c579a7290 space 0x558c5794df98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~5a000 = 368640 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read read request=368640 r=368640 len=368640 2019-12-07T19:34:09.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 43 ==== MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 ==== 368805+0+0 (unknown 0 0 0) 0x558c5738d400 con 0x558c54c19c00 2019-12-07T19:34:09.613+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000012 epoch 31 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:09.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read: Checking hash of 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 -- 0x558c54e13b80 con 0x558c53ae5400 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.002151 MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:09.613+0000 7f8f0297e700 25 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:09.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~5a000 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~5a000 size 0x5a000 (368640) 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 91 ==== MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 ==== 368805+0+0 (unknown 0 0 0) 0x558c5738c780 con 0x558c54c5b400 2019-12-07T19:34:09.613+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000010 epoch 32 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e32) 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 136 ==== MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 ==== 368805+0+0 (crc 0 0 0) 0x558c5738ca00 con 0x558c54c64000 2019-12-07T19:34:09.613+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c5740d080 prio 127 cost 0 latency 0.000026 epoch 31 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) need 0x1000~1000 cache has 0x[1000~1000] 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~5a000 = 368640 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read read request=368640 r=368640 len=368640 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:09.613+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c54e13b80 con 0x558c53ae5400 2019-12-07T19:34:09.613+0000 7f8f1119b700 15 osd.7 32 enqueue_op 0x558c578d2b00 prio 127 cost 0 latency 0.000347 epoch 32 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.613+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e32) 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:09.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read: Checking hash of 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 -- 0x558c5738c500 con 0x558c53ae5400 2019-12-07T19:34:09.613+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:09.613+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c5738c500 con 0x558c53ae5400 2019-12-07T19:34:09.613+0000 7f8f1119b700 15 osd.7 32 enqueue_op 0x558c578d3600 prio 127 cost 0 latency 0.000010 epoch 32 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.613+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e32) 2019-12-07T19:34:09.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:09.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.613+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c54e79080 prio 127 cost 0 latency 0.002205 MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.613+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpRead(1.cs0 32/28 ECSubRead(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,368640,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:09.613+0000 7f8f06986700 25 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:09.613+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~5a000 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~5a000 size 0x5a000 (368640) 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 44 ==== MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 ==== 368805+0+0 (unknown 0 0 0) 0x558c5738d180 con 0x558c54c19c00 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c54e79600 prio 127 cost 0 latency 0.000014 epoch 32 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:09.613+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) need 0x1000~1000 cache has 0x[1000~1000] 2019-12-07T19:34:09.613+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~5a000 = 368640 2019-12-07T19:34:09.613+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read read request=368640 r=368640 len=368640 2019-12-07T19:34:09.613+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.613+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:09.613+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:09.613+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:09.613+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:09.613+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read: Checking hash of 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:09.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 -- 0x558c5738db80 con 0x558c53ae5400 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c54e79080 finish 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e31) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c5740db80 prio 127 cost 0 latency 0.001012 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=10, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [5(1),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2),7(0)) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c5740db80 finish 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c5738db80 con 0x558c53ae5400 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d2b00 prio 127 cost 0 latency 0.000726 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f1119b700 15 osd.7 32 enqueue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000091 epoch 32 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=10, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2)) 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d2b00 finish 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e31) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c5740d080 prio 127 cost 0 latency 0.000554 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=10, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [1(3),368640, 5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=4(2)) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:09.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 92 ==== MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 ==== 368805+0+0 (unknown 0 0 0) 0x558c5738cc80 con 0x558c54c5b400 2019-12-07T19:34:09.614+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000014 epoch 32 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e32) 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000685 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=10, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=10, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=3 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=2 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=1 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=0 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply Complete: ReadOp(tid=10, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [1(3),368640, 4(2),368640, 5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=) 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] objects_read_async_cb: got: {1:3a5a3f5b:::smithi13140796-23:head=0,{0~1474560(1474560)}} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:09.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 93 ==== MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 ==== 368805+0+0 (unknown 0 0 0) 0x558c57454500 con 0x558c54c5b400 2019-12-07T19:34:09.614+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000015 epoch 32 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] finish_checksum 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx 0x558c574e4000 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b,getxattrs] ov 29'18 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b,getxattrs] 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op read 0~1474560 out=1474560b 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op getxattrs 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx alloc reply 0x558c54e13b80 result 0 2019-12-07T19:34:09.614+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:09.614+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] log_op_stats osd_op(client.4540.0:266 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b,getxattrs out=75b] snapc 0=[] ondisk+read+known_if_redirected e32) v8 inb 0 outb 1474643 lat 0.006182 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:09.614+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:94 2019-12-07T19:34:09.614+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(266 smithi13140796-23 [read 0~1474560 out=1474560b,checksum out=8b,getxattrs out=75b] v0'0 uv16 ondisk = 0) v8 -- 0x558c54e13b80 con 0x558c54c9f000 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c5740d4a0 finish 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e31) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.001545 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=11, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [5(1),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2),7(0)) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c5740d1e0 finish 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d3600 prio 127 cost 0 latency 0.001117 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=11, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2)) 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000745 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=11, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [4(2),368640, 5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3)) 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c54e79600 prio 127 cost 0 latency 0.001032 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=12, attrs_read=0) 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [5(1),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2),7(0)) 2019-12-07T19:34:09.614+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) queued 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000975 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.614+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=12, attrs_read=0) 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2)) 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c578d2dc0 finish 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e32) pg 0x558c57676000 2019-12-07T19:34:09.615+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000510 MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 32/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=12, attrs_read=0) 2019-12-07T19:34:09.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply readop not complete: ReadOp(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [4(2),368640, 5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3)) 2019-12-07T19:34:09.615+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:34:09.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 137 ==== MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 ==== 368805+0+0 (crc 0 0 0) 0x558c54e13b80 con 0x558c54c64000 2019-12-07T19:34:09.615+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000018 epoch 31 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.615+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2) prio 127 cost 0 e31) pg 0x558c57676000 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000050 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=11, attrs_read=0)) v2 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=11, attrs_read=0) 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=3 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=2 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=1 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=0 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply Complete: ReadOp(tid=11, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [1(3),368640, 4(2),368640, 5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=) 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] objects_read_async_cb: got: {1:3a5a3f5b:::smithi13140796-23:head=0,{0~1474560(1474560)}} 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] finish_checksum 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx 0x558c548a2a00 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b] ov 29'18 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b] 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op read 0~1474560 out=1474560b 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx alloc reply 0x558c5738db80 result 0 2019-12-07T19:34:09.615+0000 7f8f06986700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:09.615+0000 7f8f06986700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] log_op_stats osd_op(client.4540.0:267 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e32) v8 inb 0 outb 1474568 lat 0.007179 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:09.615+0000 7f8f06986700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:95 2019-12-07T19:34:09.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(267 smithi13140796-23 [read 0~1474560 out=1474560b,checksum out=8b] v0'0 uv16 ondisk = 0) v8 -- 0x558c5738db80 con 0x558c54c9f000 2019-12-07T19:34:09.615+0000 7f8f06986700 10 osd.7 32 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:09.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 138 ==== MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=12, attrs_read=0)) v2 ==== 368805+0+0 (crc 0 0 0) 0x558c5738db80 con 0x558c54c64000 2019-12-07T19:34:09.616+0000 7f8f24457700 15 osd.7 32 enqueue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000015 epoch 31 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=12, attrs_read=0)) v2) prio 127 cost 0 e31) pg 0x558c57676000 2019-12-07T19:34:09.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 139 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e07800 con 0x558c54c64000 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000036 MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=12, attrs_read=0)) v2 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] 2019-12-07T19:34:09.616+0000 7f8f24457700 15 osd.7 32 enqueue_peering_evt 1.9s4 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] _handle_message: MOSDECSubOpReadReply(1.cs0 31/28 ECSubReadReply(tid=12, attrs_read=0)) v2 2019-12-07T19:34:09.616+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply: reply ECSubReadReply(tid=12, attrs_read=0) 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=3 2019-12-07T19:34:09.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 140 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e07800 con 0x558c54c64000 2019-12-07T19:34:09.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=2 2019-12-07T19:34:09.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=1 2019-12-07T19:34:09.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) pg 0x558c5500a000 2019-12-07T19:34:09.616+0000 7f8f24457700 15 osd.7 32 enqueue_peering_evt 1.7s4 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:09.616+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply have shard=0 2019-12-07T19:34:09.616+0000 7f8f08189700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.616+0000 7f8f08189700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_sub_read_reply Complete: ReadOp(tid=12, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,1474560,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1474560, [1(3),368640, 4(2),368640, 5(1),368640, 7(0),368640]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,7(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=) 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) pg 0x558c54d5f000 2019-12-07T19:34:09.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:09.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:09.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea0f00 seq 33 2019-12-07T19:34:09.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:09.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:09.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:09.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:09.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:09.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea0c00 seq 36 2019-12-07T19:34:09.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:09.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:09.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:09.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes shared_blobs 2019-12-07T19:34:09.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:09.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x962b0~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x962b0 0x962b0~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x662b0 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2b0 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x735 and padding block with 0x8cb 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] objects_read_async_cb: got: {1:3a5a3f5b:::smithi13140796-23:head=0,{0~1474560(1474560)}} 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x96735 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118806s (0.000000153s flush + 0.000118653s kv commit) 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x96735~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x96735 0x96735~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x66735 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x735 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbba and padding block with 0x446 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x96bba 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000099219s (0.000000061s flush + 0.000099158s kv commit) 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:09.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] finish_checksum 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx 0x558c57450000 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b] ov 29'18 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b] 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op read 0~1474560 out=1474560b 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] execute_ctx alloc reply 0x558c57454500 result 0 2019-12-07T19:34:09.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:09.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] log_op_stats osd_op(client.4540.0:268 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1474560 out=1474560b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e32) v8 inb 0 outb 1474568 lat 0.008024 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:09.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:96 2019-12-07T19:34:09.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(268 smithi13140796-23 [read 0~1474560 out=1474560b,checksum out=8b] v0'0 uv16 ondisk = 0) v8 -- 0x558c57454500 con 0x558c54c9f000 2019-12-07T19:34:09.616+0000 7f8f0297e700 10 osd.7 32 dequeue_op 0x558c578d2dc0 finish 2019-12-07T19:34:09.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 141 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e07800 con 0x558c54c64000 2019-12-07T19:34:09.617+0000 7f8f24457700 15 osd.7 32 enqueue_peering_evt 1.0s3 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:09.617+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e32) pg 0x558c5500d000 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c57ef8000 seq 23 2019-12-07T19:34:09.617+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8000 onodes shared_blobs 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8000 allocated 0x[] released 0x[] 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 prepare 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8000 2019-12-07T19:34:09.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 io_done 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x96bba~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x96bba 0x96bba~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x66bba 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbba 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3f and padding block with 0xfc1 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x9703f 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000139295s (0.000000083s flush + 0.000139212s kv commit) 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8000> 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 kv_submitted 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8000 2019-12-07T19:34:09.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 finishing 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8000 onodes 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8000 done 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:09.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8000 [] 2019-12-07T19:34:09.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.637+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:34:09.637+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:09.637+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 21 0x2000~1000 2019-12-07T19:34:09.637+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 5f5a0ec9 2019-12-07T19:34:09.637+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 21 0x1b000~1000 2019-12-07T19:34:09.637+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 8d83060d 2019-12-07T19:34:09.637+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:09.637+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:09.687+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.737+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.760+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.210309283s delta_ub 0.014037390s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58600 con 0x558c54c65400 2019-12-07T19:34:09.760+0000 7f8f24c58700 20 osd.7 32 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013421752s,0.014037390s]) 2019-12-07T19:34:09.760+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e32 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.196887531s delta_ub 18446744073.696129864s) v5 -- 0x558c55270e00 con 0x558c54c65400 2019-12-07T19:34:09.760+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.210309283s delta_ub 0.014037390s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58600 con 0x558c54c65000 2019-12-07T19:34:09.760+0000 7f8f24c58700 20 osd.7 32 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013389631s,0.014037390s]) 2019-12-07T19:34:09.760+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e32 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.196919652s delta_ub 18446744073.696161985s) v5 -- 0x558c5768be00 con 0x558c54c65000 2019-12-07T19:34:09.762+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 39 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57452300 con 0x558c54c64400 2019-12-07T19:34:09.762+0000 7f8f23c56700 15 osd.7 32 enqueue_peering_evt 1.2s4 epoch_sent: 32 epoch_requested: 32 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:09.762+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e32) pg 0x558c5731a000 2019-12-07T19:34:09.762+0000 7f8f07988700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:09.762+0000 7f8f07988700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:09.762+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54ddf800 seq 22 2019-12-07T19:34:09.762+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:09.762+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:09.762+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:09.762+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddf800 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddf800 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddf800 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 prepare 2019-12-07T19:34:09.762+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddf800 2019-12-07T19:34:09.762+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 io_done 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:09.762+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x9703f~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x9703f 0x9703f~485 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x6703f 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3f 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4c4 and padding block with 0xb3c 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x974c4 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:09.762+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122945s (0.000000083s flush + 0.000122862s kv commit) 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:09.762+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddf800> 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 kv_submitted 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddf800 2019-12-07T19:34:09.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 finishing 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddf800 onodes 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddf800 done 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:09.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddf800 [] 2019-12-07T19:34:09.762+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:09.762+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:09.762+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.787+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.838+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.887+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.921+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:09.938+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.988+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.038+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.088+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.138+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.188+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.238+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.243+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 6 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.710412538s delta_ub 0.031109064s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c5ac00 2019-12-07T19:34:10.243+0000 7f8f23c56700 20 osd.7 32 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030707094s,0.031109064s]) 2019-12-07T19:34:10.243+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.679705444s delta_ub 18446744073.678844522s) v5 -- 0x558c57518e00 con 0x558c54c5ac00 2019-12-07T19:34:10.243+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 6 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.710412538s delta_ub 0.031109064s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c5a400 2019-12-07T19:34:10.243+0000 7f8f24c58700 20 osd.7 32 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030592030s,0.031109064s]) 2019-12-07T19:34:10.243+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.679820508s delta_ub 18446744073.678959586s) v5 -- 0x558c5768be00 con 0x558c54c5a400 2019-12-07T19:34:10.268+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9ec00 0x558c552b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:10.268+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9ec00 0x558c552b5700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:10.269+0000 7f8f24457700 10 osd.7 32 ms_handle_authentication new session 0x558c57405ba0 con 0x558c54c9ec00 entity client.admin addr 2019-12-07T19:34:10.269+0000 7f8f24457700 10 osd.7 32 ms_handle_authentication session 0x558c57405ba0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:10.269+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1746078500 conn(0x558c54c9ec00 0x558c552b5700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4406 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:10.269+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4406 172.21.15.131:0/1746078500 1 ==== command(tid 14: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c575cc000 con 0x558c54c9ec00 2019-12-07T19:34:10.269+0000 7f8f164bb700 20 osd.7 32 OSD::ms_dispatch: command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:10.269+0000 7f8f164bb700 10 osd.7 32 do_waiters -- start 2019-12-07T19:34:10.269+0000 7f8f164bb700 10 osd.7 32 do_waiters -- finish 2019-12-07T19:34:10.269+0000 7f8f164bb700 20 osd.7 32 _dispatch 0x558c575cc000 command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:10.269+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1746078500 -- command_reply(tid 14: 0 ) v1 -- 0x558c54ce1500 con 0x558c54c9ec00 2019-12-07T19:34:10.289+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.292+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4406 172.21.15.131:0/1746078500 2 ==== command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c575cda40 con 0x558c54c9ec00 2019-12-07T19:34:10.292+0000 7f8f164bb700 20 osd.7 32 OSD::ms_dispatch: command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:10.292+0000 7f8f164bb700 10 osd.7 32 do_waiters -- start 2019-12-07T19:34:10.292+0000 7f8f164bb700 10 osd.7 32 do_waiters -- finish 2019-12-07T19:34:10.292+0000 7f8f164bb700 20 osd.7 32 _dispatch 0x558c575cda40 command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:10.293+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1746078500 -- command_reply(tid 15: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce0fc0 con 0x558c54c9ec00 2019-12-07T19:34:10.296+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1746078500 conn(0x558c54c9ec00 msgr2=0x558c552b5700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 64 2019-12-07T19:34:10.296+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1746078500 conn(0x558c54c9ec00 msgr2=0x558c552b5700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:10.296+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1746078500 conn(0x558c54c9ec00 0x558c552b5700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:10.296+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1746078500 conn(0x558c54c9ec00 0x558c552b5700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:10.296+0000 7f8f164bb700 2 osd.7 32 ms_handle_reset con 0x558c54c9ec00 session 0x558c57405ba0 2019-12-07T19:34:10.323+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:10.339+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.344+0000 7f8f21451700 10 osd.7 32 tick 2019-12-07T19:34:10.344+0000 7f8f21451700 10 osd.7 32 do_waiters -- start 2019-12-07T19:34:10.344+0000 7f8f21451700 10 osd.7 32 do_waiters -- finish 2019-12-07T19:34:10.344+0000 7f8f21451700 20 osd.7 32 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:10.388+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.439+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.489+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.521+0000 7f8f1219d700 20 osd.7 32 reports for 0 queries 2019-12-07T19:34:10.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c579b5500 con 0x558c5476d800 2019-12-07T19:34:10.521+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:10.521+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x408000/0x408000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:10.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(2 pgs tid 0 v 0) v2 -- 0x558c579b5880 con 0x558c5476d800 2019-12-07T19:34:10.539+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.589+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 437163 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 142 ==== osd_map(33..33 src has 1..33) v4 ==== 633+0+0 (crc 0 0 0) 0x558c54dd1b00 con 0x558c54c64000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 osd.7 32 OSD::ms_dispatch: osd_map(33..33 src has 1..33) v4 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 osd.7 32 do_waiters -- start 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 osd.7 32 do_waiters -- finish 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 osd.7 32 _dispatch 0x558c54dd1b00 osd_map(33..33 src has 1..33) v4 2019-12-07T19:34:10.618+0000 7f8f1199c700 3 osd.7 32 handle_osd_map epochs [33,33], i have 32, src has [1,33] 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 osd.7 32 handle_osd_map got inc map for epoch 33 2019-12-07T19:34:10.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3978e826:::osdmap.32:0# 0x0~0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac700 blob([0x52000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57881110 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd730 blob([!~1000,0x53000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c5757f180 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3978e826:::osdmap.32:0# 0x0~1454 = 5204 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 osd.7 32 add_map_bl 32 5204 bytes 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 osd.7 32 got_full_map 33, nothing requested 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 osd.7 32 handle_osd_map pg_num_history pg_num_history(e33 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {33={}} 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 osd.7 32 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e33 [1,33] lci=[13,33]) 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54dde000 seq 24 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:ca8b972f:::inc_osdmap.33:0# key 0x7f7fffffffffffffffca8b97'/!inc_osdmap.33!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ca8b972f:::inc_osdmap.33:0# 0x0~24d 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1147 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:ca8b972f:::inc_osdmap.33:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde000 1 blobs 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x54000~1000/1000,1000,0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x54000~1000] 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac0e0 blob([0x54000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac690 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c576ac0e0) get_ref 0x0~24d Blob(0x558c576ac0e0 blob([0x54000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac690 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c576ac0e0) get_ref init 0x1000, 1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c576ac0e0 blob([0x54000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c576ac690 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c576ac6a8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34108) dirty_range mark inline shard dirty 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ca8b972f:::inc_osdmap.33:0# 0x0~24d = 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3478e826:::osdmap.33:0# key 0x7f7fffffffffffffff3478e8'&!osdmap.33!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3478e826:::osdmap.33:0# 0x0~1438 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1148 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3478e826:::osdmap.33:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~438 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde000 2 blobs 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x55000~1000/1000,2000,0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2e2000~1000/1000,2000,0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x55000~1000,0x2e2000~1000] 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcba40 blob([0x55000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb730 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c57dcba40) get_ref 0x0~1000 Blob(0x558c57dcba40 blob([0x55000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb730 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c57dcba40) get_ref init 0x1000, 1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57dcba40 blob([0x55000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57dcb730 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57dcb748 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396bd0 blob([!~1000,0x2e2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa5b0 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c57396bd0) get_ref 0x1000~438 Blob(0x558c57396bd0 blob([!~1000,0x2e2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c574fa5b0 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c57396bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558c57396bd0 blob([!~1000,0x2e2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c574fa5b0 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c574fa5c8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1438 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34388) dirty_range mark inline shard dirty 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3478e826:::osdmap.33:0# 0x0~1438 = 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde000 0 blobs 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde000 0 blobs 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde000 cost 23246 (4 ios * 4000 + 7246 bytes) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde000 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34000,0x558c57a34280 shared_blobs 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34108) update #-1:ca8b972f:::inc_osdmap.33:0# 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34108) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:ca8b972f:::inc_osdmap.33:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34388) update #-1:3478e826:::osdmap.33:0# 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34388) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3478e826:::osdmap.33:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde000 allocated 0x[54000~2000,2e2000~1000] released 0x[] 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 freelist allocate 0x54000~2000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 freelist allocate 0x2e2000~1000 2019-12-07T19:34:10.618+0000 7f8f1199c700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 prepare 2019-12-07T19:34:10.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54dde000 2019-12-07T19:34:10.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 aio_wait 2019-12-07T19:34:10.618+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde000 2019-12-07T19:34:10.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 io_done 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 1 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:10.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2876 < min_flush_size 524288 2019-12-07T19:34:10.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8829 < min_flush_size 524288 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a34000 had 1 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a34280 had 1 2019-12-07T19:34:10.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860201472 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998627) bluestore 86 GiB free (0.959952), bluefs_ratio 0.0399496 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8900 < min_flush_size 524288 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8900 < min_flush_size 524288 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x974c4~22c4 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x974c4 0x974c4~22c4 to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~70000 x_off 0x674c4 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4c4 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x788 and padding block with 0x878 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x99788 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:10.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000163740s (0.000004968s flush + 0.000158772s kv commit) 2019-12-07T19:34:10.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde000> 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c54de8780> 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 kv_submitted 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde000 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54dde000 osr 0x558c54ea4b40 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) prepare_write seq 22 0x1b000~1000 crc 975caaf0 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) prepare_write seq 22 0x2000~1000 crc e4611af4 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 deferred_cleanup 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6000 onodes 0x558c53a91900,0x558c55276c80,0x558c574a1900,0x558c5756fb80 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e09b8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb440 space 0x558c574e09b8 0x1000~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1c18 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb3b0 space 0x558c574f1c18 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac7f8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb320 space 0x558c576ac7f8 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 done 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde300 deferred_cleanup 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6000 [] 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 deferred_cleanup 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde300 onodes 0x558c53a91900,0x558c55276c80,0x558c5756e780,0x558c5756ea00 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567518 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba750 space 0x558c57567518 0x1000~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567d68 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba6c0 space 0x558c57567d68 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576aca28 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba630 space 0x558c576aca28 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde300 done 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde600 deferred_cleanup 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde300 [] 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 deferred_cleanup 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde600 onodes 0x558c53a91900,0x558c55276c80,0x558c5756ec80,0x558c5756ef00 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552657b8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba990 space 0x558c552657b8 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396a98 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbaa20 space 0x558c57396a98 0x1000~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bc1d8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba900 space 0x558c575bc1d8 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7 32 _committed_osd_maps 33..33 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde600 done 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7 32 advance to epoch 33 (<= last 33 <= newest_map 33) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc000 deferred_cleanup 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde600 [] 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 deferred_cleanup 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc000 onodes 0x558c53a91900,0x558c54e44280,0x558c54e45400,0x558c55276c80 2019-12-07T19:34:10.619+0000 7f8f141a1700 7 osd.7 33 consume_map version 33 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa6a8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb5f0 space 0x558c574fa6a8 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f198 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb710 space 0x558c5757f198 0x1000~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.0s3 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881128 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb680 space 0x558c57881128 0x0~1000 writing nocache) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.9s4 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc000 done 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde000 deferred_queued 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.2s4 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc000 [] 2019-12-07T19:34:10.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.bs3 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.3s4 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.cs0 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.4s0 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.ds2 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.6s2 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 identify_splits_and_merges 1.7s4 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 33 prune_pg_created 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:0.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:1.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:2.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:3.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:4.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:5.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:6.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7:7.consume_map 33 (was 32) 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:10.619+0000 7f8f141a1700 10 osd.7 33 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.0s3 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.9s4 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.2s4 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.bs3 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.3s4 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c54d9b000 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c5500d000 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 32 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 32 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:10.619+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c5731a000 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 32 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.cs0 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.4s0 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c5513e000 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.ds2 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.6s2 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f141a1700 15 osd.7 33 enqueue_peering_evt 1.7s4 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f141a1700 7 osd.7 33 activate_map version 33 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 32 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:10.619+0000 7f8f03980700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f0918b700 20 osd.7 33 agent_entry empty queue 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c54d5f000 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c57676000 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c54d5c000 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 32 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 32 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] state: Active advmap 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:10.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c5500a000 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c55141000 2019-12-07T19:34:10.619+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] state: Active: handling ActMap 2019-12-07T19:34:10.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] Active: kicking snap trim 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 32 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:10.619+0000 7f8f06986700 7 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] state: handle ActMap primary 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 32 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f06986700 15 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:97 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:10.619+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558c548ab000 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 32 -> 33 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 32 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:10.619+0000 7f8f06185700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] state: Active advmap 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] state: Active: handling ActMap 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] Active: kicking snap trim 2019-12-07T19:34:10.619+0000 7f8f06986700 7 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] state: handle ActMap primary 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:10.619+0000 7f8f06986700 15 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:44 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06986700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:10.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:10.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 143 ==== MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 ==== 372651+0+0 (crc 0 0 0) 0x558c5748d800 con 0x558c54c64000 2019-12-07T19:34:10.620+0000 7f8f24457700 15 osd.7 33 enqueue_op 0x558c578d2160 prio 127 cost 0 latency 0.000036 epoch 33 MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:10.620+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e33) 2019-12-07T19:34:10.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e33) queued 2019-12-07T19:34:10.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e33) pg 0x558c5500a000 2019-12-07T19:34:10.620+0000 7f8f08189700 10 osd.7 33 dequeue_op 0x558c578d2160 prio 127 cost 0 latency 0.000088 MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 pg pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:10.620+0000 7f8f08189700 10 osd.7 33 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 32 -> 33 (as per caller) 2019-12-07T19:34:10.620+0000 7f8f08189700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:10.620+0000 7f8f08189700 20 snap_mapper.add_oid 1:9b8adabe:::smithi13140796-6:a a 2019-12-07T19:34:10.620+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:10.620+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:10.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:10.620+0000 7f8f08189700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:a got.empty() 2019-12-07T19:34:10.620+0000 7f8f08189700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:a a 2019-12-07T19:34:10.620+0000 7f8f08189700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.620+0000 7f8f08189700 20 snap_mapper.add_oid set SNA_1_000000000000000A_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.620+0000 7f8f08189700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] append_log log((0'0,29'13], crt=29'13) [33'14 (21'5) clone 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [a] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,33'15 (21'5) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:270 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0] 2019-12-07T19:34:10.620+0000 7f8f08189700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'14 (0'0,33'14] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] add_log_entry 33'14 (21'5) clone 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [a] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.621+0000 7f8f08189700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] add_log_entry 33'15 (21'5) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:270 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.621+0000 7f8f08189700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 15 2019-12-07T19:34:10.621+0000 7f8f08189700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:10.621+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:10.621+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 33'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c574ffb00 seq 34 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:a# key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0x000000000000000affffffffffffffff'o' 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:a# 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1149 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.onode(0x558c54d97900).flush flush done 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# 0x0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:a# 0x0~6000 -> 0x0~6000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10326 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54d3d960 blob([0x48000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54d3d960 blob([0x48000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=1)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup new Blob(0x558c578de150 blob([0x48000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de150) get_ref 0x0~2000 Blob(0x558c578de150 blob([0x48000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de150) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c578de150 blob([0x48000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10327 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=1)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup new Blob(0x558c578de070 blob([0x4a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de070) get_ref 0x0~2000 Blob(0x558c578de070 blob([0x4a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de070) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c578de070 blob([0x4a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10328 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54d3db20 blob([0x4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54d3db20 blob([0x4c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=1)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup new Blob(0x558c578de850 blob([0x4c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de850) get_ref 0x0~2000 Blob(0x558c578de850 blob([0x4c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de850) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c578de850 blob([0x4c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=2)))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) dirty_range mark inline shard dirty 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c57977008) dirty_range mark inline shard dirty 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:a# = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# snapset 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# snapset = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# 1 keys 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# 1 keys = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# hinfo_key (42 bytes) 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# hinfo_key (42 bytes) = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:head#f key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0xfffffffffffffffe000000000000000f'o' 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:head#f 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:head#f = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1150 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 23 keys 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 23 keys = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x0~4d000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9b8adabe:::smithi13140796-6:head# 0x0~4d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~4d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574ffb00 39 blobs 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.621+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate 0x4d000/1000,4d000,0 2019-12-07T19:34:10.621+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2e3000~26000/1000,4d000,0 2019-12-07T19:34:10.621+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x489000~27000/1000,4d000,0 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2e3000~26000,0x489000~27000] 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcb7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57dcb7a0) get_ref 0x0~2000 Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57dcb7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcb6d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcaaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57dcaaf0) get_ref 0x0~2000 Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57dcaaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcac58 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c577f3ea0) get_ref 0x0~2000 Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c577f3ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bea088 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e42540) get_ref 0x0~2000 Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e42540) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57881198 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578813b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578813b0) get_ref 0x0~2000 Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578813b0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57881438 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57881490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57881490) get_ref 0x0~2000 Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57881490) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57881358 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57880bd0) get_ref 0x0~2000 Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c57880bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57880f68 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de310) get_ref 0x0~2000 Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de310) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578de408 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de770) get_ref 0x0~2000 Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de770) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578de6a8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de5b0) get_ref 0x0~2000 Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578de5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578de4e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578debd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578debd0) get_ref 0x0~2000 Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578debd0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578deb08 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578dea10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578dea10) get_ref 0x0~2000 Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578dea10) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578de948 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578decb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578decb0) get_ref 0x0~2000 Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578decb0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578deef8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578def50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578def50) get_ref 0x0~2000 Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578def50) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578defd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df030) get_ref 0x0~2000 Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df030) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578df0b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df110) get_ref 0x0~2000 Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df110) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578df198 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df1f0) get_ref 0x0~2000 Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c018 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c070) get_ref 0x0~2000 Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c070) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c0f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c150) get_ref 0x0~2000 Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c150) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c948 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c8c0) get_ref 0x0~2000 Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c868 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c7e0) get_ref 0x0~2000 Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578f17b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578f1880) get_ref 0x0~2000 Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578f1880) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578f1748 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e0d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c574e0d90) get_ref 0x0~2000 Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c574e0d90) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c574e1cf8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e0930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c574e0930) get_ref 0x0~2000 Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c574e0930) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c574e0e18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df260) get_ref 0x0~2000 Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578df260) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578ded38 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578ded90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578ded90) get_ref 0x0~2000 Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578ded90) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578dee18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578dee70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578dee70) get_ref 0x0~2000 Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c578dee70) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c788 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c700) get_ref 0x0~2000 Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c700) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c6a8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c620) get_ref 0x0~2000 Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c620) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c5c8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c540) get_ref 0x0~2000 Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c540) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1c4e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c9a0) get_ref 0x0~2000 Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1c9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1ca28 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1ca80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1ca80) get_ref 0x0~2000 Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1ca80) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cb08 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1cb60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cb60) get_ref 0x0~2000 Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cb60) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cbe8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1cc40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cc40) get_ref 0x0~2000 Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cc40) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1ccc8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1cd20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cd20) get_ref 0x0~2000 Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cd20) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cda8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1ce00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1ce00) get_ref 0x0~2000 Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1ce00) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1ce88 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1cee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cee0) get_ref 0x0~2000 Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cee0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cf68 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1cfc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cfc0) get_ref 0x0~2000 Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1cfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1d048 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1d0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1d0a0) get_ref 0x0~1000 Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.blob(0x558c54e1d0a0) get_ref init 0x1000, 1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~1000: 0x0~1000 Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 sbid 0x0)) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1d128 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4d000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dirty_range mark inline shard dirty 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x0~4d000 = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000A_.4_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- SNA_1_000000000000000A_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000033.00000000000000000014' <- 0000000033.00000000000000000014 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000033.00000000000000000015' <- 0000000033.00000000000000000015 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:10.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ffb00 cost 533693 (40 ios * 4000 + 373693 bytes) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ffb00 onodes 0x558c54d97900,0x558c57976f00,0x558c57a35400 shared_blobs 0x558c54d3dab0,0x558c54d3dc00,0x558c55279730 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) update 4#1:9b8adabe:::smithi13140796-6:head#f 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c54d97a08) update inline shard 84 bytes from 3 extents 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head#f is 573 (483 bytes onode + 2 bytes spanning blobs + 88 bytes inline extents) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c57977008) update 4#1:9b8adabe:::smithi13140796-6:a# 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c57977008) update inline shard 84 bytes from 3 extents 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:a# is 500 (410 bytes onode + 2 bytes spanning blobs + 88 bytes inline extents) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update inline shard 741 bytes from 39 extents 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 55192 (54445 bytes onode + 2 bytes spanning blobs + 745 bytes inline extents) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2857 is 11 SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=2))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2858 is 11 SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=2))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2856 is 11 SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=2))) 2019-12-07T19:34:10.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ffb00 allocated 0x[2e3000~26000,489000~27000] released 0x[] 2019-12-07T19:34:10.621+0000 7f8f08189700 10 freelist allocate 0x2e3000~26000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 freelist _xor first_key 0x280000 last_key 0x300000 2019-12-07T19:34:10.621+0000 7f8f08189700 10 freelist allocate 0x489000~27000 2019-12-07T19:34:10.621+0000 7f8f08189700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:10.622+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 prepare 2019-12-07T19:34:10.622+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c574ffb00 2019-12-07T19:34:10.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.622+0000 7f8f08189700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] op_applied version 33'15 2019-12-07T19:34:10.622+0000 7f8f08189700 10 osd.7 33 dequeue_op 0x558c578d2160 finish 2019-12-07T19:34:10.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.622+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 aio_wait 2019-12-07T19:34:10.622+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ffb00 2019-12-07T19:34:10.622+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 io_done 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 26744 < min_flush_size 524288 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 58888 < min_flush_size 524288 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97900 had 1 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976f00 had 1 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 58911 < min_flush_size 524288 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 58911 < min_flush_size 524288 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x99788~e61f to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x99788 0x99788~e61f to file(ino 16 size 0xa0000 mtime 2019-12-07T19:34:08.613155+0000 allocated a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~70000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _allocate len 0x7da7 from 0 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _allocate len 0x7da7 from 1 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd1d0000 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd1d0000~10000/10000,10000,acd1d0000 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0xb0000 to include allocated 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 32 (was clean) 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x69788 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x788 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xda7 and padding block with 0x259 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0xb0000 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xa7da7 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (32) on file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]), flushing log 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 32 len 0x49 crc 0xd4070a62) 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x1f000~1000 to file(ino 1 size 0x1f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x1f000 0x1f000~1000 to file(ino 1 size 0x1f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x20000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1f000 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x20000 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:10.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000018 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 32 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000533148s (0.000036813s flush + 0.000496335s kv commit) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:10.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ffb00> 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:10.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 kv_submitted 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ffb00 2019-12-07T19:34:10.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 finishing 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ffb00 onodes 0x558c54d97900,0x558c57976f00,0x558c57a35400 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c018 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58360 space 0x558c54e1c018 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c0f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a583f0 space 0x558c54e1c0f8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c4e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58ab0 space 0x558c54e1c4e8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c5c8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58a20 space 0x558c54e1c5c8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c6a8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58990 space 0x558c54e1c6a8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c788 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58900 space 0x558c54e1c788 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c868 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58510 space 0x558c54e1c868 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c948 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58480 space 0x558c54e1c948 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1ca28 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58b40 space 0x558c54e1ca28 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1cb08 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58bd0 space 0x558c54e1cb08 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1cbe8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58c60 space 0x558c54e1cbe8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 33/28 ECSubWriteReply(tid=154, last_complete=33'15, committed=1, applied=1)) v2 -- 0x558c54dfa400 con 0x558c54c64000 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1ccc8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58cf0 space 0x558c54e1ccc8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1cda8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58d80 space 0x558c54e1cda8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1ce88 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58e10 space 0x558c54e1ce88 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1cf68 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58ea0 space 0x558c54e1cf68 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1d048 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58f30 space 0x558c54e1d048 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1d128 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58fc0 space 0x558c54e1d128 0x0~1000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0e18 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58750 space 0x558c574e0e18 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e1cf8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a586c0 space 0x558c574e1cf8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880f68 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7d40 space 0x558c57880f68 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881198 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7b90 space 0x558c57881198 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881358 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7cb0 space 0x558c57881358 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881438 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7c20 space 0x558c57881438 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de408 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7dd0 space 0x558c578de408 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de4e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7ef0 space 0x558c578de4e8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de6a8 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7e60 space 0x558c578de6a8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de948 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58090 space 0x558c578de948 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578deb08 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58000 space 0x558c578deb08 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578ded38 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a587e0 space 0x558c578ded38 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dee18 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58870 space 0x558c578dee18 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578deef8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58120 space 0x558c578deef8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578defd8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a581b0 space 0x558c578defd8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df0b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58240 space 0x558c578df0b8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df198 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a582d0 space 0x558c578df198 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1748 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a58630 space 0x558c578f1748 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f17b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a585a0 space 0x558c578f17b8 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bea088 in 0x558c53a59a40) _finish_write discard buffer(0x558c579a7b00 space 0x558c57bea088 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcac58 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15b90 space 0x558c57dcac58 0x0~2000 writing nocache) 2019-12-07T19:34:10.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcb6d8 in 0x558c53a59a40) _finish_write discard buffer(0x558c54e15b00 space 0x558c57dcb6d8 0x0~2000 writing nocache) 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ffb00 done 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:10.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ffb00 [] 2019-12-07T19:34:10.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:10.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 144 ==== MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c574a2800 con 0x558c54c64000 2019-12-07T19:34:10.623+0000 7f8f24457700 15 osd.7 33 enqueue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000019 epoch 33 MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:10.623+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e33) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e33) queued 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e33) pg 0x558c5500a000 2019-12-07T19:34:10.623+0000 7f8f04181700 10 osd.7 33 dequeue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 pg pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:10.623+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s4 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:10.623+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] append_log log((0'0,33'15], crt=29'13) [] 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] rollforward: entry=33'14 (21'5) clone 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [a] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] rollforward: entry=33'15 (21'5) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:270 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.623+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 15 2019-12-07T19:34:10.623+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:10.623+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:10.623+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:10.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740af00 seq 35 2019-12-07T19:34:10.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head#f onode 0x558c54d97900 txc 0x558c5740af00 2019-12-07T19:34:10.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head#f 0x0 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d960) put_ref 0x0~2000 Blob(0x558c54d3d960 blob([0x48000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=2)))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d9d0) put_ref 0x0~2000 Blob(0x558c54d3d9d0 blob([0x4a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=2)))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore.blob(0x558c54d3db20) put_ref 0x0~2000 Blob(0x558c54d3db20 blob([0x4c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=2)))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c54d97a08) dirty_range mark inline shard dirty 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d3d960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=2)))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x48000~2000] 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=1))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d3d9d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=2)))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4a000~2000] 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=1))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54d3db20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=2)))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4c000~2000] 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=1))) 2019-12-07T19:34:10.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3dab0,0x558c54d3dc00,0x558c55279730 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c57a35400 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:10.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head#f = 0 2019-12-07T19:34:10.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:10.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:10.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 0x558c54d3dab0,0x558c54d3dc00,0x558c55279730 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2857 is 11 SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=1))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2858 is 11 SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=1))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2856 is 11 SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=1))) 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[] 2019-12-07T19:34:10.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:10.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:10.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:10.623+0000 7f8f04181700 10 osd.7 33 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 601 < min_flush_size 524288 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97900 had 1 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1326 < min_flush_size 524288 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1326 < min_flush_size 524288 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xa7da7~52e to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xa7da7 0xa7da7~52e to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x77da7 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xda7 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2d5 and padding block with 0xd2b 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xa82d5 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:10.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000103137s (0.000000094s flush + 0.000103043s kv commit) 2019-12-07T19:34:10.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00> 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:10.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:10.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:10.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [] 2019-12-07T19:34:10.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:10.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:10.623+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 33/28 ECSubWriteReply(tid=155, last_complete=33'15, committed=1, applied=1)) v2 -- 0x558c54dfa600 con 0x558c54c64000 2019-12-07T19:34:10.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 10 osd.7 33 tick_without_osd_lock 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fb50000/0x40000000/0x1680000000, data 0x4782be/0x4ae000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 25 osd.7 33 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274103+0000 last_rx_front 2019-12-07T19:34:09.274169+0000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 25 osd.7 33 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274127+0000 last_rx_front 2019-12-07T19:34:09.274111+0000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 25 osd.7 33 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274078+0000 last_rx_front 2019-12-07T19:34:09.274140+0000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 25 osd.7 33 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.274091+0000 last_rx_front 2019-12-07T19:34:09.274157+0000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 25 osd.7 33 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.273917+0000 last_rx_front 2019-12-07T19:34:09.273950+0000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 25 osd.7 33 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:09.273650+0000 last_rx_back 2019-12-07T19:34:09.273936+0000 last_rx_front 2019-12-07T19:34:09.273966+0000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 20 osd.7 33 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 20 osd.7 33 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 20 osd.7 33 scrub_load_below_threshold loadavg per cpu 0.0525 < max 5 = yes 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 20 osd.7 33 sched_scrub load_is_low=1 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 10 osd.7 33 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:10.637758+0000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 20 osd.7 33 sched_scrub done 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 10 osd.7 33 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 20 osd.7 33 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:10.636+0000 7f8f1fbbe700 10 osd.7 33 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:10.639+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.689+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.739+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.743+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 7 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.210730876s delta_ub 0.030982557s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c5ac00 2019-12-07T19:34:10.743+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 7 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.210730876s delta_ub 0.030982557s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58600 con 0x558c54c5a400 2019-12-07T19:34:10.743+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030666031s,0.030982557s]) 2019-12-07T19:34:10.743+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.180064845s delta_ub 18446744073.678885585s) v5 -- 0x558c57518e00 con 0x558c54c5ac00 2019-12-07T19:34:10.743+0000 7f8f23c56700 10 osd.7 33 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 32 -> 33 (as per caller) 2019-12-07T19:34:10.743+0000 7f8f24c58700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030648392s,0.030982557s]) 2019-12-07T19:34:10.743+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.180082484s delta_ub 18446744073.678903224s) v5 -- 0x558c5768be00 con 0x558c54c5a400 2019-12-07T19:34:10.753+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 12 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.211306508s delta_ub 0.022524754s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c18000 2019-12-07T19:34:10.753+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021855030s,0.022524754s]) 2019-12-07T19:34:10.753+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.189451478s delta_ub 18446744073.687696586s) v5 -- 0x558c57518e00 con 0x558c54c18000 2019-12-07T19:34:10.753+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 12 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.211306508s delta_ub 0.022524754s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54dfa600 con 0x558c552d1400 2019-12-07T19:34:10.753+0000 7f8f23c56700 10 osd.7 33 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 31 -> 33 (as per caller) 2019-12-07T19:34:10.753+0000 7f8f24457700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021838744s,0.022524754s]) 2019-12-07T19:34:10.753+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.189467764s delta_ub 18446744073.687712872s) v5 -- 0x558c54dfa400 con 0x558c552d1400 2019-12-07T19:34:10.789+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.839+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.889+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.939+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.0 172.21.15.131:0/37079 10 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 0.019695794s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c18c00 2019-12-07T19:34:10.953+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019137115s,0.019695794s]) 2019-12-07T19:34:10.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.390602411s delta_ub 18446744073.690414501s) v5 -- 0x558c57518e00 con 0x558c54c18c00 2019-12-07T19:34:10.953+0000 7f8f23c56700 10 osd.7 33 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 31 -> 33 (as per caller) 2019-12-07T19:34:10.954+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.0 172.21.15.131:0/37079 10 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 0.019695794s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4c00 con 0x558c54c18400 2019-12-07T19:34:10.954+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019057743s,0.019695794s]) 2019-12-07T19:34:10.954+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.390681783s delta_ub 18446744073.690493873s) v5 -- 0x558c57518e00 con 0x558c54c18400 2019-12-07T19:34:10.989+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.040+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.090+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.140+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.190+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.240+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.290+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.341+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.390+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.392+0000 7f8f21451700 10 osd.7 33 tick 2019-12-07T19:34:11.392+0000 7f8f21451700 10 osd.7 33 do_waiters -- start 2019-12-07T19:34:11.392+0000 7f8f21451700 10 osd.7 33 do_waiters -- finish 2019-12-07T19:34:11.392+0000 7f8f21451700 20 osd.7 33 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:11.398+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:11.440+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.460+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.910731969s delta_ub 0.013995162s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c65400 2019-12-07T19:34:11.460+0000 7f8f24c58700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013582286s,0.013995162s]) 2019-12-07T19:34:11.460+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.897149683s delta_ub 18446744073.695969330s) v5 -- 0x558c5768be00 con 0x558c54c65400 2019-12-07T19:34:11.460+0000 7f8f24c58700 10 osd.7 33 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 32 -> 33 (as per caller) 2019-12-07T19:34:11.460+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.910731969s delta_ub 0.013995162s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58600 con 0x558c54c65000 2019-12-07T19:34:11.460+0000 7f8f24c58700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013516941s,0.013995162s]) 2019-12-07T19:34:11.460+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.897215028s delta_ub 18446744073.696034675s) v5 -- 0x558c5768be00 con 0x558c54c65000 2019-12-07T19:34:11.490+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.502+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:11.540+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.572+0000 7f8f0097a700 5 osd.7 33 heartbeat osd_stat(store_statfs(0x163fb50000/0x40000000/0x1680000000, data 0x4782be/0x4ae000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5] op hist []) 2019-12-07T19:34:11.572+0000 7f8f0097a700 20 osd.7 33 check_full_status cur ratio 5.08626e-05, physical ratio 5.08626e-05, new state none 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.690493873s) v5 -- 0x558c54cea800 con 0x558c5476d400 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.690493873s) v5 -- 0x558c57569a00 con 0x558c551b6800 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.688493287s) v5 -- 0x558c57569800 con 0x558c54c9e000 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.688493287s) v5 -- 0x558c57569c00 con 0x558c54c9e400 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.673338903s) v5 -- 0x558c57569200 con 0x558c551b7800 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.673338903s) v5 -- 0x558c53a5e800 con 0x558c551b7c00 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.687712872s) v5 -- 0x558c53a5e600 con 0x558c552c2000 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.687712872s) v5 -- 0x558c53a5f200 con 0x558c552c2400 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.696034675s) v5 -- 0x558c548cc200 con 0x558c552c2800 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.696034675s) v5 -- 0x558c551afc00 con 0x558c552c2c00 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.678903224s) v5 -- 0x558c576b5200 con 0x558c54c65800 2019-12-07T19:34:11.572+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.678903224s) v5 -- 0x558c576b5a00 con 0x558c54c65c00 2019-12-07T19:34:11.572+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 6 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.040424672s delta_ub 0.031051297s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c65800 2019-12-07T19:34:11.572+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.273936+0000 -> 2019-12-07T19:34:11.574165+0000 last_rx_front 2019-12-07T19:34:09.273966+0000 2019-12-07T19:34:11.572+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030820011s,0.031051297s]) 2019-12-07T19:34:11.572+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 6 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.040442490s delta_ub 0.031069115s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c65c00 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574165+0000 last_rx_front 2019-12-07T19:34:09.273966+0000 -> 2019-12-07T19:34:11.574203+0000 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:11.573934+0000) and older pending ping(s) 2019-12-07T19:34:11.573+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030799556s,0.031069115s]) 2019-12-07T19:34:11.573+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.023359805s delta_ub 0.013986430s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55271e00 con 0x558c552c2c00 2019-12-07T19:34:11.573+0000 7f8f24457700 25 osd.7 33 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.273917+0000 last_rx_front 2019-12-07T19:34:09.273950+0000 -> 2019-12-07T19:34:11.574226+0000 2019-12-07T19:34:11.573+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 6 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.031126074s delta_ub 0.021752699s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c9e000 2019-12-07T19:34:11.573+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.023330026s delta_ub 0.013956651s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c552c2800 2019-12-07T19:34:11.573+0000 7f8f24457700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013694391s,0.013986430s]) 2019-12-07T19:34:11.573+0000 7f8f24c58700 25 osd.7 33 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.273917+0000 -> 2019-12-07T19:34:11.574240+0000 last_rx_front 2019-12-07T19:34:11.574226+0000 2019-12-07T19:34:11.573+0000 7f8f24c58700 25 osd.7 33 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:11.573934+0000) and older pending ping(s) 2019-12-07T19:34:11.573+0000 7f8f24c58700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013650058s,0.013956651s]) 2019-12-07T19:34:11.573+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.031741253s delta_ub 0.022367878s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55271e00 con 0x558c552c2000 2019-12-07T19:34:11.573+0000 7f8f24457700 25 osd.7 33 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.274091+0000 -> 2019-12-07T19:34:11.574261+0000 last_rx_front 2019-12-07T19:34:09.274157+0000 2019-12-07T19:34:11.573+0000 7f8f24457700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022040755s,0.022367878s]) 2019-12-07T19:34:11.573+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.045927259s delta_ub 0.036553884s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c551b7c00 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.274127+0000 -> 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:09.274111+0000 2019-12-07T19:34:11.573+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021413579s,0.021752699s]) 2019-12-07T19:34:11.573+0000 7f8f24c58700 25 osd.7 33 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.274078+0000 last_rx_front 2019-12-07T19:34:09.274140+0000 -> 2019-12-07T19:34:11.574282+0000 2019-12-07T19:34:11.573+0000 7f8f24c58700 10 osd.7 33 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 31 -> 33 (as per caller) 2019-12-07T19:34:11.573+0000 7f8f24c58700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036206046s,0.036553884s]) 2019-12-07T19:34:11.573+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.031707138s delta_ub 0.022333763s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c552c2400 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574261+0000 last_rx_front 2019-12-07T19:34:09.274157+0000 -> 2019-12-07T19:34:11.574293+0000 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:11.573934+0000) and older pending ping(s) 2019-12-07T19:34:11.573+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6826/37079 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.028887611s delta_ub 0.019514236s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c551b6800 2019-12-07T19:34:11.573+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021974879s,0.022333763s]) 2019-12-07T19:34:11.573+0000 7f8f24c58700 25 osd.7 33 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.274103+0000 last_rx_front 2019-12-07T19:34:09.274169+0000 -> 2019-12-07T19:34:11.574305+0000 2019-12-07T19:34:11.573+0000 7f8f24c58700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019143138s,0.019514236s]) 2019-12-07T19:34:11.573+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.045951799s delta_ub 0.036578424s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c551b7800 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.274078+0000 -> 2019-12-07T19:34:11.574316+0000 last_rx_front 2019-12-07T19:34:11.574282+0000 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:11.573934+0000) and older pending ping(s) 2019-12-07T19:34:11.573+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 6 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.031170363s delta_ub 0.021796988s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c54c9e400 2019-12-07T19:34:11.573+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036195670s,0.036578424s]) 2019-12-07T19:34:11.573+0000 7f8f24c58700 25 osd.7 33 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:09.274111+0000 -> 2019-12-07T19:34:11.574325+0000 2019-12-07T19:34:11.573+0000 7f8f24c58700 25 osd.7 33 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:11.573934+0000) and older pending ping(s) 2019-12-07T19:34:11.573+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.0 v2:172.21.15.131:6830/37079 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.028929713s delta_ub 0.019556338s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c5476d400 2019-12-07T19:34:11.573+0000 7f8f24c58700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021405513s,0.021796988s]) 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:09.274103+0000 -> 2019-12-07T19:34:11.574337+0000 last_rx_front 2019-12-07T19:34:11.574305+0000 2019-12-07T19:34:11.573+0000 7f8f23c56700 25 osd.7 33 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:34:11.573934+0000) and older pending ping(s) 2019-12-07T19:34:11.573+0000 7f8f23c56700 20 osd.7 33 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.019152845s,0.019556338s]) 2019-12-07T19:34:11.591+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 509168 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 145 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c54e07800 con 0x558c54c64000 2019-12-07T19:34:11.609+0000 7f8f24457700 15 osd.7 33 enqueue_peering_evt 1.0s3 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:11.609+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e34) pg 0x558c5500d000 2019-12-07T19:34:11.609+0000 7f8f0898a700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 34, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.610+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 89 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c57452300 con 0x558c54c5bc00 2019-12-07T19:34:11.610+0000 7f8f23c56700 15 osd.7 33 enqueue_peering_evt 1.6s2 epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:11.610+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e34) pg 0x558c54d5c000 2019-12-07T19:34:11.610+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 34, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 146 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c54e07800 con 0x558c54c64000 2019-12-07T19:34:11.611+0000 7f8f24457700 15 osd.7 33 enqueue_peering_evt 1.7s4 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:11.611+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) pg 0x558c54d5f000 2019-12-07T19:34:11.611+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 34, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.613+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 147 ==== osd_map(34..34 src has 1..34) v4 ==== 661+0+0 (crc 0 0 0) 0x558c5738bd40 con 0x558c54c64000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 osd.7 33 OSD::ms_dispatch: osd_map(34..34 src has 1..34) v4 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 osd.7 33 do_waiters -- start 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 osd.7 33 do_waiters -- finish 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 osd.7 33 _dispatch 0x558c5738bd40 osd_map(34..34 src has 1..34) v4 2019-12-07T19:34:11.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 148 ==== MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 ==== 2632+0+0 (crc 0 0 0) 0x558c574a4800 con 0x558c54c64000 2019-12-07T19:34:11.613+0000 7f8f24457700 15 osd.7 33 enqueue_op 0x558c578d2b00 prio 127 cost 0 latency 0.000032 epoch 34 MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:11.613+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e34) 2019-12-07T19:34:11.613+0000 7f8f1199c700 3 osd.7 33 handle_osd_map epochs [34,34], i have 33, src has [1,34] 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 osd.7 33 handle_osd_map got inc map for epoch 34 2019-12-07T19:34:11.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3478e826:::osdmap.33:0# 0x0~0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1438 size 0x1438 (5176) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcba40 blob([0x55000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57dcb730 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396bd0 blob([!~1000,0x2e2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c574fa5b0 sbid 0x0)) need 0x1000~438 cache has 0x[1000~438] 2019-12-07T19:34:11.613+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.613+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e34) queued 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3478e826:::osdmap.33:0# 0x0~1438 = 5176 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 osd.7 33 add_map_bl 33 5176 bytes 2019-12-07T19:34:11.613+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.613+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e34) pg 0x558c5500a000 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 33 dequeue_op 0x558c578d2b00 prio 127 cost 0 latency 0.000081 MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 pg pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 33 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 33 -> 34 (as per caller) 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:11.613+0000 7f8f04181700 20 snap_mapper.remove_oid 1:9b8adabe:::smithi13140796-6:a 2019-12-07T19:34:11.613+0000 7f8f04181700 20 snap_mapper._remove_oid 1:9b8adabe:::smithi13140796-6:a 2019-12-07T19:34:11.613+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:11.613+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:a a 2019-12-07T19:34:11.613+0000 7f8f04181700 20 snap_mapper.clear_snaps 1:9b8adabe:::smithi13140796-6:a 2019-12-07T19:34:11.613+0000 7f8f04181700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.613+0000 7f8f04181700 20 snap_mapper._remove_oid rm SNA_1_000000000000000A_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 osd.7 33 got_full_map 34, nothing requested 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 33'15 (0'0,33'15] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log log((0'0,33'15], crt=33'15) [34'16 (33'14) delete 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,34'17 (33'15) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 34'16 (0'0,34'16] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] add_log_entry 34'16 (33'14) delete 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] add_log_entry 34'17 (33'15) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 17 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 osd.7 33 handle_osd_map pg_num_history pg_num_history(e34 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:11.613+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {34={}} 2019-12-07T19:34:11.613+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 osd.7 33 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e34 [1,34] lci=[13,34]) 2019-12-07T19:34:11.613+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 34'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54ddef00 seq 25 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c76b972f:::inc_osdmap.34:0# key 0x7f7fffffffffffffffc76b97'/!inc_osdmap.34!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c548f7b00 seq 36 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c76b972f:::inc_osdmap.34:0# 0x0~269 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1151 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c76b972f:::inc_osdmap.34:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:a#10 key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0x000000000000000a0000000000000010'o' 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ddef00 1 blobs 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4b0000~1000/1000,1000,0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4b0000~1000] 2019-12-07T19:34:11.613+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# -> 4#1:9b8adabe:::smithi13140796-6:a#10 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d1f0 blob([0x4b0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c000 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a# -> 4#1:9b8adabe:::smithi13140796-6:a#10 = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c5794d1f0) get_ref 0x0~269 Blob(0x558c5794d1f0 blob([0x4b0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c000 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c5794d1f0) get_ref init 0x1000, 1000 2019-12-07T19:34:11.613+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c5794d1f0 blob([0x4b0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5794c000 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5794c018 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.613+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34608) dirty_range mark inline shard dirty 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c76b972f:::inc_osdmap.34:0# 0x0~269 = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3ff8e826:::osdmap.34:0# key 0x7f7fffffffffffffff3ff8e8'&!osdmap.34!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ff8e826:::osdmap.34:0# 0x0~1464 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1152 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3ff8e826:::osdmap.34:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ddef00 2 blobs 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4b1000~2000/1000,2000,0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4b1000~2000] 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f880 blob([0x4b1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a80 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c5757f880) get_ref 0x0~1000 Blob(0x558c5757f880 blob([0x4b1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a80 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c5757f880) get_ref init 0x1000, 1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5757f880 blob([0x4b1000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c38a80 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c38a98 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c38af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c38af0 blob([!~1000,0x4b2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38b60 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57c38af0) get_ref 0x1000~464 Blob(0x558c57c38af0 blob([!~1000,0x4b2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57c38b60 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57c38af0) get_ref init 0x2000, 1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c57c38af0 blob([!~1000,0x4b2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57c38b60 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c38b78 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34888) dirty_range mark inline shard dirty 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ff8e826:::osdmap.34:0# 0x0~1464 = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ddef00 0 blobs 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ddef00 0 blobs 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddef00 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddef00 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34500,0x558c57a34780 shared_blobs 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:11.613+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34608) update #-1:c76b972f:::inc_osdmap.34:0# 2019-12-07T19:34:11.613+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34608) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c76b972f:::inc_osdmap.34:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34888) update #-1:3ff8e826:::osdmap.34:0# 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34888) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3ff8e826:::osdmap.34:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000A_.4_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- SNA_1_000000000000000A_.4_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddef00 allocated 0x[4b0000~3000] released 0x[] 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 freelist allocate 0x4b0000~3000 2019-12-07T19:34:11.613+0000 7f8f1199c700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:11.613+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000034.00000000000000000016' <- 0000000034.00000000000000000016 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000034.00000000000000000017' <- 0000000034.00000000000000000017 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 prepare 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:11.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ddef00 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:11.613+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 7506 (1 ios * 4000 + 3506 bytes) 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes 0x558c57976f00,0x558c57a35400 shared_blobs 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore.extentmap(0x558c57977008) update 4#1:9b8adabe:::smithi13140796-6:a#10 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:a#10 is 500 (410 bytes onode + 2 bytes spanning blobs + 88 bytes inline extents) 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:11.613+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 55136 (54389 bytes onode + 2 bytes spanning blobs + 745 bytes inline extents) 2019-12-07T19:34:11.614+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.614+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:11.614+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:11.614+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.614+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] op_applied version 34'17 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 32043 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f04181700 10 osd.7 33 dequeue_op 0x558c578d2b00 finish 2019-12-07T19:34:11.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 aio_wait 2019-12-07T19:34:11.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddef00 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 58069 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 io_done 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976f00 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572c80 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 58092 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 58092 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xa82d5~e2ec to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xa82d5 0xa82d5~e2ec to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:10.623744+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _allocate len 0x65c1 from 0 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _allocate len 0x65c1 from 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd1e0000 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd1e0000~10000/10000,10000,acd1e0000 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0xc0000 to include allocated 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 33 (was clean) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x782d5 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2d5 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5c1 and padding block with 0xa3f 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0xc0000 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xb65c1 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (33) on file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]), flushing log 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 33 len 0x49 crc 0x30c0ded9) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x20000~1000 to file(ino 1 size 0x20000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x20000 0x20000~1000 to file(ino 1 size 0x20000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x21000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x20000 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x21000 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000022 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 33 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000457418s (0.000000183s flush + 0.000457235s kv commit) 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7b00> 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:11.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:11.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 0x558c57976f00,0x558c57a35400 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:11.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:11.614+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6719 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 33/28 ECSubWriteReply(tid=156, last_complete=34'17, committed=1, applied=1)) v2 -- 0x558c54dfa800 con 0x558c54c64000 2019-12-07T19:34:11.614+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a34500 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a34780 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xb65c1~2277 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xb65c1 0xb65c1~2277 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x865c1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5c1 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x838 and padding block with 0x7c8 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xb8838 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122746s (0.000004254s flush + 0.000118492s kv commit) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddef00> 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 kv_submitted 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddef00 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54ddef00 osr 0x558c54ea4b40 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) _discard drop 22 0x1b000~1000 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7 33 _committed_osd_maps 34..34 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7 33 advance to epoch 34 (<= last 34 <= newest_map 34) 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) prepare_write seq 23 0x1b000~1000 crc b83c5ff7 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) _discard drop 22 0x2000~1000 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) prepare_write seq 23 0x2000~1000 crc 2cc05042 2019-12-07T19:34:11.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:11.614+0000 7f8f141a1700 7 osd.7 34 consume_map version 34 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.0s3 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.9s4 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.2s4 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.bs3 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.3s4 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.cs0 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.4s0 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.ds2 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.6s2 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 identify_splits_and_merges 1.7s4 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 34 prune_pg_created 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:0.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:1.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:2.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:3.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:4.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:5.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:11.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:6.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s)) prio 255 cost 10 e34) pg 0x558c5500d000 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7:7.consume_map 34 (was 33) 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e34) pg 0x558c54d5c000 2019-12-07T19:34:11.614+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 33 -> 34 2019-12-07T19:34:11.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) pg 0x558c54d5f000 2019-12-07T19:34:11.614+0000 7f8f04982700 10 osd.7 pg_epoch: 33 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:11.614+0000 7f8f141a1700 10 osd.7 34 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.0s3 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 33 -> 34 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.9s4 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 33 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.2s4 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.bs3 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.3s4 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.cs0 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.4s0 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.ds2 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.6s2 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.614+0000 7f8f141a1700 15 osd.7 34 enqueue_peering_evt 1.7s4 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.614+0000 7f8f141a1700 7 osd.7 34 activate_map version 34 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c54d9b000 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:11.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c57676000 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 33 -> 34 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c574ff500 seq 52 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:11.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.614+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 33 -> 34 2019-12-07T19:34:11.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:11.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 33 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:11.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2]] state: Active advmap 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff500 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff500 onodes shared_blobs 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff500 allocated 0x[] released 0x[] 2019-12-07T19:34:11.614+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:11.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] on_active_advmap new removed_snaps [a~1], snap_trimq now [a~1] 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 prepare 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff500 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 io_done 2019-12-07T19:34:11.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c54d5c000 2019-12-07T19:34:11.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.614+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:11.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c55141000 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.614+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c5513e000 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xb8838~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.614+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 33 -> 34 2019-12-07T19:34:11.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xb8838 0xb8838~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 33 -> 34 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x88838 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x838 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xccd and padding block with 0x333 2019-12-07T19:34:11.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 33 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 33 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:11.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.614+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:11.614+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:11.614+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.614+0000 7f8f07187700 20 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7f8f07187700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7f8f0217d700 20 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7f8f0217d700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7f8f07187700 20 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7f8f0918b700 20 osd.7 34 agent_entry empty queue 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c5731a000 2019-12-07T19:34:11.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f0217d700 20 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s4 33 -> 34 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 33 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xb8ccd 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:11.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000197385s (0.000000116s flush + 0.000197269s kv commit) 2019-12-07T19:34:11.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff500> 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 kv_submitted 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff500 2019-12-07T19:34:11.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 finishing 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff500 onodes 2019-12-07T19:34:11.615+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff500 done 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:11.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff500 [] 2019-12-07T19:34:11.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.615+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:11.615+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] state: Active: handling ActMap 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] Active: kicking snap trim 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] exit NotTrimming 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming/WaitReservation 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c5500a000 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 32:98 2019-12-07T19:34:11.615+0000 7f8f0297e700 7 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] state: handle ActMap primary 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 33:99 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 33 -> 34 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 33 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c548ab000 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 33 -> 34 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 33 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2]] state: Active advmap 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] on_active_advmap new removed_snaps [a~1], snap_trimq now [a~1] 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 33 while current is 34 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] state: Active: handling ActMap 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] Active: kicking snap trim 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] exit NotTrimming 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming/WaitReservation 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 32:45 2019-12-07T19:34:11.615+0000 7f8f0297e700 7 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] state: handle ActMap primary 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 33:46 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 149 ==== MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c574a3000 con 0x558c54c64000 2019-12-07T19:34:11.615+0000 7f8f24457700 15 osd.7 34 enqueue_op 0x558c54e79080 prio 127 cost 0 latency 0.000017 epoch 34 MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 2019-12-07T19:34:11.615+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2) prio 127 cost 0 e34) 2019-12-07T19:34:11.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 94 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c54e07800 con 0x558c54c5b400 2019-12-07T19:34:11.615+0000 7f8f24457700 15 osd.7 34 enqueue_peering_evt 1.bs3 epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:11.615+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e34) pg 0x558c54d9b000 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730a600 seq 17 2019-12-07T19:34:11.615+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a600 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a600 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 prepare 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a600 2019-12-07T19:34:11.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 io_done 2019-12-07T19:34:11.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2) prio 127 cost 0 e34) queued 2019-12-07T19:34:11.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2) prio 127 cost 0 e34) pg 0x558c5500a000 2019-12-07T19:34:11.615+0000 7f8f08189700 10 osd.7 34 dequeue_op 0x558c54e79080 prio 127 cost 0 latency 0.000301 MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:11.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 150 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c54e07800 con 0x558c54c64000 2019-12-07T19:34:11.615+0000 7f8f08189700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s4 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 2019-12-07T19:34:11.615+0000 7f8f08189700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log log((0'0,34'17], crt=33'15) [] 2019-12-07T19:34:11.615+0000 7f8f24457700 15 osd.7 34 enqueue_peering_evt 1.9s4 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:11.615+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7f8f08189700 20 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] rollforward: entry=34'16 (33'14) delete 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.615+0000 7f8f08189700 20 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] rollforward: entry=34'17 (33'15) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:11.615+0000 7f8f08189700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 17 2019-12-07T19:34:11.615+0000 7f8f08189700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:11.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: NotTrimming: trimming a 2019-12-07T19:34:11.615+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:11.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 95 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c54e07800 con 0x558c54c5b400 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] exit Trimming/WaitReservation 2019-12-07T19:34:11.615+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:11.615+0000 7f8f24457700 15 osd.7 34 enqueue_peering_evt 1.3s4 epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:11.615+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 34:100 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:11.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5730b200 seq 37 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:11.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a#10 onode 0x558c57976f00 txc 0x558c5730b200 2019-12-07T19:34:11.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7f8f0b990700 10 osd.7 34 queueing pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean trimq=[a~1] ps=[4~1,6~2]] for snaptrim 2019-12-07T19:34:11.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a#10 0x0 2019-12-07T19:34:11.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e34) pg 0x558c5513e000 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=34) prio 5 cost 1048576 e34) 2019-12-07T19:34:11.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:11.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore.blob(0x558c578de150) put_ref 0x0~2000 Blob(0x558c578de150 blob([0x48000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=1)))) 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore.blob(0x558c578de070) put_ref 0x0~2000 Blob(0x558c578de070 blob([0x4a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=1)))) 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore.blob(0x558c578de850) put_ref 0x0~2000 Blob(0x558c578de850 blob([0x4c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=1)))) 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore.extentmap(0x558c57977008) dirty_range mark inline shard dirty 2019-12-07T19:34:11.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c578de150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map(0x48000~2000=1)))) 2019-12-07T19:34:11.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 34:101 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x48000~2000] 2019-12-07T19:34:11.615+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x48000~2000] from SharedBlob(0x558c55279730 loaded (sbid 0x2856 ref_map())) 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x48000~2000 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c578de070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map(0x4a000~2000=1)))) 2019-12-07T19:34:11.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4a000~2000] 2019-12-07T19:34:11.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:11.615+0000 7f8f07187700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x4a000~2000] from SharedBlob(0x558c54d3dab0 loaded (sbid 0x2857 ref_map())) 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x4a000~2000 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: NotTrimming: trimming a 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c578de850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map(0x4c000~2000=1)))) 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4c000~2000] 2019-12-07T19:34:11.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x4c000~2000] from SharedBlob(0x558c54d3dc00 loaded (sbid 0x2858 ref_map())) 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] exit Trimming/WaitReservation 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x4c000~2000 2019-12-07T19:34:11.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3dab0,0x558c54d3dc00,0x558c55279730 2019-12-07T19:34:11.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea1200 seq 34 2019-12-07T19:34:11.615+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:a#10 = 0 2019-12-07T19:34:11.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 34:47 2019-12-07T19:34:11.615+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:11.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:11.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:11.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:11.615+0000 7f8f0b990700 10 osd.7 34 queueing pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean trimq=[a~1] ps=[4~1,6~2]] for snaptrim 2019-12-07T19:34:11.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=34) prio 5 cost 1048576 e34) 2019-12-07T19:34:11.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:11.615+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:11.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 5462 (1 ios * 4000 + 1462 bytes) 2019-12-07T19:34:11.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes shared_blobs 0x558c54d3dab0,0x558c54d3dc00,0x558c55279730 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2857 is empty 2019-12-07T19:34:11.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.615+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2858 is empty 2019-12-07T19:34:11.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2856 is empty 2019-12-07T19:34:11.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[] released 0x[48000~6000] 2019-12-07T19:34:11.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x48000~6000 2019-12-07T19:34:11.616+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:11.616+0000 7f8f08189700 10 freelist release 0x48000~6000 2019-12-07T19:34:11.616+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:11.616+0000 7f8f08189700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:11.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 34:48 2019-12-07T19:34:11.616+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=34) prio 5 cost 1048576 e34) queued 2019-12-07T19:34:11.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=34) prio 5 cost 1048576 e34) pg 0x558c57676000 2019-12-07T19:34:11.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:11.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:11.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:11.616+0000 7f8f08189700 10 osd.7 34 dequeue_op 0x558c54e79080 finish 2019-12-07T19:34:11.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] snap_trimmer posting 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap a 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:11.614+0000 7f8f04982700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.616+0000 7f8f04982700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.616+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:11.616+0000 7f8f04982700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.616+0000 7f8f04982700 20 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54ce5c80) upper_bound after SNA_1_000000000000000A_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000000A_.0_0000000000000001.C' 2019-12-07T19:34:11.616+0000 7f8f04982700 20 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.616+0000 7f8f04982700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.616+0000 7f8f04982700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54ce5c80) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:11.616+0000 7f8f04982700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.616+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000A_.0_0000000000000001.C) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:11.616+0000 7f8f04982700 20 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.616+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:11.616+0000 7f8f04982700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.075403950s ub 44.075403950s int 16s) 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2]] SnapTrimmer state: adding snap a to purged_snaps 2019-12-07T19:34:11.616+0000 7f8f04982700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:11.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d59200 seq 24 2019-12-07T19:34:11.616+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2,a~1]] SnapTrimmer state: purged_snaps now [4~1,6~2,a~1], snap_trimq now [] 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:11.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54d59b00 seq 24 2019-12-07T19:34:11.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:11.616+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:11.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:11.616+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:11.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2,a~1]] share_pg_info 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558c548a2a00 con 0x558c54c64000 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558c54e98000 con 0x558c54c19800 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558c54e9aa00 con 0x558c54c5bc00 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558c548a1c00 con 0x558c54c5b400 2019-12-07T19:34:11.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c5500d000 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558c576baa00 con 0x558c54c19c00 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2,a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:11.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean+snaptrim ps=[4~1,6~2,a~1]] exit Trimming 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7f8f06986700 15 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 34:102 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] enter NotTrimming 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:11.614+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e34) pg 0x558c5500a000 2019-12-07T19:34:11.616+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 33 -> 34 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:11.616+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:11.616+0000 7f8f04181700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] snap_trimmer complete 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 33 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=34) prio 5 cost 1048576 e34) queued 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=34) prio 5 cost 1048576 e34) pg 0x558c548ab000 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] snap_trimmer posting 2019-12-07T19:34:11.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c548f6300 seq 38 2019-12-07T19:34:11.616+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap a 2019-12-07T19:34:11.616+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xb8ccd~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:11.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:11.616+0000 7f8f05183700 20 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:11.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:11.616+0000 7f8f05183700 20 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xb8ccd 0xb8ccd~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7f8f05183700 20 osd.7 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x88ccd 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xccd 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:11.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:11.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x162 and padding block with 0xe9e 2019-12-07T19:34:11.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f05183700 20 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.616+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54ce4540) upper_bound after SNA_1_000000000000000A_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000000A_.0_0000000000000001.4' 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54ce4540) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:11.616+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000A_.0_0000000000000001.4) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:11.616+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2]] SnapTrimmer state: adding snap a to purged_snaps 2019-12-07T19:34:11.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:11.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ddec00 seq 37 2019-12-07T19:34:11.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:11.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2,a~1]] SnapTrimmer state: purged_snaps now [4~1,6~2,a~1], snap_trimq now [] 2019-12-07T19:34:11.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c548f7800 seq 15 2019-12-07T19:34:11.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:11.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:11.616+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:11.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddec00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddec00 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddec00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 prepare 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddec00 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 io_done 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:34:11.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:34:11.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:34:11.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2,a~1]] share_pg_info 2019-12-07T19:34:11.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 40 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57452300 con 0x558c54c64400 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 -- 0x558c576bb100 con 0x558c54c64400 2019-12-07T19:34:11.616+0000 7f8f23c56700 15 osd.7 34 enqueue_peering_evt 1.2s4 epoch_sent: 34 epoch_requested: 34 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:11.616+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 -- 0x558c576de000 con 0x558c54c64000 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 -- 0x558c57378000 con 0x558c54c19800 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 -- 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:11.616+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 -- 0x558c57378e00 con 0x558c54c19c00 2019-12-07T19:34:11.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e34) pg 0x558c5731a000 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2,a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean+snaptrim ps=[4~1,6~2,a~1]] exit Trimming 2019-12-07T19:34:11.616+0000 7f8f03980700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:11.616+0000 7f8f03980700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7f8f06986700 15 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 34:49 2019-12-07T19:34:11.616+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] enter NotTrimming 2019-12-07T19:34:11.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:11.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5740a300 seq 23 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:11.616+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:11.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] snap_trimmer complete 2019-12-07T19:34:11.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xb9162 2019-12-07T19:34:11.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:11.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:11.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000710660s (0.000000100s flush + 0.000710560s kv commit) 2019-12-07T19:34:11.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:34:11.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 7 submitting 7 deferred done 0 stable 0 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2459 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976f00 had 1 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3609 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4759 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5885 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7035 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8185 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8208 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8208 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xb9162~2010 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xb9162 0xb9162~2010 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x89162 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x162 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x172 and padding block with 0xe8e 2019-12-07T19:34:11.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a600> 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 kv_submitted 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a600 2019-12-07T19:34:11.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 finishing 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a600 onodes 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a600 done 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:11.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a600 [] 2019-12-07T19:34:11.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:11.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xbb172 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 7 cleaned 0 in 0.000454928s (0.000000090s flush + 0.000454838s kv commit) 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200,0x558c5730b200,0x558c54d59200,0x558c54d59b00,0x558c548f6300,0x558c54ddec00,0x558c548f7800> 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xbb172~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [48000~6000] 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x48000~6000 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xbb172 0xbb172~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c55279730) put 0x558c55279730 removing self from set 0x558c54d0efd0 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279748 in 0x558c53a59a40) _clear 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x8b172 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 _rm buffer(0x558c54cdea20 space 0x558c55279748 0x0~1000 clean) 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x172 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 _rm buffer(0x558c53a699e0 space 0x558c55279748 0x1000~1000 clean) 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x607 and padding block with 0x9f9 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3dc00) put 0x558c54d3dc00 removing self from set 0x558c54d0efd0 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dc18 in 0x558c53a59a40) _clear 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 _rm buffer(0x558c54cdeb40 space 0x558c54d3dc18 0x0~1000 clean) 2019-12-07T19:34:11.617+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 34/28 ECSubWriteReply(tid=157, last_complete=34'17, committed=1, applied=1)) v2 -- 0x558c54dfaa00 con 0x558c54c64000 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 _rm buffer(0x558c5796fcb0 space 0x558c54d3dc18 0x1000~1000 clean) 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3dab0) put 0x558c54d3dab0 removing self from set 0x558c54d0efd0 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dac8 in 0x558c53a59a40) _clear 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 _rm buffer(0x558c54c9acf0 space 0x558c54d3dac8 0x0~1000 clean) 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 _rm buffer(0x558c54cdeab0 space 0x558c54d3dac8 0x1000~1000 clean) 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:11.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:11.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddec00 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddec00 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddec00 done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddec00 [] 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:34:11.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:11.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xbb607 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:11.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000113346s (0.000000079s flush + 0.000113267s kv commit) 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:11.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a300> 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:34:11.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:11.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558c54d5f000 2019-12-07T19:34:11.617+0000 7f8f0117b700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.617+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:11.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:11.641+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258947 2019-12-07T19:34:11.641+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.646+0000 7f8f1fbbe700 10 osd.7 34 tick_without_osd_lock 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fb53000/0x40000000/0x1680000000, data 0x47398b/0x4ab000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 25 osd.7 34 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574337+0000 last_rx_front 2019-12-07T19:34:11.574305+0000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 25 osd.7 34 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:11.574325+0000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 25 osd.7 34 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574316+0000 last_rx_front 2019-12-07T19:34:11.574282+0000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 25 osd.7 34 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574261+0000 last_rx_front 2019-12-07T19:34:11.574293+0000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 25 osd.7 34 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574240+0000 last_rx_front 2019-12-07T19:34:11.574226+0000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 25 osd.7 34 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574165+0000 last_rx_front 2019-12-07T19:34:11.574203+0000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 20 osd.7 34 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 20 osd.7 34 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 20 osd.7 34 scrub_load_below_threshold loadavg per cpu 0.0525 < max 5 = yes 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 20 osd.7 34 sched_scrub load_is_low=1 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 10 osd.7 34 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:11.648268+0000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 20 osd.7 34 sched_scrub done 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 10 osd.7 34 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 20 osd.7 34 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:11.647+0000 7f8f1fbbe700 10 osd.7 34 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:11.691+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.705+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:11.735+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 50 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c19800 2019-12-07T19:34:11.735+0000 7f8f24c58700 15 osd.7 34 enqueue_peering_evt 1.ds2 epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:11.735+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e34) pg 0x558c55141000 2019-12-07T19:34:11.735+0000 7f8f06185700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:11.735+0000 7f8f06185700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.735+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:11.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:11.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c574fec00 seq 21 2019-12-07T19:34:11.735+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:11.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:11.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:11.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:11.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:11.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fec00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fec00 onodes shared_blobs 2019-12-07T19:34:11.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fec00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 prepare 2019-12-07T19:34:11.735+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fec00 2019-12-07T19:34:11.735+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 io_done 2019-12-07T19:34:11.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:11.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.735+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.735+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.735+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:11.735+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860070400 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998593) bluestore 86 GiB free (0.959949), bluefs_ratio 0.0399484 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xbb607~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xbb607 0xbb607~495 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x8b607 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x607 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa9c and padding block with 0x564 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xbba9c 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:11.736+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000165830s (0.000000147s flush + 0.000165683s kv commit) 2019-12-07T19:34:11.736+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fec00> 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.736+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 kv_submitted 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fec00 2019-12-07T19:34:11.736+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 finishing 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fec00 onodes 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fec00 done 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:11.736+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fec00 [] 2019-12-07T19:34:11.736+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:11.736+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.736+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:11.736+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.741+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.791+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.841+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.891+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.942+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.992+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.041+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.092+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.142+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.192+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.242+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.242+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 9 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.715393440s delta_ub 0.036578424s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54dfaa00 con 0x558c552d1c00 2019-12-07T19:34:12.242+0000 7f8f24457700 20 osd.7 34 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036106497s,0.036578424s]) 2019-12-07T19:34:12.242+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.679286943s delta_ub 18446744073.673445119s) v5 -- 0x558c54dfa600 con 0x558c552d1c00 2019-12-07T19:34:12.242+0000 7f8f24457700 10 osd.7 34 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 33 -> 34 (as per caller) 2019-12-07T19:34:12.242+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 9 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.715393440s delta_ub 0.036578424s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57569c00 con 0x558c552d1800 2019-12-07T19:34:12.242+0000 7f8f24c58700 20 osd.7 34 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036090069s,0.036578424s]) 2019-12-07T19:34:12.242+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.679303371s delta_ub 18446744073.673461547s) v5 -- 0x558c548cc200 con 0x558c552d1800 2019-12-07T19:34:12.292+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.312+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:12.342+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.385+0000 7f8f21451700 10 osd.7 34 tick 2019-12-07T19:34:12.385+0000 7f8f21451700 10 osd.7 34 do_waiters -- start 2019-12-07T19:34:12.385+0000 7f8f21451700 10 osd.7 34 do_waiters -- finish 2019-12-07T19:34:12.385+0000 7f8f21451700 20 osd.7 34 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:12.392+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.442+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.492+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.542+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.560+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 7 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 36.011028212s delta_ub 0.013986430s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c54c65400 2019-12-07T19:34:12.560+0000 7f8f24c58700 20 osd.7 34 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013583575s,0.013986430s]) 2019-12-07T19:34:12.560+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 35.997444637s delta_ub 18446744073.695968041s) v5 -- 0x558c548cc200 con 0x558c54c65400 2019-12-07T19:34:12.560+0000 7f8f24c58700 10 osd.7 34 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 33 -> 34 (as per caller) 2019-12-07T19:34:12.560+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 7 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 36.011028212s delta_ub 0.013986430s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57569c00 con 0x558c54c65000 2019-12-07T19:34:12.560+0000 7f8f24c58700 20 osd.7 34 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013511698s,0.013986430s]) 2019-12-07T19:34:12.560+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 35.997516514s delta_ub 18446744073.696039918s) v5 -- 0x558c548cc200 con 0x558c54c65000 2019-12-07T19:34:12.592+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 510225 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.595+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c552c3400 0x558c574b9700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:12.595+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c552c3400 0x558c574b9700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:12.595+0000 7f8f24457700 10 osd.7 34 ms_handle_authentication new session 0x558c57404b60 con 0x558c552c3400 entity client.admin addr 2019-12-07T19:34:12.595+0000 7f8f24457700 10 osd.7 34 ms_handle_authentication session 0x558c57404b60 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:12.595+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3190856828 conn(0x558c552c3400 0x558c574b9700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4705 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:12.595+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4705 172.21.15.131:0/3190856828 1 ==== command(tid 14: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c54d9f680 con 0x558c552c3400 2019-12-07T19:34:12.595+0000 7f8f164bb700 20 osd.7 34 OSD::ms_dispatch: command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:12.595+0000 7f8f164bb700 10 osd.7 34 do_waiters -- start 2019-12-07T19:34:12.595+0000 7f8f164bb700 10 osd.7 34 do_waiters -- finish 2019-12-07T19:34:12.595+0000 7f8f164bb700 20 osd.7 34 _dispatch 0x558c54d9f680 command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:12.596+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3190856828 -- command_reply(tid 14: 0 ) v1 -- 0x558c54ce08c0 con 0x558c552c3400 2019-12-07T19:34:12.616+0000 7f8f0998c700 15 osd.7 34 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:12.616+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) queued 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:34:12.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.1 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.5 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.2 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until readable_until[_ub] 44.052553893s (sent 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_lease(1.4s1 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) e34/34) v1 -- 0x558c54c58c00 con 0x558c54c64400 2019-12-07T19:34:12.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.4s2 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) e34/34) v1 -- 0x558c54c58e00 con 0x558c54c64000 2019-12-07T19:34:12.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.4s4 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) e34/34) v1 -- 0x558c54c58200 con 0x558c54c19800 2019-12-07T19:34:12.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.4s5 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) e34/34) v1 -- 0x558c54c58000 con 0x558c54c5b400 2019-12-07T19:34:12.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.4s3 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) e34/34) v1 -- 0x558c54c58a00 con 0x558c54c19c00 2019-12-07T19:34:12.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 96 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 52.052753396s) e35/35) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c54d9fc20 con 0x558c54c5b400 2019-12-07T19:34:12.616+0000 7f8f24457700 15 osd.7 34 enqueue_peering_evt 1.4s0 epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 45 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 52.052753396s) e34/34) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c54d9fc20 con 0x558c54c19c00 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.616+0000 7f8f24457700 15 osd.7 34 enqueue_peering_evt 1.4s0 epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.5 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.5 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e34) 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) pg 0x558c548ab000 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 35, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.5 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.5 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e34) pg 0x558c548ab000 2019-12-07T19:34:12.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.5 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 51 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 52.052753396s) e34/34) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54d9eb40 con 0x558c54c19800 2019-12-07T19:34:12.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 151 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 52.052753396s) e35/35) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54d9fc20 con 0x558c54c64000 2019-12-07T19:34:12.616+0000 7f8f24c58700 15 osd.7 34 enqueue_peering_evt 1.4s0 epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.2 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f24457700 15 osd.7 34 enqueue_peering_evt 1.4s0 epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.2 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e34) 2019-12-07T19:34:12.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.1 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 41 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 52.052753396s) e35/35) v1 ==== 42+0+0 (crc 0 0 0) 0x558c576dde00 con 0x558c54c64400 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.5 ruub 52.052753396s 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.2 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:12.616+0000 7f8f23c56700 15 osd.7 34 enqueue_peering_evt 1.4s0 epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.2 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until readable_until[_ub] 44.052553893s (sent 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:12.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.2 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e34) pg 0x558c548ab000 2019-12-07T19:34:12.616+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MLeaseAck epoch 34 from osd.2 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.1 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.5 ruub 52.052753396s 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.2 ruub 52.052753396s 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.4 ruub 44.052553893s 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until readable_until[_ub] 44.052553893s (sent 52.052753396s) 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:12.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) pg 0x558c548ab000 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 35, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {35=} 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) pg 0x558c548ab000 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 35, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 152 ==== pg_lease(1.0s3 pg_lease(ru 44.075403950s ub 52.075598409s int 16s) e35/35) v1 ==== 58+0+0 (crc 0 0 0) 0x558c54c58e00 con 0x558c54c64000 2019-12-07T19:34:12.617+0000 7f8f24457700 15 osd.7 34 enqueue_peering_evt 1.0s3 epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s) 2019-12-07T19:34:12.617+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s)) prio 255 cost 10 e35) pg 0x558c5500d000 2019-12-07T19:34:12.617+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 35, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s)) prio 255 cost 10 e35) 2019-12-07T19:34:12.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:12.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 52 ==== pg_lease(1.ds2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s) e34/34) v1 ==== 58+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c54c19800 2019-12-07T19:34:12.618+0000 7f8f24c58700 15 osd.7 34 enqueue_peering_evt 1.ds2 epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s) 2019-12-07T19:34:12.618+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s)) prio 255 cost 10 e34) pg 0x558c55141000 2019-12-07T19:34:12.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s) 2019-12-07T19:34:12.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_lease pg_lease(ru 44.090670001s ub 52.090950317s int 16s) 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_ub 0.036578424s -> ru 44.054091577s 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until now 44.054091577s 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_lb 0.036090069s -> ruub 52.054860248s 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until_ub now 52.054860248s 2019-12-07T19:34:12.618+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.ds0 pg_lease_ack(ruub 52.090950317s) e34/34) v1 -- 0x558c576dcb40 con 0x558c54c19800 2019-12-07T19:34:12.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:12.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:12.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 10 ==== osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 ==== 233+0+802816 (crc 0 0 0) 0x558c54ce9340 con 0x558c54c9f000 2019-12-07T19:34:12.619+0000 7f8f24457700 15 osd.7 34 enqueue_op 0x558c54e78420 prio 63 cost 802816 latency 0.000021 epoch 35 osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 802816 e35) 2019-12-07T19:34:12.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 802816 e35) queued 2019-12-07T19:34:12.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 802816 e35) pg 0x558c57676000 2019-12-07T19:34:12.619+0000 7f8f0297e700 10 osd.7 34 dequeue_op 0x558c54e78420 prio 63 cost 802816 latency 0.000050 osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.619+0000 7f8f0297e700 10 osd.7 34 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 32 -> 35 (as per caller) 2019-12-07T19:34:12.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_request min 35, queue on waiting_for_map client.4540 2019-12-07T19:34:12.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=35}) v3 -- 0x558c57519200 con 0x558c5476d000 2019-12-07T19:34:12.619+0000 7f8f0297e700 10 osd.7 34 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:12.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.620+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 20 ==== osd_map(35..35 src has 1..35) v4 ==== 633+0+0 (crc 0 0 0) 0x558c57944fc0 con 0x558c5476d000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 osd.7 34 OSD::ms_dispatch: osd_map(35..35 src has 1..35) v4 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 osd.7 34 do_waiters -- start 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 osd.7 34 do_waiters -- finish 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 osd.7 34 _dispatch 0x558c57944fc0 osd_map(35..35 src has 1..35) v4 2019-12-07T19:34:12.620+0000 7f8f164bb700 3 osd.7 34 handle_osd_map epochs [35,35], i have 34, src has [1,35] 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 osd.7 34 handle_osd_map got inc map for epoch 35 2019-12-07T19:34:12.620+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ff8e826:::osdmap.34:0# 0x0~0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f880 blob([0x4b1000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c38a80 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c38af0 blob([!~1000,0x4b2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57c38b60 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ff8e826:::osdmap.34:0# 0x0~1464 = 5220 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 osd.7 34 add_map_bl 34 5220 bytes 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 osd.7 34 got_full_map 35, nothing requested 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 osd.7 34 handle_osd_map pg_num_history pg_num_history(e35 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {35={}} 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 osd.7 34 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e35 [1,35] lci=[13,35]) 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c577fc300 seq 26 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cdeb972f:::inc_osdmap.35:0# key 0x7f7fffffffffffffffcdeb97'/!inc_osdmap.35!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cdeb972f:::inc_osdmap.35:0# 0x0~24d 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1153 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cdeb972f:::inc_osdmap.35:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc300 1 blobs 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x48000~1000/1000,1000,0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x48000~1000] 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcb8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcb8f0 blob([0x48000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb960 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c57dcb8f0) get_ref 0x0~24d Blob(0x558c57dcb8f0 blob([0x48000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb960 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c57dcb8f0) get_ref init 0x1000, 1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c57dcb8f0 blob([0x48000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c57dcb960 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57dcb978 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2c888) dirty_range mark inline shard dirty 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cdeb972f:::inc_osdmap.35:0# 0x0~24d = 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3138e826:::osdmap.35:0# key 0x7f7fffffffffffffff3138e8'&!osdmap.35!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3138e826:::osdmap.35:0# 0x0~1448 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1154 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3138e826:::osdmap.35:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc300 2 blobs 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x49000~2000/1000,2000,0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x49000~2000] 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574faaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbab0 blob([0x49000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faaf0 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c57dcbab0) get_ref 0x0~1000 Blob(0x558c57dcbab0 blob([0x49000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faaf0 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c57dcbab0) get_ref init 0x1000, 1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57dcbab0 blob([0x49000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574faaf0 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c574fab08 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bea0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fa770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bea0e0 blob([!~1000,0x4a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa770 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c57bea0e0) get_ref 0x1000~448 Blob(0x558c57bea0e0 blob([!~1000,0x4a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c574fa770 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c57bea0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c57bea0e0 blob([!~1000,0x4a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c574fa770 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c574fa788 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2c108) dirty_range mark inline shard dirty 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3138e826:::osdmap.35:0# 0x0~1448 = 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc300 0 blobs 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc300 0 blobs 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc300 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc300 onodes 0x558c53a91900,0x558c54d2c000,0x558c54d2c780,0x558c55276c80 shared_blobs 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2c108) update #-1:3138e826:::osdmap.35:0# 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2c108) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3138e826:::osdmap.35:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2c888) update #-1:cdeb972f:::inc_osdmap.35:0# 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d2c888) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cdeb972f:::inc_osdmap.35:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc300 allocated 0x[48000~3000] released 0x[] 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 freelist allocate 0x48000~3000 2019-12-07T19:34:12.620+0000 7f8f164bb700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 prepare 2019-12-07T19:34:12.620+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fc300 2019-12-07T19:34:12.620+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 aio_wait 2019-12-07T19:34:12.620+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc300 2019-12-07T19:34:12.620+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 io_done 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c000 had 1 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c780 had 1 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xbba9c~226c to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xbba9c 0xbba9c~226c to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x8ba9c 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa9c 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd08 and padding block with 0x2f8 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xbdd08 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:12.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000120546s (0.000005286s flush + 0.000115260s kv commit) 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:12.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc300> 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:12.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 kv_submitted 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc300 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c577fc300 osr 0x558c54ea4b40 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) _discard drop 23 0x1b000~1000 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7 34 _committed_osd_maps 35..35 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7 34 advance to epoch 35 (<= last 35 <= newest_map 35) 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) prepare_write seq 24 0x1b000~1000 crc a2e3f30a 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) _discard drop 23 0x2000~1000 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57394280) prepare_write seq 24 0x2000~1000 crc 97fb447f 2019-12-07T19:34:12.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:12.620+0000 7f8f141a1700 7 osd.7 35 consume_map version 35 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.0s3 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.9s4 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.2s4 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.bs3 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.3s4 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.cs0 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.4s0 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.ds2 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.6s2 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 identify_splits_and_merges 1.7s4 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 35 prune_pg_created 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:0.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 35 <= 35, requeueing 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {35=} 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:1.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:2.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:3.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:4.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 pending_peering first epoch 35 <= 35, requeueing 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.4s0 to_process <> waiting <> waiting_peering {35=} 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:5.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:6.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7:7.consume_map 35 (was 34) 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s)) prio 255 cost 10 e35) pg 0x558c5500d000 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) pg 0x558c548ab000 2019-12-07T19:34:12.620+0000 7f8f141a1700 10 osd.7 35 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 34 -> 35 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.0s3 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 34 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.9s4 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 34 -> 35 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.2s4 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.bs3 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 11 ==== osd_op(client.4540.0:278 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 ==== 233+0+376832 (crc 0 0 0) 0x558c54ce98c0 con 0x558c54c9f000 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c5500a000 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] state: Active advmap 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.3s4 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.620+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c57335340 prio 63 cost 376832 latency 0.000013 epoch 35 osd_op(client.4540.0:278 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:278 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 376832 e35) 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.cs0 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.4s0 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.ds2 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 12 ==== osd_op(client.4540.0:279 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 ==== 233+0+62 (crc 0 0 0) 0x558c57807b80 con 0x558c54c9f000 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.620+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:12.620+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c57335760 prio 63 cost 62 latency 0.000006 epoch 35 osd_op(client.4540.0:279 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 34 -> 35 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.6s2 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:279 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 62 e35) 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 34 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:12.620+0000 7f8f141a1700 15 osd.7 35 enqueue_peering_evt 1.7s4 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 13 ==== osd_op(client.4540.0:280 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e35) v8 ==== 225+0+0 (crc 0 0 0) 0x558c54ce9b80 con 0x558c54c9f000 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.620+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:12.620+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c5740d080 prio 63 cost 0 latency 0.000006 epoch 35 osd_op(client.4540.0:280 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e35) v8 2019-12-07T19:34:12.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:280 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e35) v8) prio 63 cost 0 e35) 2019-12-07T19:34:12.620+0000 7f8f141a1700 7 osd.7 35 activate_map version 35 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 MLease epoch 35 from osd.1 pg_lease(ru 44.075403950s ub 52.075598409s int 16s) 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] proc_lease pg_lease(ru 44.075403950s ub 52.075598409s int 16s) 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.620+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f0918b700 20 osd.7 35 agent_entry empty queue 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_ub 0.021796988s -> ru 44.053606962s 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c54d5c000 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until now 44.053606962s 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] state: Active: handling ActMap 2019-12-07T19:34:12.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_lb 0.021405513s -> ruub 52.054192896s 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] Active: kicking snap trim 2019-12-07T19:34:12.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c54d9b000 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until_ub now 52.054192896s 2019-12-07T19:34:12.620+0000 7f8f06986700 7 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] state: handle ActMap primary 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 34 -> 35 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.620+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.0s0 pg_lease_ack(ruub 52.075598409s) e35/35) v1 -- 0x558c576dd2c0 con 0x558c54c64000 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.620+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:12.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 34 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:12.620+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 34:50 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:12.620+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:12.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c5500d000 2019-12-07T19:34:12.620+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 34 -> 35 2019-12-07T19:34:12.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.620+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.4 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:12.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:12.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:12.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:12.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 34 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:12.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c55141000 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.1 ruub 44.052553893s 2019-12-07T19:34:12.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.5 ruub 52.052753396s 2019-12-07T19:34:12.620+0000 7f8f06185700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.620+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 34 -> 35 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.2 ruub 52.052753396s 2019-12-07T19:34:12.620+0000 7f8f06185700 10 osd.7 pg_epoch: 34 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:12.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:12.620+0000 7f8f06185700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.4 ruub 52.052753396s 2019-12-07T19:34:12.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.620+0000 7f8f06185700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.620+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:12.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.621+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:12.620+0000 7f8f06185700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.620+0000 7f8f0317f700 20 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.621+0000 7f8f06185700 20 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.620+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until readable_until[_ub] 44.052553893s (sent 52.052753396s) 2019-12-07T19:34:12.621+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.621+0000 7f8f06185700 20 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f06185700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:12.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c5731a000 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f06185700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c54d5f000 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) pg 0x558c548ab000 2019-12-07T19:34:12.621+0000 7f8f06185700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.1 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06185700 20 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s4 34 -> 35 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.0 ruub 44.052553893s 2019-12-07T19:34:12.621+0000 7f8f06185700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 34 -> 35 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.1 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 34 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.5 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 34 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.2 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c5513e000 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.4 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until readable_until[_ub] 44.052553893s (sent 52.052753396s) 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:12.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s)) prio 255 cost 10 e35) pg 0x558c548ab000 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 MLeaseAck epoch 35 from osd.0 pg_lease_ack(ruub 52.052753396s) 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 34 -> 35 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.0 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.1 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 34 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.5 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.2 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until peer osd.4 ruub 52.052753396s 2019-12-07T19:34:12.621+0000 7f8f07988700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recalc_readable_until readable_until[_ub] 52.052753396s (sent 52.052753396s) 2019-12-07T19:34:12.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:12.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:12.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c57676000 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 34 -> 35 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 34 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] state: Active advmap 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.621+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4705 172.21.15.131:0/3190856828 2 ==== command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c576dd2c0 con 0x558c552c3400 2019-12-07T19:34:12.621+0000 7f8f164bb700 20 osd.7 35 OSD::ms_dispatch: command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:12.621+0000 7f8f164bb700 10 osd.7 35 do_waiters -- start 2019-12-07T19:34:12.621+0000 7f8f164bb700 10 osd.7 35 do_waiters -- finish 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.621+0000 7f8f164bb700 20 osd.7 35 _dispatch 0x558c576dd2c0 command(tid 15: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] state: Active: handling ActMap 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] Active: kicking snap trim 2019-12-07T19:34:12.621+0000 7f8f06986700 7 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] state: handle ActMap primary 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:12.621+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 34:103 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] requeue_map_waiters client.4540 0x558c54e78420 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 802816 e35) 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558c548ab000 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 802816 e35) queued 2019-12-07T19:34:12.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 802816 e35) pg 0x558c57676000 2019-12-07T19:34:12.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:278 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 376832 e35) queued 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c54e78420 prio 63 cost 802816 latency 0.001330 osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 pg pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: osd_op(client.4540.0:277 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op: op osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_object_context: obc NOT found in cache: 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# _ 2019-12-07T19:34:12.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# _ = 0 2019-12-07T19:34:12.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# snapset 2019-12-07T19:34:12.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# snapset = 0 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] populate_obc_watchers 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:12.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# = 0 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_object_context: creating obc from disk: 0x558c5738e240 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_object_context: 0x558c5738e240 1:3c88320e:::smithi13140796-1:head rwstate(none n=0 w=0) oi: 1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6480 snapset: 0=[]:{} 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:34:12.621+0000 7f8f06986700 25 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op oi 1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op obc obc(1:3c88320e:::smithi13140796-1:head rwstate(write n=1 w=0)) 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx 0x558c57379500 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] ov 21'4 av 35'19 snapc c=[c] snapset 0=[]:{} 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op append 0~802816 in=802816b 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op 1:3c88320e:::smithi13140796-1:head [write 32768~802816 in=802816b] 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op write 32768~802816 in=802816b 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] clean_regions modifiedclean_offsets: [0~32768,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] make_writeable 1:3c88320e:::smithi13140796-1:head snapset=0=[]:{} snapc=c=[c] 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] got greedy write on clone_obc obc(1:3c88320e:::smithi13140796-1:c rwstate(write n=1 w=0)) 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] cloning v 21'4 to 1:3c88320e:::smithi13140796-1:c v 35'19 snaps=[c] snapset=0=[]:{c=[c]} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] make_writeable 1:3c88320e:::smithi13140796-1:head done, snapset=c=[]:{c=[c]} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] finish_ctx 1:3c88320e:::smithi13140796-1:head 0x558c57379500 op modify 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] set mtime to 2019-12-07T19:34:12.619057+0000 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] final snapset c=[]:{c=[c]} in 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] finish_ctx object 1:3c88320e:::smithi13140796-1:head marks clean_regions clean_offsets: [0~32768,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx alloc reply 0x558c54e13b80 result 0 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] calc_trim_to_aggressive limit = 29'18 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op order client.4540 tid 277 (first) 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] new_repop rep_tid 13 on osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] new_repop: repgather(0x558c576b65a0 0'0 rep_tid=13 committed?=0 r=0) 2019-12-07T19:34:12.621+0000 7f8f06986700 7 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] issue_repop rep_tid 13 o 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] submit_transaction: op Op(1:3c88320e:::smithi13140796-1:head v=35'20 tt=0'0 tid=13 reqid=client.4540.0:277 client_op=osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:12.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# 2019-12-07T19:34:12.621+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3c88320e:::smithi13140796-1:c# key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0x000000000000000cffffffffffffffff'o' 2019-12-07T19:34:12.621+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: found on disk, size 8192 2019-12-07T19:34:12.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key 2019-12-07T19:34:12.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key = 0 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:c projected size 32768 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:head projected size 835584 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] start_rmw: Op(1:3c88320e:::smithi13140796-1:head v=35'20 tt=0'0 tid=13 reqid=client.4540.0:277 client_op=osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[32768~802816]}) 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_state_to_reads: invalidating cache after this op 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_state_to_reads: Op(1:3c88320e:::smithi13140796-1:head v=35'20 tt=0'0 tid=13 reqid=client.4540.0:277 client_op=osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[32768~802816]}) 2019-12-07T19:34:12.621+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: starting commit on Op(1:3c88320e:::smithi13140796-1:head v=35'20 tt=0'0 tid=13 reqid=client.4540.0:277 client_op=osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[32768~802816]}) 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(32768~802816:2) ) 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] generate_transactions: 1:3c88320e:::smithi13140796-1:c, current size is 0 buffers are {} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): new_size start 32768 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_append: {} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:c resetting hinfo to logical size 32768 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] generate_transactions: 1:3c88320e:::smithi13140796-1:head, current size is 32768 buffers are {32768~802816(802816)} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): new_size start 32768 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): adding buffer_update 32768,802816 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_append: {32768~802816(802816)} 2019-12-07T19:34:12.621+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): appending 32768~802816 2019-12-07T19:34:12.622+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3190856828 -- command_reply(tid 15: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c53a6fa40 con 0x558c552c3400 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] encode_and_write: 1:3c88320e:::smithi13140796-1:head new_size 835584 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:head resetting hinfo to logical size 835584 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): marking append 32768 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(32768~802816:2) ) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written: {1:3c88320e:::smithi13140796-1:c={},1:3c88320e:::smithi13140796-1:head={32768~802816(802816)}} 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: op: Op(1:3c88320e:::smithi13140796-1:head v=35'20 tt=0'0 tid=13 reqid=client.4540.0:277 client_op=osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[32768~802816]}) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written_set: {1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[32768~802816]} 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: 1:3c88320e:::smithi13140796-1:c,{} 2019-12-07T19:34:12.623+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: 1:3c88320e:::smithi13140796-1:head,{32768~802816(802816)} 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 send_incremental_map 34 -> 35 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cdeb972f:::inc_osdmap.35:0# 0x0~0 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcb8f0 blob([0x48000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c57dcb960 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cdeb972f:::inc_osdmap.35:0# 0x0~24d = 589 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 add_map_inc_bl 35 589 bytes 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(35..35 src has 1..35) v4 -- 0x558c5738efc0 con 0x558c54c64000 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 34 -> 35 (shared) 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 35/28 ECSubWrite(tid=13, reqid=client.4540.0:277, at_version=35'20, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c5748e800 con 0x558c54c64000 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 send_incremental_map 34 -> 35 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(35..35 src has 1..35) v4 -- 0x558c5738f200 con 0x558c54c19800 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 34 -> 35 (shared) 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 35/28 ECSubWrite(tid=13, reqid=client.4540.0:277, at_version=35'20, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c5748d000 con 0x558c54c19800 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 send_incremental_map 33 -> 35 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c76b972f:::inc_osdmap.34:0# 0x0~0 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794d1f0 blob([0x4b0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5794c000 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c76b972f:::inc_osdmap.34:0# 0x0~269 = 617 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 add_map_inc_bl 34 617 bytes 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(34..35 src has 1..35) v4 -- 0x558c5738f440 con 0x558c54c5bc00 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 33 -> 35 (shared) 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 35/28 ECSubWrite(tid=13, reqid=client.4540.0:277, at_version=35'20, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c551dd000 con 0x558c54c5bc00 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 send_incremental_map 34 -> 35 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(35..35 src has 1..35) v4 -- 0x558c5524c480 con 0x558c54c5b400 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 34 -> 35 (shared) 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 35/28 ECSubWrite(tid=13, reqid=client.4540.0:277, at_version=35'20, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c57a45000 con 0x558c54c5b400 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 send_incremental_map 33 -> 35 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(34..35 src has 1..35) v4 -- 0x558c5524c240 con 0x558c54c19c00 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 35 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 33 -> 35 (shared) 2019-12-07T19:34:12.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 35/28 ECSubWrite(tid=13, reqid=client.4540.0:277, at_version=35'20, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c57a45800 con 0x558c54c19c00 2019-12-07T19:34:12.623+0000 7f8f06986700 20 snap_mapper.add_oid 1:3c88320e:::smithi13140796-1:c c 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 20 snap_mapper.get_snaps 1:3c88320e:::smithi13140796-1:c got.empty() 2019-12-07T19:34:12.623+0000 7f8f06986700 20 snap_mapper.set_snaps 1:3c88320e:::smithi13140796-1:c c 2019-12-07T19:34:12.623+0000 7f8f06986700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:12.623+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:12.623+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 20 trim 35'19 (21'4) clone 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:33:58.625650+0000 0 snaps [c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.623+0000 7f8f06986700 20 trim 35'20 (21'4) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:277 2019-12-07T19:34:12.619057+0000 0 ObjectCleanRegions clean_offsets: [0~32768,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 29'18 (0'0,29'18] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log log((0'0,29'18], crt=29'18) [35'19 (21'4) clone 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:33:58.625650+0000 0 snaps [c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,35'20 (21'4) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:277 2019-12-07T19:34:12.619057+0000 0 ObjectCleanRegions clean_offsets: [0~32768,835584~18446744073708716031], clean_omap: 1, new_object: 0] 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'19 (0'0,35'19] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] add_log_entry 35'19 (21'4) clone 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:33:58.625650+0000 0 snaps [c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] add_log_entry 35'20 (21'4) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:277 2019-12-07T19:34:12.619057+0000 0 ObjectCleanRegions clean_offsets: [0~32768,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log approx pg log length = 20 2019-12-07T19:34:12.623+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log transaction_applied = 1 2019-12-07T19:34:12.623+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:12.623+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 35'19, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57ef8000 seq 25 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3c88320e:::smithi13140796-1:c# key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0x000000000000000cffffffffffffffff'o' 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# -> 0#1:3c88320e:::smithi13140796-1:c# 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1155 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.onode(0x558c574a0f00).flush flush done 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# 0x0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# -> 0#1:3c88320e:::smithi13140796-1:c# 0x0~2000 -> 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) dup src 0x0~2000: 0x0~2000 Blob(0x558c54cf87e0 blob([0x337000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10329 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c54cf87e0 blob([0x337000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c54cf87e0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=1)))) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) dup new Blob(0x558c54d753b0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c54d753b0) get_ref 0x0~2000 Blob(0x558c54d753b0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c54d753b0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) dup dst 0x0~2000: 0x0~2000 Blob(0x558c54d753b0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) dirty_range mark inline shard dirty 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.extentmap(0x558c5738d508) dirty_range mark inline shard dirty 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# -> 0#1:3c88320e:::smithi13140796-1:c# = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# snapset 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# snapset = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# 1 keys 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# 1 keys = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# hinfo_key (42 bytes) 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# hinfo_key (42 bytes) = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:12.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x2000~31000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3c88320e:::smithi13140796-1:head# 0x2000~31000 - have 0x2000 (8192) bytes fadvise_flags 0x0 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x2000~31000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef8000 25 blobs 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.623+0000 7f8f06986700 10 fbmap_alloc 0x558c53a72e00 allocate 0x31000/1000,31000,0 2019-12-07T19:34:12.623+0000 7f8f06986700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4b000~3000/1000,31000,0 2019-12-07T19:34:12.623+0000 7f8f06986700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4b3000~2e000/1000,31000,0 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4b000~3000,0x4b3000~2e000] 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971f10 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9b20 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57971f10) get_ref 0x0~2000 Blob(0x558c57971f10 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9b20 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57971f10) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57971f10 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9b20 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54cf9b38 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971f80 blob([0x4d000~1000,0x4b3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57971f80) get_ref 0x0~2000 Blob(0x558c57971f80 blob([0x4d000~1000,0x4b3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57971f80) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57971f80 blob([0x4d000~1000,0x4b3000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ab0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54d75ac8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75b90 blob([0x4b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d340 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c54d75b90) get_ref 0x0~2000 Blob(0x558c54d75b90 blob([0x4b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d340 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c54d75b90) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54d75b90 blob([0x4b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d340 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54d3d358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57396a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396d90 blob([0x4b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396a10 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57396d90) get_ref 0x0~2000 Blob(0x558c57396d90 blob([0x4b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396a10 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57396d90) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57396d90 blob([0x4b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396a10 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57396a28 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad110 blob([0x4b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad340 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c576ad110) get_ref 0x0~2000 Blob(0x558c576ad110 blob([0x4b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad340 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c576ad110) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c576ad110 blob([0x4b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad340 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c576ad358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad500 blob([0x4ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac620 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c576ad500) get_ref 0x0~2000 Blob(0x558c576ad500 blob([0x4ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac620 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c576ad500) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c576ad500 blob([0x4ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac620 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c576ac638 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c384d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad420 blob([0x4bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c384d0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c576ad420) get_ref 0x0~2000 Blob(0x558c576ad420 blob([0x4bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c384d0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c576ad420) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c576ad420 blob([0x4bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c384d0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c384e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c38460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c383f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c38460 blob([0x4be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c383f0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c38460) get_ref 0x0~2000 Blob(0x558c57c38460 blob([0x4be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c383f0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c38460) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57c38460 blob([0x4be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c383f0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c38408 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c38380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c38380 blob([0x4c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38310 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c38380) get_ref 0x0~2000 Blob(0x558c57c38380 blob([0x4c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38310 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c38380) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57c38380 blob([0x4c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38310 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c38328 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c382a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c382a0 blob([0x4c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39880 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c382a0) get_ref 0x0~2000 Blob(0x558c57c382a0 blob([0x4c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39880 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c382a0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57c382a0 blob([0x4c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39880 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c398f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c398f0 blob([0x4c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39960 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c398f0) get_ref 0x0~2000 Blob(0x558c57c398f0 blob([0x4c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39960 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c398f0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57c398f0 blob([0x4c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39960 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39978 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c399d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c399d0 blob([0x4c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39a40 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c399d0) get_ref 0x0~2000 Blob(0x558c57c399d0 blob([0x4c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39a40 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c399d0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c57c399d0 blob([0x4c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39a40 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39a58 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39ab0 blob([0x4c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39b20 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39ab0) get_ref 0x0~2000 Blob(0x558c57c39ab0 blob([0x4c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39b20 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57c39ab0 blob([0x4c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39b20 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39b38 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39b90 blob([0x4ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39c00 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39b90) get_ref 0x0~2000 Blob(0x558c57c39b90 blob([0x4ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39c00 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39b90) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57c39b90 blob([0x4ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39c00 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39c18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39c70 blob([0x4cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39ce0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39c70) get_ref 0x0~2000 Blob(0x558c57c39c70 blob([0x4cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39ce0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39c70) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57c39c70 blob([0x4cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39ce0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39cf8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39d50 blob([0x4ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39dc0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39d50) get_ref 0x0~2000 Blob(0x558c57c39d50 blob([0x4ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39dc0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39d50) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57c39d50 blob([0x4ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39dc0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39dd8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39e30 blob([0x4d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39ea0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39e30) get_ref 0x0~2000 Blob(0x558c57c39e30 blob([0x4d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39ea0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39e30) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57c39e30 blob([0x4d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39ea0 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39eb8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39f10 blob([0x4d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39f80 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39f10) get_ref 0x0~2000 Blob(0x558c57c39f10 blob([0x4d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39f80 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c57c39f10) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57c39f10 blob([0x4d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c39f80 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57c39f98 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55278ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55278ee0 blob([0x4d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279110 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c55278ee0) get_ref 0x0~2000 Blob(0x558c55278ee0 blob([0x4d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279110 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.blob(0x558c55278ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c55278ee0 blob([0x4d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279110 sbid 0x0)) 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c55279128 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55264620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265ea0 blob([0x4d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264620 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c55265ea0) get_ref 0x0~2000 Blob(0x558c55265ea0 blob([0x4d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264620 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c55265ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c55265ea0 blob([0x4d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55264620 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c55264638 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e62000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e62070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e62000 blob([0x4d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62070 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e62000) get_ref 0x0~2000 Blob(0x558c57e62000 blob([0x4d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62070 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e62000) get_ref init 0x2000, 1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57e62000 blob([0x4d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e62070 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57e62088 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e620e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e62150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e620e0 blob([0x4da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62150 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e620e0) get_ref 0x0~2000 Blob(0x558c57e620e0 blob([0x4da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62150 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e620e0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57e620e0 blob([0x4da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e62150 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57e62168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e621c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e62230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e621c0 blob([0x4dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62230 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e621c0) get_ref 0x0~2000 Blob(0x558c57e621c0 blob([0x4dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62230 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e621c0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57e621c0 blob([0x4dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e62230 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57e62248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e622a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e62310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e622a0 blob([0x4de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62310 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e622a0) get_ref 0x0~2000 Blob(0x558c57e622a0 blob([0x4de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e62310 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e622a0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57e622a0 blob([0x4de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e62310 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57e62328 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e62380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e623f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e62380 blob([0x4e0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e623f0 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e62380) get_ref 0x0~1000 Blob(0x558c57e62380 blob([0x4e0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e623f0 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.blob(0x558c57e62380) get_ref init 0x1000, 1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~1000: 0x0~1000 Blob(0x558c57e62380 blob([0x4e0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57e623f0 sbid 0x0)) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57e62408 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x33000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) dirty_range mark inline shard dirty 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x2000~31000 = 0 2019-12-07T19:34:12.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:12.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C311C407.c.smithi13140796-1..' <- OBJ_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:12.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..' <- SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:12.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000035.00000000000000000019' <- 0000000035.00000000000000000019 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000035.00000000000000000020' <- 0000000035.00000000000000000020 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8000 cost 405036 (50 ios * 4000 + 205036 bytes) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8000 onodes 0x558c5738d400,0x558c574a0f00 shared_blobs 0x558c54cf8a80 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.extentmap(0x558c5738d508) update 0#1:3c88320e:::smithi13140796-1:c# 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.extentmap(0x558c5738d508) update inline shard 30 bytes from 1 extents 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3c88320e:::smithi13140796-1:c# is 445 (409 bytes onode + 2 bytes spanning blobs + 34 bytes inline extents) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) update 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) update inline shard 507 bytes from 26 extents 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3c88320e:::smithi13140796-1:head# is 1068 (555 bytes onode + 2 bytes spanning blobs + 511 bytes inline extents) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2859 is 11 SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2))) 2019-12-07T19:34:12.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8000 allocated 0x[4b000~3000,4b3000~2e000] released 0x[] 2019-12-07T19:34:12.624+0000 7f8f06986700 10 freelist allocate 0x4b000~3000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:12.624+0000 7f8f06986700 10 freelist allocate 0x4b3000~2e000 2019-12-07T19:34:12.624+0000 7f8f06986700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 prepare 2019-12-07T19:34:12.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef8000 2019-12-07T19:34:12.624+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op_applied version 35'20 2019-12-07T19:34:12.624+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] eval_repop repgather(0x558c576b65a0 35'20 rep_tid=13 committed?=0 r=0) 2019-12-07T19:34:12.624+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:12.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:279 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 62 e35) queued 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:278 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 376832 e35) pg 0x558c57676000 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c57335340 prio 63 cost 376832 latency 0.003780 osd_op(client.4540.0:278 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 pg pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: osd_op(client.4540.0:278 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op: op osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.624+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 aio_wait 2019-12-07T19:34:12.624+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8000 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:12.624+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 io_done 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_object_context: found obc in cache: 0x558c5738e240 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_object_context: 0x558c5738e240 1:3c88320e:::smithi13140796-1:head rwstate(write n=1 w=0) oi: 1:3c88320e:::smithi13140796-1:head(35'20 client.4540.0:277 dirty|data_digest s 835584 uv 20 dd a234accf alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6480 snapset: c=[]:{c=[c]} 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(35'20 client.4540.0:277 dirty|data_digest s 835584 uv 20 dd a234accf alloc_hint [0 0 0]) 2019-12-07T19:34:12.624+0000 7f8f0297e700 25 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op oi 1:3c88320e:::smithi13140796-1:head(35'20 client.4540.0:277 dirty|data_digest s 835584 uv 20 dd a234accf alloc_hint [0 0 0]) 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4144 < min_flush_size 524288 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op obc obc(1:3c88320e:::smithi13140796-1:head rwstate(write n=2 w=0)) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx 0x558c57450000 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738d400 had 1 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0f00 had 1 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] ov 35'20 av 35'21 snapc c=[c] snapset c=[]:{c=[c]} 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4167 < min_flush_size 524288 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4167 < min_flush_size 524288 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xbdd08~1047 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op append 0~376832 in=376832b 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xbdd08 0xbdd08~1047 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x8dd08 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd08 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd4f and padding block with 0x2b1 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op 1:3c88320e:::smithi13140796-1:head [write 835584~376832 in=376832b] 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op write 835584~376832 in=376832b 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] clean_regions modifiedclean_offsets: [0~835584,1212416~18446744073708339199], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] make_writeable 1:3c88320e:::smithi13140796-1:head snapset=c=[]:{c=[c]} snapc=c=[c] 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] make_writeable 1:3c88320e:::smithi13140796-1:head done, snapset=c=[]:{c=[c]} 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] finish_ctx 1:3c88320e:::smithi13140796-1:head 0x558c57450000 op modify 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] set mtime to 2019-12-07T19:34:12.621193+0000 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] final snapset c=[]:{c=[c]} in 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xbed4f 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:12.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] finish_ctx object 1:3c88320e:::smithi13140796-1:head marks clean_regions clean_offsets: [0~835584,1212416~18446744073708339199], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000175868s (0.000044432s flush + 0.000131436s kv commit) 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx alloc reply 0x558c5738db80 result 0 2019-12-07T19:34:12.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] calc_trim_to_aggressive limit = 29'18 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8000> 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:12.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 kv_submitted 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8000 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op order client.4540 tid 278 last was 277 2019-12-07T19:34:12.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 finishing 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8000 onodes 0x558c5738d400,0x558c574a0f00 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf9b38 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a598c0 space 0x558c54cf9b38 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d358 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a599e0 space 0x558c54d3d358 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] new_repop rep_tid 14 on osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75ac8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59950 space 0x558c54d75ac8 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55264638 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e643f0 space 0x558c55264638 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279128 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64360 space 0x558c55279128 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396a28 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59a70 space 0x558c57396a28 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac638 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59b90 space 0x558c576ac638 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] new_repop: repgather(0x558c576b6360 0'0 rep_tid=14 committed?=0 r=0) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad358 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59b00 space 0x558c576ad358 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38328 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59d40 space 0x558c57c38328 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38408 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59cb0 space 0x558c57c38408 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c384e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59c20 space 0x558c57c384e8 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39898 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59dd0 space 0x558c57c39898 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f0297e700 7 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] issue_repop rep_tid 14 o 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39978 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59e60 space 0x558c57c39978 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39a58 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a59ef0 space 0x558c57c39a58 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39b38 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64000 space 0x558c57c39b38 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39c18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64090 space 0x558c57c39c18 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39cf8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64120 space 0x558c57c39cf8 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39dd8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e641b0 space 0x558c57c39dd8 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39eb8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64240 space 0x558c57c39eb8 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39f98 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e642d0 space 0x558c57c39f98 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e62088 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64480 space 0x558c57e62088 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e62168 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64510 space 0x558c57e62168 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e62248 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e645a0 space 0x558c57e62248 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] submit_transaction: op Op(1:3c88320e:::smithi13140796-1:head v=35'21 tt=0'0 tid=14 reqid=client.4540.0:278 client_op=osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e62328 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e64630 space 0x558c57e62328 0x0~2000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e62408 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57e646c0 space 0x558c57e62408 0x0~1000 writing nocache) 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8000 done 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:12.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8000 [] 2019-12-07T19:34:12.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:head projected size 1212416 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] start_rmw: Op(1:3c88320e:::smithi13140796-1:head v=35'21 tt=0'0 tid=14 reqid=client.4540.0:278 client_op=osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[835584~376832]}) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_state_to_reads: Op(1:3c88320e:::smithi13140796-1:head v=35'21 tt=0'0 tid=14 reqid=client.4540.0:278 client_op=osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[835584~376832]}) 2019-12-07T19:34:12.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: starting commit on Op(1:3c88320e:::smithi13140796-1:head v=35'21 tt=0'0 tid=14 reqid=client.4540.0:278 client_op=osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[835584~376832]}) 2019-12-07T19:34:12.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(32768~802816:2) ) 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] generate_transactions: 1:3c88320e:::smithi13140796-1:head, current size is 835584 buffers are {835584~376832(376832)} 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): new_size start 835584 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): adding buffer_update 835584,376832 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_append: {835584~376832(376832)} 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): appending 835584~376832 2019-12-07T19:34:12.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 97 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c58e00 con 0x558c54c5b400 2019-12-07T19:34:12.625+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c5740d1e0 prio 196 cost 0 latency 0.000014 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.625+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 46 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfa800 con 0x558c54c19c00 2019-12-07T19:34:12.625+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d2f20 prio 196 cost 0 latency 0.000011 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.625+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3190856828 conn(0x558c552c3400 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 64 2019-12-07T19:34:12.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3190856828 conn(0x558c552c3400 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:12.625+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3190856828 conn(0x558c552c3400 0x558c574b9700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:12.625+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3190856828 conn(0x558c552c3400 0x558c574b9700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:12.625+0000 7f8f164bb700 2 osd.7 35 ms_handle_reset con 0x558c552c3400 session 0x558c57404b60 2019-12-07T19:34:12.625+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] reap_dead start 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] encode_and_write: 1:3c88320e:::smithi13140796-1:head new_size 1212416 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:head resetting hinfo to logical size 1212416 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): marking append 835584 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(32768~802816:2) ) 2019-12-07T19:34:12.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written: {1:3c88320e:::smithi13140796-1:head={835584~376832(376832)}} 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: op: Op(1:3c88320e:::smithi13140796-1:head v=35'21 tt=0'0 tid=14 reqid=client.4540.0:278 client_op=osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[835584~376832]}) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written_set: {1:3c88320e:::smithi13140796-1:head=[835584~376832]} 2019-12-07T19:34:12.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 35/28 ECSubWrite(tid=14, reqid=client.4540.0:278, at_version=35'21, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c54e41800 con 0x558c54c64000 2019-12-07T19:34:12.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 35/28 ECSubWrite(tid=14, reqid=client.4540.0:278, at_version=35'21, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c54e3e800 con 0x558c54c19800 2019-12-07T19:34:12.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 35/28 ECSubWrite(tid=14, reqid=client.4540.0:278, at_version=35'21, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c575bf000 con 0x558c54c5bc00 2019-12-07T19:34:12.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 35/28 ECSubWrite(tid=14, reqid=client.4540.0:278, at_version=35'21, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c54d87000 con 0x558c54c5b400 2019-12-07T19:34:12.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 35/28 ECSubWrite(tid=14, reqid=client.4540.0:278, at_version=35'21, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c54d87800 con 0x558c54c19c00 2019-12-07T19:34:12.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 53 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57569a00 con 0x558c54c19800 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:12.626+0000 7f8f24c58700 15 osd.7 35 enqueue_op 0x558c54e79a20 prio 196 cost 0 latency 0.000020 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.626+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 trim 35'21 (35'20) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:278 2019-12-07T19:34:12.621193+0000 0 ObjectCleanRegions clean_offsets: [0~835584,1212416~18446744073708339199], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'20 (0'0,35'20] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log log((0'0,35'20], crt=29'18) [35'21 (35'20) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:278 2019-12-07T19:34:12.621193+0000 0 ObjectCleanRegions clean_offsets: [0~835584,1212416~18446744073708339199], clean_omap: 1, new_object: 0] 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'20 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] add_log_entry 35'21 (35'20) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:278 2019-12-07T19:34:12.621193+0000 0 ObjectCleanRegions clean_offsets: [0~835584,1212416~18446744073708339199], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'20 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log approx pg log length = 21 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'20 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log transaction_applied = 1 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:12.626+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 35'21, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c577fcf00 seq 26 2019-12-07T19:34:12.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys = 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x33000~17000 2019-12-07T19:34:12.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 153 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c54c64000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3c88320e:::smithi13140796-1:head# 0x33000~17000 - have 0x33000 (208896) bytes fadvise_flags 0x0 2019-12-07T19:34:12.626+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d3e40 prio 196 cost 0 latency 0.000020 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.626+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x33000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fcf00 12 blobs 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate 0x17000/1000,17000,0 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4e1000~17000/1000,17000,0 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4e1000~17000] 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beaa10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d180 blob([0x4e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beaa10 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d180) get_ref 0x0~2000 Blob(0x558c5794d180 blob([0x4e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beaa10 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d180) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c5794d180 blob([0x4e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaa10 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57beaa28 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396930 blob([0x4e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d110 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c57396930) get_ref 0x0~2000 Blob(0x558c57396930 blob([0x4e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d110 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c57396930) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c57396930 blob([0x4e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d110 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d128 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3d50 blob([0x4e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3c70 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3d50) get_ref 0x0~2000 Blob(0x558c574d3d50 blob([0x4e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3c70 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3d50) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c574d3d50 blob([0x4e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3c70 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d3c88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fde30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d8f0 blob([0x4e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fde30 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c54d3d8f0) get_ref 0x0~2000 Blob(0x558c54d3d8f0 blob([0x4e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fde30 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c54d3d8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c54d3d8f0 blob([0x4e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fde30 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574fde48 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3ce0 blob([0x4e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3c00 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3ce0) get_ref 0x0~2000 Blob(0x558c574d3ce0 blob([0x4e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3c00 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c574d3ce0 blob([0x4e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3c00 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d3c18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fdf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fdf10 blob([0x4eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3b20 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574fdf10) get_ref 0x0~2000 Blob(0x558c574fdf10 blob([0x4eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3b20 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574fdf10) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c574fdf10 blob([0x4eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3b20 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c574d3b38 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b110 blob([0x4ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567ab0 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5742b110) get_ref 0x0~2000 Blob(0x558c5742b110 blob([0x4ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567ab0 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5742b110) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c5742b110 blob([0x4ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ab0 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57567ac8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3b90 blob([0x4ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d0a0 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3b90) get_ref 0x0~2000 Blob(0x558c574d3b90 blob([0x4ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d0a0 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3b90) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c574d3b90 blob([0x4ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d0a0 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5794d0b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57397c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b030 blob([0x4f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57397c00 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5742b030) get_ref 0x0~2000 Blob(0x558c5742b030 blob([0x4f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57397c00 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5742b030) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c5742b030 blob([0x4f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57397c00 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57397c18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794cfc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57397960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794cfc0 blob([0x4f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57397960 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cfc0) get_ref 0x0~2000 Blob(0x558c5794cfc0 blob([0x4f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57397960 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c5794cfc0 blob([0x4f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57397960 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57397978 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57397c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396310 blob([0x4f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57397c70 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c57396310) get_ref 0x0~2000 Blob(0x558c57396310 blob([0x4f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57397c70 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c57396310) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c57396310 blob([0x4f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57397c70 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57397c88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573968c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573968c0 blob([!~1000,0x4f7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567030 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c573968c0) get_ref 0x1000~1000 Blob(0x558c573968c0 blob([!~1000,0x4f7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567030 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.blob(0x558c573968c0) get_ref init 0x2000, 1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~1000: 0x1000~1000 Blob(0x558c573968c0 blob([!~1000,0x4f7000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57567030 sbid 0x0)) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c57567048 in 0x558c53a59ce0) _discard 0x1000~1000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4a000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1008) dirty_range mark inline shard dirty 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x33000~17000 = 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000035.00000000000000000021' <- 0000000035.00000000000000000021 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcf00 cost 192326 (24 ios * 4000 + 96326 bytes) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcf00 onodes 0x558c574a0f00 shared_blobs 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1008) update 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1008) update inline shard 749 bytes from 38 extents 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3c88320e:::smithi13140796-1:head# is 1310 (555 bytes onode + 2 bytes spanning blobs + 753 bytes inline extents) 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcf00 allocated 0x[4e1000~17000] released 0x[] 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 freelist allocate 0x4e1000~17000 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 prepare 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fcf00 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'20 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op_applied version 35'21 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] eval_repop repgather(0x558c576b6360 35'21 rep_tid=14 committed?=0 r=0) 2019-12-07T19:34:12.626+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c57335340 finish 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:279 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8) prio 63 cost 62 e35) pg 0x558c57676000 2019-12-07T19:34:12.626+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c57335760 prio 63 cost 62 latency 0.006019 osd_op(client.4540.0:279 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 pg pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.626+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: osd_op(client.4540.0:279 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.626+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 aio_wait 2019-12-07T19:34:12.626+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcf00 2019-12-07T19:34:12.626+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 io_done 2019-12-07T19:34:12.626+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op: op osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.626+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:12.626+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:12.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:12.626+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_object_context: found obc in cache: 0x558c5738e240 2019-12-07T19:34:12.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:12.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:12.626+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_object_context: 0x558c5738e240 1:3c88320e:::smithi13140796-1:head rwstate(write n=2 w=0) oi: 1:3c88320e:::smithi13140796-1:head(35'21 client.4540.0:278 dirty|data_digest s 1212416 uv 21 dd f05001eb alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6480 snapset: c=[]:{c=[c]} 2019-12-07T19:34:12.626+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(35'21 client.4540.0:278 dirty|data_digest s 1212416 uv 21 dd f05001eb alloc_hint [0 0 0]) 2019-12-07T19:34:12.627+0000 7f8f06986700 25 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op oi 1:3c88320e:::smithi13140796-1:head(35'21 client.4540.0:278 dirty|data_digest s 1212416 uv 21 dd f05001eb alloc_hint [0 0 0]) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_op obc obc(1:3c88320e:::smithi13140796-1:head rwstate(write n=3 w=0)) 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2519 < min_flush_size 524288 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx 0x558c5737aa00 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] ov 35'21 av 35'22 snapc c=[c] snapset c=[]:{c=[c]} 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0f00 had 1 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2542 < min_flush_size 524288 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] do_osd_op setxattr _header (55) in=62b 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2542 < min_flush_size 524288 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xbed4f~9ee to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] make_writeable 1:3c88320e:::smithi13140796-1:head snapset=c=[]:{c=[c]} snapc=c=[c] 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xbed4f 0xbed4f~9ee to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] make_writeable 1:3c88320e:::smithi13140796-1:head done, snapset=c=[]:{c=[c]} 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~90000 x_off 0x8ed4f 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd4f 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x73d and padding block with 0x8c3 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] finish_ctx 1:3c88320e:::smithi13140796-1:head 0x558c5737aa00 op modify 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] set mtime to 2019-12-07T19:34:12.621204+0000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] final snapset c=[]:{c=[c]} in 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] finish_ctx object 1:3c88320e:::smithi13140796-1:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] execute_ctx alloc reply 0x558c5738d680 result 0 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xbf73d 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] calc_trim_to_aggressive limit = 29'18 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op order client.4540 tid 279 last was 278 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000136325s (0.000046925s flush + 0.000089400s kv commit) 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] new_repop rep_tid 15 on osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcf00> 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:12.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 kv_submitted 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcf00 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] new_repop: repgather(0x558c576b6900 0'0 rep_tid=15 committed?=0 r=0) 2019-12-07T19:34:12.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 finishing 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcf00 onodes 0x558c574a0f00 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57397978 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a7950 space 0x558c57397978 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f06986700 7 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] issue_repop rep_tid 15 o 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57397c18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a78c0 space 0x558c57397c18 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57397c88 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a79e0 space 0x558c57397c88 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3b38 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a7710 space 0x558c574d3b38 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3c18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a7680 space 0x558c574d3c18 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3c88 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a7560 space 0x558c574d3c88 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fde48 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a75f0 space 0x558c574fde48 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] submit_transaction: op Op(1:3c88320e:::smithi13140796-1:head v=35'22 tt=0'0 tid=15 reqid=client.4540.0:279 client_op=osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567048 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a7a70 space 0x558c57567048 0x1000~1000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567ac8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a77a0 space 0x558c57567ac8 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d0b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a7830 space 0x558c5794d0b8 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d128 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a74d0 space 0x558c5794d128 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beaa28 in 0x558c53a59ce0) _finish_write discard buffer(0x558c579a7440 space 0x558c57beaa28 0x0~2000 writing nocache) 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcf00 done 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:12.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcf00 [] 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:head projected size 1212416 2019-12-07T19:34:12.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:12.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] start_rmw: Op(1:3c88320e:::smithi13140796-1:head v=35'22 tt=0'0 tid=15 reqid=client.4540.0:279 client_op=osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_state_to_reads: Op(1:3c88320e:::smithi13140796-1:head v=35'22 tt=0'0 tid=15 reqid=client.4540.0:279 client_op=osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: starting commit on Op(1:3c88320e:::smithi13140796-1:head v=35'22 tt=0'0 tid=15 reqid=client.4540.0:279 client_op=osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(32768~802816:2) ) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] generate_transactions: 1:3c88320e:::smithi13140796-1:head, current size is 1212416 buffers are {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): new_size start 1212416 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): to_append: {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] operator(): 1:3c88320e:::smithi13140796-1:head resetting hinfo to logical size 1212416 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(32768~802816:2) ) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written: {1:3c88320e:::smithi13140796-1:head={}} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: op: Op(1:3c88320e:::smithi13140796-1:head v=35'22 tt=0'0 tid=15 reqid=client.4540.0:279 client_op=osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written_set: {1:3c88320e:::smithi13140796-1:head=[]} 2019-12-07T19:34:12.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 35/28 ECSubWrite(tid=15, reqid=client.4540.0:279, at_version=35'22, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c575be800 con 0x558c54c64000 2019-12-07T19:34:12.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 35/28 ECSubWrite(tid=15, reqid=client.4540.0:279, at_version=35'22, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c575be000 con 0x558c54c19800 2019-12-07T19:34:12.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 35/28 ECSubWrite(tid=15, reqid=client.4540.0:279, at_version=35'22, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c575c0000 con 0x558c54c5bc00 2019-12-07T19:34:12.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 90 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c548cc000 con 0x558c54c5bc00 2019-12-07T19:34:12.627+0000 7f8f23c56700 15 osd.7 35 enqueue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000014 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 35/28 ECSubWrite(tid=15, reqid=client.4540.0:279, at_version=35'22, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c575c1800 con 0x558c54c5b400 2019-12-07T19:34:12.627+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 35/28 ECSubWrite(tid=15, reqid=client.4540.0:279, at_version=35'22, trim_to=0'0, roll_forward_to=29'18)) v2 -- 0x558c575c0800 con 0x558c54c19c00 2019-12-07T19:34:12.627+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:12.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 98 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfaa00 con 0x558c54c5b400 2019-12-07T19:34:12.627+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d3ce0 prio 196 cost 0 latency 0.000014 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 20 trim 35'22 (35'21) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:279 2019-12-07T19:34:12.621204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.627+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'21 (0'0,35'21] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log log((0'0,35'21], crt=29'18) [35'22 (35'21) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:279 2019-12-07T19:34:12.621204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'21 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] add_log_entry 35'22 (35'21) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:279 2019-12-07T19:34:12.621204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'21 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log approx pg log length = 22 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'21 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] append_log transaction_applied = 1 2019-12-07T19:34:12.627+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:12.627+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 35'22, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:12.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ddf800 seq 27 2019-12-07T19:34:12.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 3 keys 2019-12-07T19:34:12.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 3 keys = 0 2019-12-07T19:34:12.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:12.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000035.00000000000000000022' <- 0000000035.00000000000000000022 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:12.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:12.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddf800 cost 6082 (1 ios * 4000 + 2082 bytes) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddf800 onodes 0x558c574a0f00 shared_blobs 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c574a1008) update 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3c88320e:::smithi13140796-1:head# is 1310 (555 bytes onode + 2 bytes spanning blobs + 753 bytes inline extents) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddf800 allocated 0x[] released 0x[] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 prepare 2019-12-07T19:34:12.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddf800 2019-12-07T19:34:12.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 io_done 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2243 < min_flush_size 524288 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 lua=35'21 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] op_applied version 35'22 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2489 < min_flush_size 524288 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0f00 had 1 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] eval_repop repgather(0x558c576b6900 35'22 rep_tid=15 committed?=0 r=0) 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2512 < min_flush_size 524288 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2512 < min_flush_size 524288 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c57335760 finish 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xbf73d~9d0 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xbf73d 0xbf73d~9d0 to file(ino 16 size 0xc0000 mtime 2019-12-07T19:34:11.615309+0000 allocated c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~90000]) 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _allocate len 0x10d from 0 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _allocate len 0x10d from 1 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd1f0000 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd1f0000~10000/10000,10000,acd1f0000 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0xd0000 to include allocated 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 34 (was clean) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:12.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 47 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57569e00 con 0x558c54c19c00 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x8f73d 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x73d 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x10d and padding block with 0xef3 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0xd0000 2019-12-07T19:34:12.627+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d34a0 prio 196 cost 0 latency 0.000012 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c5740d1e0 prio 196 cost 0 latency 0.002141 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c5740d1e0 finish 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d2f20 prio 196 cost 0 latency 0.002129 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d2f20 finish 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3e40 prio 196 cost 0 latency 0.001379 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3e40 finish 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c54e79a20 prio 196 cost 0 latency 0.001532 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000364 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc010d 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=13, last_complete=35'20, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:12.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3c88320e:::smithi13140796-1:head v=35'20 tt=0'0 tid=13 reqid=client.4540.0:277 client_op=osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[32768~802816]}) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=29'18 crt=29'18 lcod 27'16 mlcod 27'16 active+clean ps=[4~1,6~2,a~1]] repop_all_committed: repop tid 13 all committed 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] eval_repop repgather(0x558c576b65a0 35'20 rep_tid=13 committed?=1 r=0) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] commit: repgather(0x558c576b65a0 35'20 rep_tid=13 committed?=1 r=0) 2019-12-07T19:34:12.627+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] log_op_stats osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 inb 802816 outb 0 lat 0.007805 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] sending reply on osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 0x558c54e13b80 2019-12-07T19:34:12.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(277 smithi13140796-1 [append 0~802816] v35'20 uv20 ondisk = 0) v8 -- 0x558c54e13b80 con 0x558c54c9f000 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:12.627+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 34:104 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] removing repgather(0x558c576b65a0 35'20 rep_tid=13 committed?=1 r=0) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] q front is repgather(0x558c576b65a0 35'20 rep_tid=13 committed?=1 r=0) 2019-12-07T19:34:12.627+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] remove_repop repgather(0x558c576b65a0 35'20 rep_tid=13 committed?=1 r=0) 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: Op(1:3c88320e:::smithi13140796-1:head v=35'20 tt=0'0 tid=13 reqid=client.4540.0:277 client_op=osd_op(client.4540.0:277 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~802816 in=802816b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[32768~802816]}) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(32768~802816:2) ) 2019-12-07T19:34:12.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 99 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57569c00 con 0x558c54c5b400 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:12.627+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000012 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3ce0 prio 196 cost 0 latency 0.000493 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3ce0 finish 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d34a0 prio 196 cost 0 latency 0.000335 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:280 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e35) v8) prio 63 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000069 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:280 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e35) v8) prio 63 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.627+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c5740d080 prio 63 cost 0 latency 0.007018 osd_op(client.4540.0:280 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e35) v8 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: osd_op(client.4540.0:280 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e35) v8 2019-12-07T19:34:12.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] do_op: op osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8 2019-12-07T19:34:12.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:12.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] do_op osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:12.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] get_object_context: found obc in cache: 0x558c5738e240 2019-12-07T19:34:12.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] get_object_context: 0x558c5738e240 1:3c88320e:::smithi13140796-1:head rwstate(write n=2 w=0) oi: 1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6480 snapset: c=[]:{c=[c]} 2019-12-07T19:34:12.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) 2019-12-07T19:34:12.627+0000 7f8f0297e700 25 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] do_op oi 1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) 2019-12-07T19:34:12.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] do_op waiting for rw locks 2019-12-07T19:34:12.627+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:12.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 154 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54dfa000 con 0x558c54c64000 2019-12-07T19:34:12.627+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d3600 prio 196 cost 0 latency 0.000007 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.627+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3600 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 155 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55251200 con 0x558c54c64000 2019-12-07T19:34:12.628+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d29a0 prio 196 cost 0 latency 0.000011 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d29a0 prio 196 cost 0 latency 0.000026 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d29a0 finish 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.628+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 54 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54dfb200 con 0x558c54c19800 2019-12-07T19:34:12.628+0000 7f8f24c58700 15 osd.7 35 enqueue_op 0x558c578d2f20 prio 196 cost 0 latency 0.000013 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.628+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 55 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c58400 con 0x558c54c19800 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.628+0000 7f8f24c58700 15 osd.7 35 enqueue_op 0x558c578d3e40 prio 196 cost 0 latency 0.000005 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d2f20 prio 196 cost 0 latency 0.000028 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d2f20 finish 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3e40 prio 196 cost 0 latency 0.000029 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 91 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c55270a00 con 0x558c54c5bc00 2019-12-07T19:34:12.628+0000 7f8f23c56700 15 osd.7 35 enqueue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000017 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3e40 finish 2019-12-07T19:34:12.628+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 92 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c576b4c00 con 0x558c54c5bc00 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=14, last_complete=35'21, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f23c56700 15 osd.7 35 enqueue_op 0x558c578d38c0 prio 196 cost 0 latency 0.000007 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3c88320e:::smithi13140796-1:head v=35'21 tt=0'0 tid=14 reqid=client.4540.0:278 client_op=osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[835584~376832]}) 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'20 crt=29'18 lcod 29'18 mlcod 29'18 active+clean ps=[4~1,6~2,a~1]] repop_all_committed: repop tid 14 all committed 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] eval_repop repgather(0x558c576b6360 35'21 rep_tid=14 committed?=1 r=0) 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] commit: repgather(0x558c576b6360 35'21 rep_tid=14 committed?=1 r=0) 2019-12-07T19:34:12.628+0000 7f8f0297e700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] log_op_stats osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 inb 376832 outb 0 lat 0.007495 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] sending reply on osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 0x558c5738db80 2019-12-07T19:34:12.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(278 smithi13140796-1 [append 0~376832] v35'21 uv21 ondisk = 0) v8 -- 0x558c5738db80 con 0x558c54c9f000 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:12.628+0000 7f8f0297e700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 35:105 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] removing repgather(0x558c576b6360 35'21 rep_tid=14 committed?=1 r=0) 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] q front is repgather(0x558c576b6360 35'21 rep_tid=14 committed?=1 r=0) 2019-12-07T19:34:12.628+0000 7f8f0297e700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] remove_repop repgather(0x558c576b6360 35'21 rep_tid=14 committed?=1 r=0) 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: Op(1:3c88320e:::smithi13140796-1:head v=35'21 tt=0'0 tid=14 reqid=client.4540.0:278 client_op=osd_op(client.4540.0:278 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~376832 in=376832b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[835584~376832]}) 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:12.628+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3a20 finish 2019-12-07T19:34:12.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d38c0 prio 196 cost 0 latency 0.000117 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.628+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c578d38c0 finish 2019-12-07T19:34:12.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (34) on file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]), flushing log 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:12.629+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 34 len 0x49 crc 0x8b926614) 2019-12-07T19:34:12.629+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x21000~1000 to file(ino 1 size 0x21000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:12.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x21000 0x21000~1000 to file(ino 1 size 0x21000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x22000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x21000 2019-12-07T19:34:12.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 48 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c548cce00 con 0x558c54c19c00 2019-12-07T19:34:12.629+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c578d2dc0 prio 196 cost 0 latency 0.000008 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.629+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x22000 2019-12-07T19:34:12.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:12.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.629+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d2dc0 prio 196 cost 0 latency 0.000025 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000023 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:12.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=15, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.629+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d2dc0 finish 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 34 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002336346s (0.000000100s flush + 0.002336246s kv commit) 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:12.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddf800> 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:12.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 kv_submitted 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddf800 2019-12-07T19:34:12.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 finishing 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddf800 onodes 0x558c574a0f00 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddf800 done 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:12.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddf800 [] 2019-12-07T19:34:12.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:12.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3c88320e:::smithi13140796-1:head v=35'22 tt=0'0 tid=15 reqid=client.4540.0:279 client_op=osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'21 crt=29'18 lcod 35'20 mlcod 35'20 active+clean ps=[4~1,6~2,a~1]] repop_all_committed: repop tid 15 all committed 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] eval_repop repgather(0x558c576b6900 35'22 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] commit: repgather(0x558c576b6900 35'22 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:12.629+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] log_op_stats osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 inb 0 outb 0 lat 0.009011 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] sending reply on osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) v8 0x558c5738d680 2019-12-07T19:34:12.629+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(279 smithi13140796-1 [setxattr (55)] v35'22 uv22 ondisk = 0) v8 -- 0x558c5738d680 con 0x558c54c9f000 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:12.629+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 35:106 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] removing repgather(0x558c576b6900 35'22 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] q front is repgather(0x558c576b6900 35'22 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:12.629+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] remove_repop repgather(0x558c576b6900 35'22 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] requeue_op 0x558c5740d080 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8) prio 63 cost 0 e35) 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: Op(1:3c88320e:::smithi13140796-1:head v=35'22 tt=0'0 tid=15 reqid=client.4540.0:279 client_op=osd_op(client.4540.0:279 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc c=[c] ondisk+write+known_if_redirected e35) roll_forward_to=29'18 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:12.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8) prio 63 cost 0 e35) queued 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:12.629+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: starting commit on Op(1:3c88320e:::smithi13140796-1:head v=0'0 tt=0'0 tid=16 reqid=client.4540.0:279 roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: op: Op(1:3c88320e:::smithi13140796-1:head v=0'0 tt=0'0 tid=16 reqid=client.4540.0:279 roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:12.629+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:12.630+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 35/28 ECSubWrite(tid=16, reqid=client.4540.0:279, at_version=0'0, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c54693000 con 0x558c54c64000 2019-12-07T19:34:12.630+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 35/28 ECSubWrite(tid=16, reqid=client.4540.0:279, at_version=0'0, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c54895800 con 0x558c54c19800 2019-12-07T19:34:12.630+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 35/28 ECSubWrite(tid=16, reqid=client.4540.0:279, at_version=0'0, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c54897000 con 0x558c54c5bc00 2019-12-07T19:34:12.630+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 35/28 ECSubWrite(tid=16, reqid=client.4540.0:279, at_version=0'0, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c57dfe000 con 0x558c54c5b400 2019-12-07T19:34:12.630+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 35/28 ECSubWrite(tid=16, reqid=client.4540.0:279, at_version=0'0, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c54e40800 con 0x558c54c19c00 2019-12-07T19:34:12.630+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=29'18 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] append_log log((0'0,35'22], crt=29'18) [] 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] rollforward: entry=35'19 (21'4) clone 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:33:58.625650+0000 0 snaps [c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] rollforward: entry=35'20 (21'4) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:277 2019-12-07T19:34:12.619057+0000 0 ObjectCleanRegions clean_offsets: [0~32768,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] rollforward: entry=35'21 (35'20) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:278 2019-12-07T19:34:12.621193+0000 0 ObjectCleanRegions clean_offsets: [0~835584,1212416~18446744073708339199], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] rollforward: entry=35'22 (35'21) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:279 2019-12-07T19:34:12.621204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:12.630+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] append_log approx pg log length = 22 2019-12-07T19:34:12.630+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] append_log transaction_applied = 1 2019-12-07T19:34:12.630+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:12.630+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:12.630+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea0f00 seq 28 2019-12-07T19:34:12.630+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:12.630+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:12.630+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:12.630+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:12.630+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc010d~471 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc010d 0xc010d~471 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9010d 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x10d 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x57e and padding block with 0xa82 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8) prio 63 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc057e 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c5740d080 prio 63 cost 0 latency 0.009506 osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000179344s (0.000000096s flush + 0.000179248s kv commit) 2019-12-07T19:34:12.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8 2019-12-07T19:34:12.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:12.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:12.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_op: op osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8 2019-12-07T19:34:12.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:12.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_op osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_object_context: found obc in cache: 0x558c5738e240 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_object_context: 0x558c5738e240 1:3c88320e:::smithi13140796-1:head rwstate(none n=0 w=0) oi: 1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6480 snapset: c=[]:{c=[c]} 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) 2019-12-07T19:34:12.630+0000 7f8f0297e700 25 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_op oi 1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_op obc obc(1:3c88320e:::smithi13140796-1:head rwstate(excl n=1 w=0)) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] execute_ctx 0x558c57450000 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] execute_ctx 1:3c88320e:::smithi13140796-1:head [read 0~1] ov 35'22 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_osd_op 1:3c88320e:::smithi13140796-1:head [read 0~1] 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_osd_op read 0~1 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_read 2019-12-07T19:34:12.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 56 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c576b4600 con 0x558c54c19800 2019-12-07T19:34:12.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 100 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57568200 con 0x558c54c5b400 2019-12-07T19:34:12.630+0000 7f8f24c58700 15 osd.7 35 enqueue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000018 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c54e79e40 prio 196 cost 0 latency 0.000013 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] async_read noted for 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.630+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:34:12.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 49 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c58000 con 0x558c54c19c00 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:34:12.630+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c54e79760 prio 196 cost 0 latency 0.000006 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:12.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 156 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55271e00 con 0x558c54c64000 2019-12-07T19:34:12.630+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c54e79600 prio 196 cost 0 latency 0.000005 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] start_read_op: starting ReadOp(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_read_op: starting read ReadOp(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:12.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 93 ==== MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57519200 con 0x558c54c5bc00 2019-12-07T19:34:12.630+0000 7f8f23c56700 15 osd.7 35 enqueue_op 0x558c578d3760 prio 196 cost 0 latency 0.000013 epoch 35 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738c500 con 0x558c54c64000 2019-12-07T19:34:12.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5738c780 con 0x558c54c5b400 2019-12-07T19:34:12.630+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c54d97b80 con 0x558c54c19c00 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 send_incremental_map 32 -> 35 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:34:12.630+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ca8b972f:::inc_osdmap.33:0# 0x0~0 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac0e0 blob([0x54000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c576ac690 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ca8b972f:::inc_osdmap.33:0# 0x0~24d = 589 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 add_map_inc_bl 33 589 bytes 2019-12-07T19:34:12.630+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(33..35 src has 1..35) v4 -- 0x558c57556b40 con 0x558c53ae5400 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 32 -> 35 (shared) 2019-12-07T19:34:12.630+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57efa000 con 0x558c53ae5400 2019-12-07T19:34:12.630+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c57efa000 con 0x558c53ae5400 2019-12-07T19:34:12.630+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(33..35 src has 1..35) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c57556b40 con 0x558c53ae5400 2019-12-07T19:34:12.630+0000 7f8f1119b700 15 osd.7 35 enqueue_op 0x558c57efe000 prio 127 cost 0 latency 0.000014 epoch 35 MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:12.630+0000 7f8f1199c700 20 osd.7 35 OSD::ms_dispatch: osd_map(33..35 src has 1..35) v4 2019-12-07T19:34:12.630+0000 7f8f1199c700 10 osd.7 35 do_waiters -- start 2019-12-07T19:34:12.630+0000 7f8f1199c700 10 osd.7 35 do_waiters -- finish 2019-12-07T19:34:12.630+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f1199c700 20 osd.7 35 _dispatch 0x558c57556b40 osd_map(33..35 src has 1..35) v4 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_read_op: started ReadOp(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,7(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=1(3),4(2),5(1),7(0)) 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000201 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3760 prio 196 cost 0 latency 0.000149 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c578d3760 finish 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e79e40 prio 196 cost 0 latency 0.000262 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e79760 prio 196 cost 0 latency 0.000251 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e79760 finish 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) queued 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2) prio 196 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 101 ==== MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 4260+0+0 (unknown 0 0 0) 0x558c5738c000 con 0x558c54c5b400 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e79600 prio 196 cost 0 latency 0.000272 MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.630+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000015 epoch 35 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e35) queued 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 35/28 ECSubWriteReply(tid=16, last_complete=35'22, committed=1, applied=1)) v2 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: Op(1:3c88320e:::smithi13140796-1:head v=0'0 tt=0'0 tid=16 reqid=client.4540.0:279 roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:12.630+0000 7f8f1199c700 3 osd.7 35 handle_osd_map epochs [33,35], i have 35, src has [1,35] 2019-12-07T19:34:12.630+0000 7f8f1199c700 10 osd.7 35 no new maps here, dropping 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:12.630+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) queued 2019-12-07T19:34:12.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 157 ==== MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x558c5738c280 con 0x558c54c64000 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.630+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c5740c160 prio 127 cost 0 latency 0.000012 epoch 35 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.630+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c57efe000 prio 127 cost 0 latency 0.000240 MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 50 ==== MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 4260+0+0 (unknown 0 0 0) 0x558c5738cf00 con 0x558c54c19c00 2019-12-07T19:34:12.630+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpRead(1.cs0 35/28 ECSubRead(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:12.630+0000 7f8f24457700 15 osd.7 35 enqueue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000011 epoch 35 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) 2019-12-07T19:34:12.630+0000 7f8f06986700 25 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:12.630+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~1000 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x4a000 (303104) 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf87e0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf87e0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:12.630+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:12.631+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf87e0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:12.631+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c54cf8a98 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:12.631+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14d80 space 0x558c54cf8a98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:12.631+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~1000 = 4096 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:12.631+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] get_hash_info: found on disk, size 303104 2019-12-07T19:34:12.631+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key 2019-12-07T19:34:12.631+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key = 0 2019-12-07T19:34:12.631+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 -- 0x558c5738d900 con 0x558c53ae5400 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c57efe000 finish 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) queued 2019-12-07T19:34:12.631+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c5738d900 con 0x558c53ae5400 2019-12-07T19:34:12.631+0000 7f8f1119b700 15 osd.7 35 enqueue_op 0x558c57efe160 prio 127 cost 0 latency 0.000011 epoch 35 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.631+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) 2019-12-07T19:34:12.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000305 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,7(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=1(3),5(1),7(0)) 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:34:12.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) queued 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c5740c160 prio 127 cost 0 latency 0.000313 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,7(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=5(1),7(0)) 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c5740c160 finish 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) queued 2019-12-07T19:34:12.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000322 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,7(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=7(0)) 2019-12-07T19:34:12.631+0000 7f8f0297e700 10 osd.7 35 dequeue_op 0x558c5740d4a0 finish 2019-12-07T19:34:12.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e35) pg 0x558c57676000 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c57efe160 prio 127 cost 0 latency 0.000164 MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] _handle_message: MOSDECSubOpReadReply(1.cs0 35/28 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_sub_read_reply Complete: ReadOp(tid=17, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096, 7(0),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),7(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,7(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=) 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] objects_read_async_cb: got: {1:3c88320e:::smithi13140796-1:head=0,{0~16384(16384)}} 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] execute_ctx 0x558c57450000 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] execute_ctx 1:3c88320e:::smithi13140796-1:head [read 0~1 out=1b] ov 35'22 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_osd_op 1:3c88320e:::smithi13140796-1:head [read 0~1 out=1b] 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_osd_op read 0~1 out=1b 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] execute_ctx alloc reply 0x558c5738c280 result 0 2019-12-07T19:34:12.631+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:12.631+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] log_op_stats osd_op(client.4540.0:280 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e35) v8 inb 0 outb 1 lat 0.010496 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:12.631+0000 7f8f06986700 15 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 35:107 2019-12-07T19:34:12.631+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(280 smithi13140796-1 [read 0~1 out=1b] v0'0 uv22 ondisk = 0) v8 -- 0x558c5738c280 con 0x558c54c9f000 2019-12-07T19:34:12.631+0000 7f8f06986700 10 osd.7 35 dequeue_op 0x558c57efe160 finish 2019-12-07T19:34:12.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 10 osd.7 35 tick_without_osd_lock 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fb08000/0x40000000/0x1680000000, data 0x4bf020/0x4f6000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 25 osd.7 35 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574337+0000 last_rx_front 2019-12-07T19:34:11.574305+0000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 25 osd.7 35 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:11.574325+0000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 25 osd.7 35 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574316+0000 last_rx_front 2019-12-07T19:34:11.574282+0000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 25 osd.7 35 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574261+0000 last_rx_front 2019-12-07T19:34:11.574293+0000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 25 osd.7 35 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574240+0000 last_rx_front 2019-12-07T19:34:11.574226+0000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 25 osd.7 35 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574165+0000 last_rx_front 2019-12-07T19:34:11.574203+0000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 20 osd.7 35 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 20 osd.7 35 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 20 osd.7 35 scrub_load_below_threshold loadavg per cpu 0.0525 < max 5 = yes 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 20 osd.7 35 sched_scrub load_is_low=1 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 10 osd.7 35 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:12.644124+0000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 20 osd.7 35 sched_scrub done 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 10 osd.7 35 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 20 osd.7 35 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:12.643+0000 7f8f1fbbe700 10 osd.7 35 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:12.643+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:12.643+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:12.643+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 24 0x2000~1000 2019-12-07T19:34:12.643+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 97fb447f 2019-12-07T19:34:12.643+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 24 0x1b000~1000 2019-12-07T19:34:12.643+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc a2e3f30a 2019-12-07T19:34:12.643+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.643+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:12.643+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:12.693+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.743+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.793+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.844+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.893+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.944+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.994+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.044+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.094+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.144+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.194+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.244+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.294+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.337+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:13.344+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.374+0000 7f8f21451700 10 osd.7 35 tick 2019-12-07T19:34:13.374+0000 7f8f21451700 10 osd.7 35 do_waiters -- start 2019-12-07T19:34:13.374+0000 7f8f21451700 10 osd.7 35 do_waiters -- finish 2019-12-07T19:34:13.374+0000 7f8f21451700 20 osd.7 35 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:13.394+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.445+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 9 ==== osd_ping(ping e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.911626419s delta_ub 0.021796988s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c19000 2019-12-07T19:34:13.454+0000 7f8f23c56700 20 osd.7 35 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021226366s,0.021796988s]) 2019-12-07T19:34:13.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.890400053s delta_ub 18446744073.688325250s) v5 -- 0x558c576b5a00 con 0x558c54c19000 2019-12-07T19:34:13.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 9 ==== osd_ping(ping e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.911626419s delta_ub 0.021796988s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c18800 2019-12-07T19:34:13.454+0000 7f8f23c56700 20 osd.7 35 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021161936s,0.021796988s]) 2019-12-07T19:34:13.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.890464483s delta_ub 18446744073.688389680s) v5 -- 0x558c576b5200 con 0x558c54c18800 2019-12-07T19:34:13.495+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.545+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.595+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 529066 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.602+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 21 ==== osd_map(35..36 src has 1..36) v4 ==== 1274+0+0 (crc 0 0 0) 0x558c5738f440 con 0x558c5476d000 2019-12-07T19:34:13.602+0000 7f8f164bb700 20 osd.7 35 OSD::ms_dispatch: osd_map(35..36 src has 1..36) v4 2019-12-07T19:34:13.602+0000 7f8f164bb700 10 osd.7 35 do_waiters -- start 2019-12-07T19:34:13.602+0000 7f8f164bb700 10 osd.7 35 do_waiters -- finish 2019-12-07T19:34:13.602+0000 7f8f164bb700 20 osd.7 35 _dispatch 0x558c5738f440 osd_map(35..36 src has 1..36) v4 2019-12-07T19:34:13.602+0000 7f8f164bb700 3 osd.7 35 handle_osd_map epochs [35,36], i have 35, src has [1,36] 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 osd.7 35 handle_osd_map got inc map for epoch 36 2019-12-07T19:34:13.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3138e826:::osdmap.35:0# 0x0~0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbab0 blob([0x49000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574faaf0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bea0e0 blob([!~1000,0x4a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c574fa770 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3138e826:::osdmap.35:0# 0x0~1448 = 5192 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 osd.7 35 add_map_bl 35 5192 bytes 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 osd.7 35 got_full_map 36, nothing requested 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 osd.7 35 handle_osd_map pg_num_history pg_num_history(e36 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {36={1=[4~1,6~2]}} 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) lower_bound to PSN__1_0000000000000003 key 0x00000000000000000000000000000403'.PSN__1_0000000000000003' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 3 key 'PSN__1_0000000000000003' lower_bound not found 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) lower_bound to PSN__1_0000000000000005 key 0x00000000000000000000000000000403'.PSN__1_0000000000000005' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 5 key 'PSN__1_0000000000000005' lower_bound not found 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) lower_bound to PSN__1_0000000000000005 key 0x00000000000000000000000000000403'.PSN__1_0000000000000005' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 5 key 'PSN__1_0000000000000005' lower_bound not found 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) lower_bound to PSN__1_0000000000000008 key 0x00000000000000000000000000000403'.PSN__1_0000000000000008' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c552da280) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 8 key 'PSN__1_0000000000000008' lower_bound not found 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 2 keys 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 osd.7 35 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e36 [1,36] lci=[13,36]) 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c577fc600 seq 27 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c2eb972f:::inc_osdmap.36:0# key 0x7f7fffffffffffffffc2eb97'/!inc_osdmap.36!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c2eb972f:::inc_osdmap.36:0# 0x0~279 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1156 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c2eb972f:::inc_osdmap.36:0# 0x0~279 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~279 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd87 on front/back, now 0x0~1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc600 1 blobs 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4f8000~1000/1000,1000,0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4f8000~1000] 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d24d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d24d0 blob([0x4f8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d260 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c574d24d0) get_ref 0x0~279 Blob(0x558c574d24d0 blob([0x4f8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d260 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c574d24d0) get_ref init 0x1000, 1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~279: 0x0~279 Blob(0x558c574d24d0 blob([0x4f8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x279) SharedBlob(0x558c5794d260 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5794d278 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x279 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d96108) dirty_range mark inline shard dirty 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c2eb972f:::inc_osdmap.36:0# 0x0~279 = 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3a38e826:::osdmap.36:0# key 0x7f7fffffffffffffff3a38e8'&!osdmap.36!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3a38e826:::osdmap.36:0# 0x0~1454 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1157 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3a38e826:::osdmap.36:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~454 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc600 2 blobs 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4f9000~2000/1000,2000,0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4f9000~2000] 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55278e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8d20 blob([0x4f9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278e00 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54cf8d20) get_ref 0x0~1000 Blob(0x558c54cf8d20 blob([0x4f9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278e00 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54cf8d20) get_ref init 0x1000, 1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c54cf8d20 blob([0x4f9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55278e00 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278e18 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55278930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55278930 blob([!~1000,0x4fa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279570 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c55278930) get_ref 0x1000~454 Blob(0x558c55278930 blob([!~1000,0x4fa000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c55279570 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c55278930) get_ref init 0x2000, 1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558c55278930 blob([!~1000,0x4fa000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c55279570 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55279588 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1454 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efb788) dirty_range mark inline shard dirty 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3a38e826:::osdmap.36:0# 0x0~1454 = 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc600 0 blobs 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000004' <- PSN__1_0000000000000004 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000007' <- PSN__1_0000000000000007 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc600 0 blobs 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc600 cost 23428 (4 ios * 4000 + 7428 bytes) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc600 onodes 0x558c53a91900,0x558c54d96000,0x558c55276c80,0x558c57efb680 shared_blobs 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d96108) update #-1:c2eb972f:::inc_osdmap.36:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c54d96108) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c2eb972f:::inc_osdmap.36:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efb788) update #-1:3a38e826:::osdmap.36:0# 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efb788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3a38e826:::osdmap.36:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc600 allocated 0x[4f8000~3000] released 0x[] 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 freelist allocate 0x4f8000~3000 2019-12-07T19:34:13.603+0000 7f8f164bb700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 prepare 2019-12-07T19:34:13.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fc600 2019-12-07T19:34:13.603+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 158 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c64000 2019-12-07T19:34:13.603+0000 7f8f24457700 15 osd.7 35 enqueue_peering_evt 1.9s4 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:13.603+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.603+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 159 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c64000 2019-12-07T19:34:13.603+0000 7f8f24457700 15 osd.7 35 enqueue_peering_evt 1.7s4 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:13.603+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.603+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 aio_wait 2019-12-07T19:34:13.603+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc600 2019-12-07T19:34:13.603+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 io_done 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:13.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) pg 0x558c5500a000 2019-12-07T19:34:13.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.603+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 36, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) pg 0x558c54d5f000 2019-12-07T19:34:13.603+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 36, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8927 < min_flush_size 524288 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96000 had 1 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efb680 had 1 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860004864 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998576) bluestore 86 GiB free (0.959945), bluefs_ratio 0.0399479 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8986 < min_flush_size 524288 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8986 < min_flush_size 524288 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc057e~231a to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc057e 0xc057e~231a to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9057e 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x57e 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x898 and padding block with 0x768 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc2898 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000201295s (0.000041300s flush + 0.000159995s kv commit) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc600> 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c57394280> 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 kv_submitted 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc600 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c577fc600 osr 0x558c54ea4b40 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) prepare_write seq 25 0x1b000~1000 crc e6fdb5f9 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) prepare_write seq 25 0x2000~1000 crc b86eb3df 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 deferred_cleanup 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde000 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34000,0x558c57a34280 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa5c8 in 0x558c53a59340) _finish_write discard buffer(0x558c57e28870 space 0x558c574fa5c8 0x1000~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac6a8 in 0x558c53a59340) _finish_write discard buffer(0x558c57e286c0 space 0x558c576ac6a8 0x0~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcb748 in 0x558c53a59340) _finish_write discard buffer(0x558c57e287e0 space 0x558c57dcb748 0x0~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde000 done 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddef00 deferred_cleanup 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde000 [] 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 deferred_cleanup 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddef00 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34500,0x558c57a34780 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c018 in 0x558c53a59340) _finish_write discard buffer(0x558c57e28a20 space 0x558c5794c018 0x0~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38a98 in 0x558c53a59340) _finish_write discard buffer(0x558c548df830 space 0x558c57c38a98 0x0~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38b78 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba2d0 space 0x558c57c38b78 0x1000~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 160 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c64000 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddef00 done 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc300 deferred_cleanup 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddef00 [] 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 deferred_cleanup 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc300 onodes 0x558c53a91900,0x558c54d2c000,0x558c54d2c780,0x558c55276c80 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fa788 in 0x558c53a59340) _finish_write discard buffer(0x558c57a59170 space 0x558c574fa788 0x1000~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fab08 in 0x558c53a59340) _finish_write discard buffer(0x558c57a59050 space 0x558c574fab08 0x0~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f24457700 15 osd.7 35 enqueue_peering_evt 1.0s3 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcb978 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb8c0 space 0x558c57dcb978 0x0~1000 writing nocache) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc300 done 2019-12-07T19:34:13.603+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc600 deferred_queued 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc300 [] 2019-12-07T19:34:13.603+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e36) pg 0x558c5500d000 2019-12-07T19:34:13.603+0000 7f8f0898a700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 36, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7 35 _committed_osd_maps 36..36 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7 35 advance to epoch 36 (<= last 36 <= newest_map 36) 2019-12-07T19:34:13.603+0000 7f8f141a1700 7 osd.7 36 consume_map version 36 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.0s3 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.9s4 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.2s4 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.bs3 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.3s4 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.cs0 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.4s0 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.ds2 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.6s2 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 identify_splits_and_merges 1.7s4 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7 36 prune_pg_created 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:0.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:1.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:2.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:3.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:13.603+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:4.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:5.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:13.603+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:6.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:13.603+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e36) pg 0x558c5500d000 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7:7.consume_map 36 (was 35) 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.603+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.603+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.603+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) pg 0x558c5500a000 2019-12-07T19:34:13.603+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.603+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.603+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) pg 0x558c54d5f000 2019-12-07T19:34:13.603+0000 7f8f04982700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.603+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 35 -> 36 2019-12-07T19:34:13.603+0000 7f8f04982700 10 osd.7 pg_epoch: 35 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:13.603+0000 7f8f141a1700 10 osd.7 36 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.0s3 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.9s4 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.2s4 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.603+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.603+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.bs3 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 35 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.3s4 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.cs0 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c5731a000 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 35 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:13.604+0000 7f8f04982700 20 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.4s0 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.ds2 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f04982700 20 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:13.604+0000 7f8f04982700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.6s2 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:13.604+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f141a1700 15 osd.7 36 enqueue_peering_evt 1.7s4 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7f8f05183700 20 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f141a1700 7 osd.7 36 activate_map version 36 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 35 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c54d5c000 2019-12-07T19:34:13.604+0000 7f8f04982700 20 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f05183700 20 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f05183700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f0918b700 20 osd.7 36 agent_entry empty queue 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c55141000 2019-12-07T19:34:13.604+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.0s0( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s) 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c57676000 2019-12-07T19:34:13.604+0000 7f8f05183700 20 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 35 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:13.604+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 35 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d58300 seq 25 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:13.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:13.604+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:13.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:13.604+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:13.604+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:13.604+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[4~1,6~2,a~1]] state: Active advmap 2019-12-07T19:34:13.604+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740af00 seq 38 2019-12-07T19:34:13.604+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes shared_blobs 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[] released 0x[] 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] on_active_advmap new purged_snaps [4~1,6~2], now [a~1] 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:34:13.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:34:13.604+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] share_pg_info 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.604+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.604+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.604+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 2019-12-07T19:34:13.604+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[] 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558c574e4000 con 0x558c54c64000 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:13.604+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:13.604+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:13.604+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558c57390700 con 0x558c54c19800 2019-12-07T19:34:13.604+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f08189700 20 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c54d5f000 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc2898~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f08189700 20 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f08189700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc2898 0xc2898~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558c57390e00 con 0x558c54c5bc00 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x92898 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x898 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.604+0000 7f8f0117b700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd0d and padding block with 0x2f3 2019-12-07T19:34:13.604+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558c57391500 con 0x558c54c5b400 2019-12-07T19:34:13.604+0000 7f8f08189700 20 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558c57391c00 con 0x558c54c19c00 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:13.604+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:13.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c5500d000 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] publish_stats_to_osd 35:108 2019-12-07T19:34:13.604+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:13.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5730a300 seq 39 2019-12-07T19:34:13.604+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:13.604+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:13.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:13.604+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:13.604+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a300 onodes shared_blobs 2019-12-07T19:34:13.604+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a300 allocated 0x[] released 0x[] 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 prepare 2019-12-07T19:34:13.604+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a300 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 io_done 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] state: Active: handling ActMap 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] Active: kicking snap trim 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc2d0d 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.604+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.604+0000 7f8f0297e700 7 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:13.604+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:13.604+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c5500a000 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138237s (0.000000147s flush + 0.000138090s kv commit) 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] publish_stats_to_osd 35:109 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300> 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c54d9b000 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] activate_map: Not dirtying info: last_persisted is 35 while current is 36 2019-12-07T19:34:13.604+0000 7f8f04181700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 2019-12-07T19:34:13.604+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:34:13.604+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c577fd200 seq 29 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 35 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc2d0d~8d3 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc2d0d 0xc2d0d~8d3 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x92d0d 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd0d 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5e0 and padding block with 0xa20 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:13.604+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd200 onodes shared_blobs 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd200 allocated 0x[] released 0x[] 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 prepare 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd200 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 io_done 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c548ab000 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558c5513e000 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 35 -> 36 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 35 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 35 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[4~1,6~2,a~1]] state: Active advmap 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc35e0 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] on_active_advmap new purged_snaps [4~1,6~2], now [a~1] 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] share_pg_info 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000129740s (0.000000092s flush + 0.000129648s kv commit) 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00,0x558c5730a300> 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 -- 0x558c57392300 con 0x558c54c64400 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [] 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.604+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 kv_submitted 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a300 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a300 finishing 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 -- 0x558c57392a00 con 0x558c54c64000 2019-12-07T19:34:13.604+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a300 onodes 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a300 done 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a300 [] 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc35e0~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 -- 0x558c57393100 con 0x558c54c19800 2019-12-07T19:34:13.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc35e0 0xc35e0~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x935e0 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5e0 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.604+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa55 and padding block with 0x5ab 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 -- 0x558c57393800 con 0x558c54c5b400 2019-12-07T19:34:13.604+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 -- 0x558c576e1800 con 0x558c54c19c00 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] publish_stats_to_osd 34:51 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] state: Started advmap 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc3a55 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] hit_set_clear 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] agent_stop 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116196s (0.000000087s flush + 0.000116109s kv commit) 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] handle_activate_map: 36 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd200> 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 kv_submitted 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd200 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 finishing 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] activate_map 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd200 onodes 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd200 done 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd200 [] 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] state: Active: handling ActMap 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] Active: kicking snap trim 2019-12-07T19:34:13.604+0000 7f8f0297e700 7 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] publish_stats_to_osd 35:52 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c577fd500 seq 16 2019-12-07T19:34:13.604+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd500 onodes shared_blobs 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd500 allocated 0x[] released 0x[] 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd500 prepare 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd500 2019-12-07T19:34:13.604+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd500 io_done 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc3a55~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc3a55 0xc3a55~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x93a55 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa55 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xeca and padding block with 0x136 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc3eca 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000121287s (0.000000090s flush + 0.000121197s kv commit) 2019-12-07T19:34:13.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd500> 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd500 kv_submitted 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd500 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd500 finishing 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd500 onodes 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd500 done 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd500 [] 2019-12-07T19:34:13.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 94 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c57392300 con 0x558c54c5bc00 2019-12-07T19:34:13.606+0000 7f8f23c56700 15 osd.7 36 enqueue_peering_evt 1.6s2 epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:13.606+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s)) prio 255 cost 10 e36) pg 0x558c54d5c000 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.6s0( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115942144s ub 46.115942144s int 16s) 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d20600 seq 53 2019-12-07T19:34:13.606+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes shared_blobs 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:13.606+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc3eca~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc3eca 0xc3eca~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x93eca 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xeca 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x33f and padding block with 0xcc1 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc433f 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.606+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127747s (0.000000088s flush + 0.000127659s kv commit) 2019-12-07T19:34:13.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20600> 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:13.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:13.606+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:13.606+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.606+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 102 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c576e1800 con 0x558c54c5b400 2019-12-07T19:34:13.612+0000 7f8f24457700 15 osd.7 36 enqueue_peering_evt 1.bs3 epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:13.612+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e36) pg 0x558c54d9b000 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c54ea1200 seq 18 2019-12-07T19:34:13.612+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:13.612+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc433f~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc433f 0xc433f~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9433f 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x33f 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7b4 and padding block with 0x84c 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc47b4 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000098573s (0.000000123s flush + 0.000098450s kv commit) 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200> 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:13.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:13.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:13.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 103 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c576e1800 con 0x558c54c5b400 2019-12-07T19:34:13.612+0000 7f8f24457700 15 osd.7 36 enqueue_peering_evt 1.3s4 epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:13.612+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.612+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e36) pg 0x558c5513e000 2019-12-07T19:34:13.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.612+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:13.612+0000 7f8f07187700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.612+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:13.612+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:13.612+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea1500 seq 35 2019-12-07T19:34:13.612+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:13.612+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:13.612+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:13.612+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:13.612+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:13.612+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.612+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:34:13.612+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:34:13.612+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:13.612+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:13.612+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.612+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc47b4~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc47b4 0xc47b4~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x947b4 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7b4 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc29 and padding block with 0x3d7 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc4c29 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118014s (0.000000060s flush + 0.000117954s kv commit) 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 42 ==== pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57392300 con 0x558c54c64400 2019-12-07T19:34:13.613+0000 7f8f23c56700 15 osd.7 36 enqueue_peering_evt 1.2s4 epoch_sent: 36 epoch_requested: 36 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:13.613+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s)) prio 255 cost 10 e36) pg 0x558c5731a000 2019-12-07T19:34:13.613+0000 7f8f07988700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 0(0) info: 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.119144108s ub 46.119144108s int 16s) 2019-12-07T19:34:13.613+0000 7f8f07988700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.613+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:13.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c54ddfb00 seq 24 2019-12-07T19:34:13.613+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:13.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:13.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:13.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddfb00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddfb00 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddfb00 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 prepare 2019-12-07T19:34:13.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddfb00 2019-12-07T19:34:13.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 io_done 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc4c29~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc4c29 0xc4c29~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x94c29 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc29 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9e and padding block with 0xf62 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc509e 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131511s (0.000000128s flush + 0.000131383s kv commit) 2019-12-07T19:34:13.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddfb00> 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 kv_submitted 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddfb00 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 finishing 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddfb00 onodes 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddfb00 done 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddfb00 [] 2019-12-07T19:34:13.613+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.615+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 57 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57393100 con 0x558c54c19800 2019-12-07T19:34:13.615+0000 7f8f24c58700 15 osd.7 36 enqueue_peering_evt 1.ds2 epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s) 2019-12-07T19:34:13.615+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s)) prio 255 cost 10 e36) pg 0x558c55141000 2019-12-07T19:34:13.615+0000 7f8f06185700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s) 2019-12-07T19:34:13.615+0000 7f8f06185700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.615+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:13.615+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:13.615+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c574fef00 seq 22 2019-12-07T19:34:13.615+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:13.615+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:13.615+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:13.615+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:13.615+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:13.615+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fef00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.615+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fef00 onodes shared_blobs 2019-12-07T19:34:13.615+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fef00 allocated 0x[] released 0x[] 2019-12-07T19:34:13.615+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 prepare 2019-12-07T19:34:13.615+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fef00 2019-12-07T19:34:13.615+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 io_done 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:13.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc509e~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc509e 0xc509e~475 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9509e 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9e 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x513 and padding block with 0xaed 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc5513 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:13.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000125421s (0.000000116s flush + 0.000125305s kv commit) 2019-12-07T19:34:13.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fef00> 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 kv_submitted 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fef00 2019-12-07T19:34:13.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 finishing 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fef00 onodes 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fef00 done 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:13.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fef00 [] 2019-12-07T19:34:13.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:13.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:13.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.645+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 10 osd.7 36 tick_without_osd_lock 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fb05000/0x40000000/0x1680000000, data 0x4c06ed/0x4f9000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 25 osd.7 36 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574337+0000 last_rx_front 2019-12-07T19:34:11.574305+0000 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 25 osd.7 36 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:11.574325+0000 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 25 osd.7 36 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574316+0000 last_rx_front 2019-12-07T19:34:11.574282+0000 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 25 osd.7 36 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574261+0000 last_rx_front 2019-12-07T19:34:11.574293+0000 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 25 osd.7 36 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574240+0000 last_rx_front 2019-12-07T19:34:11.574226+0000 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 25 osd.7 36 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574165+0000 last_rx_front 2019-12-07T19:34:11.574203+0000 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 20 osd.7 36 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 10 osd.7 36 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 20 osd.7 36 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:13.676+0000 7f8f1fbbe700 10 osd.7 36 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:13.695+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.745+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.796+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.845+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.896+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.946+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.996+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.046+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.096+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.146+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.196+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.246+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.297+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.347+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.397+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.412+0000 7f8f21451700 10 osd.7 36 tick 2019-12-07T19:34:14.412+0000 7f8f21451700 10 osd.7 36 do_waiters -- start 2019-12-07T19:34:14.412+0000 7f8f21451700 10 osd.7 36 do_waiters -- finish 2019-12-07T19:34:14.412+0000 7f8f21451700 20 osd.7 36 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:14.447+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.497+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.547+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:14.547+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.553+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 10 ==== osd_ping(ping e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 38.011911392s delta_ub 0.021674644s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c18800 2019-12-07T19:34:14.554+0000 7f8f23c56700 20 osd.7 36 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021248750s,0.021674644s]) 2019-12-07T19:34:14.554+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 37.990662642s delta_ub 18446744073.688302866s) v5 -- 0x558c576b5200 con 0x558c54c18800 2019-12-07T19:34:14.554+0000 7f8f23c56700 10 osd.7 36 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 35 -> 36 (as per caller) 2019-12-07T19:34:14.554+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 10 ==== osd_ping(ping e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 38.011911392s delta_ub 0.021674644s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58c00 con 0x558c54c19000 2019-12-07T19:34:14.554+0000 7f8f23c56700 20 osd.7 36 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021170857s,0.021674644s]) 2019-12-07T19:34:14.554+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 37.990740535s delta_ub 18446744073.688380759s) v5 -- 0x558c576b5200 con 0x558c54c19000 2019-12-07T19:34:14.598+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 530550 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 14 ==== osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8 ==== 250+0+54 (crc 0 0 0) 0x558c57f12000 con 0x558c54c9f000 2019-12-07T19:34:14.614+0000 7f8f24457700 15 osd.7 36 enqueue_op 0x558c57efe840 prio 63 cost 54 latency 0.000031 epoch 37 osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.614+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) 2019-12-07T19:34:14.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 15 ==== osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 ==== 226+0+0 (crc 0 0 0) 0x558c57f122c0 con 0x558c54c9f000 2019-12-07T19:34:14.614+0000 7f8f24457700 15 osd.7 36 enqueue_op 0x558c57efeb00 prio 63 cost 0 latency 0.000007 epoch 37 osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.614+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) queued 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) pg 0x558c57676000 2019-12-07T19:34:14.614+0000 7f8f06986700 10 osd.7 36 dequeue_op 0x558c57efe840 prio 63 cost 54 latency 0.000090 osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.614+0000 7f8f06986700 10 osd.7 36 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 35 -> 37 (as per caller) 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_request min 37, queue on waiting_for_map client.4540 2019-12-07T19:34:14.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=37}) v3 -- 0x558c53aa8c00 con 0x558c5476d000 2019-12-07T19:34:14.614+0000 7f8f06986700 10 osd.7 36 dequeue_op 0x558c57efe840 finish 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) queued 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.614+0000 7f8f06986700 10 osd.7 36 dequeue_op 0x558c57efeb00 prio 63 cost 0 latency 0.000112 osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:34:14.614+0000 7f8f06986700 10 osd.7 36 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:14.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.614+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 22 ==== osd_map(37..37 src has 1..37) v4 ==== 633+0+0 (crc 0 0 0) 0x558c57945200 con 0x558c5476d000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 osd.7 36 OSD::ms_dispatch: osd_map(37..37 src has 1..37) v4 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 osd.7 36 do_waiters -- start 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 osd.7 36 do_waiters -- finish 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 osd.7 36 _dispatch 0x558c57945200 osd_map(37..37 src has 1..37) v4 2019-12-07T19:34:14.614+0000 7f8f164bb700 3 osd.7 36 handle_osd_map epochs [37,37], i have 36, src has [1,37] 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 osd.7 36 handle_osd_map got inc map for epoch 37 2019-12-07T19:34:14.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3a38e826:::osdmap.36:0# 0x0~0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8d20 blob([0x4f9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55278e00 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278930 blob([!~1000,0x4fa000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c55279570 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3a38e826:::osdmap.36:0# 0x0~1454 = 5204 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 osd.7 36 add_map_bl 36 5204 bytes 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 osd.7 36 got_full_map 37, nothing requested 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 osd.7 36 handle_osd_map pg_num_history pg_num_history(e37 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {37={}} 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 osd.7 36 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e37 [1,37] lci=[13,37]) 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c577fc900 seq 28 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c82b972f:::inc_osdmap.37:0# key 0x7f7fffffffffffffffc82b97'/!inc_osdmap.37!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:14.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c82b972f:::inc_osdmap.37:0# 0x0~24d 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1158 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c82b972f:::inc_osdmap.37:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc900 1 blobs 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4fb000~1000/1000,1000,0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4fb000~1000] 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57397b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57397b20 blob([0x4fb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677340 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c57397b20) get_ref 0x0~24d Blob(0x558c57397b20 blob([0x4fb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677340 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c57397b20) get_ref init 0x1000, 1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c57397b20 blob([0x4fb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c54677340 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54677358 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efba08) dirty_range mark inline shard dirty 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c82b972f:::inc_osdmap.37:0# 0x0~24d = 0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:37b8e826:::osdmap.37:0# key 0x7f7fffffffffffffff37b8e8'&!osdmap.37!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:14.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:37b8e826:::osdmap.37:0# 0x0~1428 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1159 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:37b8e826:::osdmap.37:0# 0x0~1428 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~428 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbd8 on front/back, now 0x0~1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc900 2 blobs 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4fc000~2000/1000,2000,0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4fc000~2000] 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54ea3260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57538770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54ea3260 blob([0x4fc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538770 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea3260) get_ref 0x0~1000 Blob(0x558c54ea3260 blob([0x4fc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538770 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea3260) get_ref init 0x1000, 1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c54ea3260 blob([0x4fc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57538770 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57538788 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efc150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efc1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efc150 blob([!~1000,0x4fd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc1c0 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc150) get_ref 0x1000~428 Blob(0x558c57efc150 blob([!~1000,0x4fd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57efc1c0 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc150) get_ref init 0x2000, 1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~428: 0x1000~428 Blob(0x558c57efc150 blob([!~1000,0x4fd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558c57efc1c0 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efc1d8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1428 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efbc88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:37b8e826:::osdmap.37:0# 0x0~1428 = 0 2019-12-07T19:34:14.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:14.614+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 104 ==== osd_map(36..37 src has 1..37) v4 ==== 1274+0+0 (unknown 0 0 0) 0x558c579e6000 con 0x558c54c5b400 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 105 ==== pg_update_log_missing(1.3s4 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 ==== 258+0+0 (unknown 0 0 0) 0x558c579e6240 con 0x558c54c5b400 2019-12-07T19:34:14.614+0000 7f8f1199c700 20 osd.7 36 OSD::ms_dispatch: osd_map(36..37 src has 1..37) v4 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f24457700 15 osd.7 36 enqueue_op 0x558c57efec60 prio 127 cost 0 latency 0.000019 epoch 37 pg_update_log_missing(1.3s4 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc900 0 blobs 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.614+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=pg_update_log_missing(1.3s4 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e37) 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:14.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:14.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:14.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=pg_update_log_missing(1.3s4 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc900 0 blobs 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc900 cost 23230 (4 ios * 4000 + 7230 bytes) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc900 onodes 0x558c53a91900,0x558c55276c80,0x558c57efb900,0x558c57efbb80 shared_blobs 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=pg_update_log_missing(1.3s4 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e37) pg 0x558c5513e000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efba08) update #-1:c82b972f:::inc_osdmap.37:0# 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efba08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c82b972f:::inc_osdmap.37:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efbc88) update #-1:37b8e826:::osdmap.37:0# 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efbc88) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:37b8e826:::osdmap.37:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 osd.7 36 dequeue_op 0x558c57efec60 prio 127 cost 0 latency 0.000061 pg_update_log_missing(1.3s4 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 pg pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 osd.7 36 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 35 -> 37 (as per caller) 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc900 allocated 0x[4fb000~3000] released 0x[] 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 freelist allocate 0x4fb000~3000 2019-12-07T19:34:14.614+0000 7f8f164bb700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] _handle_message: pg_update_log_missing(1.3s4 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 prepare 2019-12-07T19:34:14.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fc900 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 36 pg[1.3s4( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] do_update_log_missing op_trim_to = -- op_roll_forward_to = -- 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 update missing, append 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 36 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[a~1]] append_log_entries_update_missing trim_to bool = 0 trim_to = 0'0 2019-12-07T19:34:14.614+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ea0f00 seq 36 2019-12-07T19:34:14.614+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000037.00000000000000000011' <- 0000000037.00000000000000000011 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:14.614+0000 7f8f1199c700 10 osd.7 36 do_waiters -- start 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 4717 (1 ios * 4000 + 717 bytes) 2019-12-07T19:34:14.614+0000 7f8f1199c700 10 osd.7 36 do_waiters -- finish 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:14.614+0000 7f8f1199c700 20 osd.7 36 _dispatch 0x558c579e6000 osd_map(36..37 src has 1..37) v4 2019-12-07T19:34:14.614+0000 7f8f1199c700 3 osd.7 36 handle_osd_map epochs [36,37], i have 37, src has [1,37] 2019-12-07T19:34:14.614+0000 7f8f1199c700 10 osd.7 36 no new maps here, dropping 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:14.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 aio_wait 2019-12-07T19:34:14.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc900 2019-12-07T19:34:14.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 io_done 2019-12-07T19:34:14.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[a~1]] op_applied version 37'11 2019-12-07T19:34:14.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.614+0000 7f8f0317f700 10 osd.7 36 dequeue_op 0x558c57efec60 finish 2019-12-07T19:34:14.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 567 < min_flush_size 524288 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9356 < min_flush_size 524288 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efb900 had 1 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efbb80 had 1 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3860004864 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998576) bluestore 86 GiB free (0.959945), bluefs_ratio 0.0399479 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9379 < min_flush_size 524288 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9379 < min_flush_size 524288 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc5513~24a3 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc5513 0xc5513~24a3 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x95513 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x513 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9b6 and padding block with 0x64a 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc79b6 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000214711s (0.000052829s flush + 0.000161882s kv commit) 2019-12-07T19:34:14.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00,0x558c577fc900> 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:14.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:14.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:14.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 kv_submitted 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc900 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c577fc900 osr 0x558c54ea4b40 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) _discard drop 25 0x1b000~1000 2019-12-07T19:34:14.615+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_update_log_missing_reply(1.3s0 epoch 37/28 rep_tid 59 lcod 37'11) v3 -- 0x558c54d77c00 con 0x558c54c5b400 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) prepare_write seq 26 0x1b000~1000 crc fc221904 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7 36 _committed_osd_maps 37..37 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) _discard drop 25 0x2000~1000 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7 36 advance to epoch 37 (<= last 37 <= newest_map 37) 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) prepare_write seq 26 0x2000~1000 crc 355a7e2 2019-12-07T19:34:14.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f141a1700 7 osd.7 37 consume_map version 37 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.0s3 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.9s4 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.2s4 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.bs3 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.3s4 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.cs0 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.4s0 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.ds2 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.6s2 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 identify_splits_and_merges 1.7s4 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 37 prune_pg_created 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:0.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:1.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:2.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:3.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:4.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:5.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:6.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7:7.consume_map 37 (was 36) 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:14.615+0000 7f8f141a1700 10 osd.7 37 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.0s3 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.9s4 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.2s4 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c5500d000 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.bs3 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.3s4 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c5500a000 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.cs0 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.4s0 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 36 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.ds2 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.6s2 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 36 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:14.615+0000 7f8f141a1700 15 osd.7 37 enqueue_peering_evt 1.7s4 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:14.615+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f141a1700 7 osd.7 37 activate_map version 37 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c54d5c000 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c5731a000 2019-12-07T19:34:14.615+0000 7f8f0918b700 20 osd.7 37 agent_entry empty queue 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 36 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c54d9b000 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f08189700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:14.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 36 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c5513e000 2019-12-07T19:34:14.615+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c57676000 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 36 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 36 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:14.615+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] state: Active advmap 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] state: Active: handling ActMap 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] Active: kicking snap trim 2019-12-07T19:34:14.615+0000 7f8f0297e700 7 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:14.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] publish_stats_to_osd 36:110 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] requeue_map_waiters client.4540 0x558c57efe840,0x558c57efeb00 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c548ab000 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) queued 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) pg 0x558c57676000 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57efe840 prio 63 cost 54 latency 0.001641 osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 36 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:286 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] state: Active advmap 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] state: Active: handling ActMap 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] Active: kicking snap trim 2019-12-07T19:34:14.615+0000 7f8f06986700 7 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:14.615+0000 7f8f06986700 15 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] publish_stats_to_osd 36:53 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) queued 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c54d5f000 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 36 -> 37 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 36 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.615+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 4=[]:{} 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.615+0000 7f8f0297e700 25 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(29'18 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=1 w=0)) 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] execute_ctx 0x558c576dee00 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.615+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] ov 29'18 av 37'23 snapc e=[e,d,c] snapset 4=[]:{} 2019-12-07T19:34:14.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] 2019-12-07T19:34:14.615+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_osd_op copy-from ver 8 in=54b 2019-12-07T19:34:14.615+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] start_copy 1:3a5a3f5b:::smithi13140796-23:head from 1:131421e8:::smithi13140796-7:head @1 v8 flags 0 2019-12-07T19:34:14.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _copy_some obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=1 w=0)) 0x558c57560c10 2019-12-07T19:34:14.616+0000 7f8f0297e700 1 --2- >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x558c57d7f000 0x558c576bc580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:14.616+0000 7f8f0297e700 1 -- --> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] -- osd_op(unknown.0.13:1 1.8s0 1:131421e8:::smithi13140796-7:head [assert-version v8,copy-get max 8388608 in=29b] snapc 0=[] ondisk+read+known_if_redirected e37) v8 -- 0x558c54ce98c0 con 0x558c57d7f000 2019-12-07T19:34:14.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:14.616+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57efe840 finish 2019-12-07T19:34:14.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558c55141000 2019-12-07T19:34:14.616+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efeb00 prio 63 cost 0 latency 0.002004 osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 pg pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:287 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 36 -> 37 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 36 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:14.616+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.616+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] wait_for_blocked_object 1:3a5a3f5b:::smithi13140796-23:head 0x558c57efeb00 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:14.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7f8f24457700 1 --2- >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x558c57d7f000 0x558c576bc580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:14.616+0000 7f8f24457700 1 --2- >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x558c57d7f000 0x558c576bc580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.131:6819/37078 says I am v2:172.21.15.109:54680/0 (socket says 172.21.15.109:54680) 2019-12-07T19:34:14.616+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 learned_addr learned my addr 172.21.15.109:0/36939 (peer_addr_for_me v2:172.21.15.109:0/0) 2019-12-07T19:34:14.616+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x558c57d7f000 0x558c576bc580 crc :-1 s=READY pgs=27 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:14.618+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 95 ==== osd_map(36..37 src has 1..37) v4 ==== 1274+0+0 (unknown 0 0 0) 0x558c57945440 con 0x558c54c5bc00 2019-12-07T19:34:14.618+0000 7f8f1199c700 20 osd.7 37 OSD::ms_dispatch: osd_map(36..37 src has 1..37) v4 2019-12-07T19:34:14.618+0000 7f8f1199c700 10 osd.7 37 do_waiters -- start 2019-12-07T19:34:14.618+0000 7f8f1199c700 10 osd.7 37 do_waiters -- finish 2019-12-07T19:34:14.618+0000 7f8f1199c700 20 osd.7 37 _dispatch 0x558c57945440 osd_map(36..37 src has 1..37) v4 2019-12-07T19:34:14.618+0000 7f8f1199c700 3 osd.7 37 handle_osd_map epochs [36,37], i have 37, src has [1,37] 2019-12-07T19:34:14.618+0000 7f8f1199c700 10 osd.7 37 no new maps here, dropping 2019-12-07T19:34:14.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 96 ==== MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=92, reqid=client.4540.0:294, at_version=37'26, trim_to=0'0, roll_forward_to=29'24)) v2 ==== 3167+0+0 (unknown 0 0 0) 0x558c551dd000 con 0x558c54c5bc00 2019-12-07T19:34:14.618+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c57efe000 prio 127 cost 0 latency 0.000030 epoch 37 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=92, reqid=client.4540.0:294, at_version=37'26, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:14.618+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=92, reqid=client.4540.0:294, at_version=37'26, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=92, reqid=client.4540.0:294, at_version=37'26, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=92, reqid=client.4540.0:294, at_version=37'26, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) pg 0x558c54d5c000 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57efe000 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=92, reqid=client.4540.0:294, at_version=37'26, trim_to=0'0, roll_forward_to=29'24)) v2 pg pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 37 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 35 -> 37 (as per caller) 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=92, reqid=client.4540.0:294, at_version=37'26, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 snap_mapper.add_oid 1:6ee4dfe7:::smithi13140796-47:e c,d,e 2019-12-07T19:34:14.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6ee4dfe7:::smithi13140796-47:e got.empty() 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6ee4dfe7:::smithi13140796-47:e c,d,e 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 29'24 (0'0,29'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] append_log log((0'0,29'24], crt=29'24) [37'25 (29'24) clone 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:04.726001+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,37'26 (29'24) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:294 2019-12-07T19:34:14.617472+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'25 (0'0,37'25] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'24 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] add_log_entry 37'25 (29'24) clone 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:04.726001+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'26 (0'0,37'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'24 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] add_log_entry 37'26 (29'24) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:294 2019-12-07T19:34:14.617472+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'26 (0'0,37'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'24 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] append_log approx pg log length = 26 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'26 (0'0,37'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'24 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.618+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'25, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d20900 seq 54 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:e# key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:14.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:e# 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1160 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.onode(0x558c54ce3400).flush flush done 2019-12-07T19:34:14.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# 0x0 2019-12-07T19:34:14.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:e# 0x0~30000 -> 0x0~30000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x0~2000: 0x0~2000 Blob(0x558c57880d90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574d28c0 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) get_ref 0x0~2000 Blob(0x558c574d28c0 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574d28c0 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x2000~2000: 0x0~2000 Blob(0x558c578dfe30 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c54d750a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) get_ref 0x0~2000 Blob(0x558c54d750a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c54d750a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x4000~2000: 0x0~2000 Blob(0x558c578f1ea0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c577f39d0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c577f39d0) get_ref 0x0~2000 Blob(0x558c577f39d0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c577f39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c577f39d0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x6000~2000: 0x0~2000 Blob(0x558c578f1f80 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c57314310 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57314310) get_ref 0x0~2000 Blob(0x558c57314310 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57314310) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57314310 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x8000~2000: 0x0~2000 Blob(0x558c578f1e30 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574d39d0 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d39d0) get_ref 0x0~2000 Blob(0x558c574d39d0 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c574d39d0 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0xa000~2000: 0x0~2000 Blob(0x558c578f1dc0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574d27e0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) get_ref 0x0~2000 Blob(0x558c574d27e0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0xc000~2000: 0x0~2000 Blob(0x558c578f1d50 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574d2770 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) get_ref 0x0~2000 Blob(0x558c574d2770 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c574d2770 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57881e30 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c573144d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c573144d0) get_ref 0x0~2000 Blob(0x558c573144d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c573144d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c573144d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57881dc0 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c57566e00 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) get_ref 0x0~2000 Blob(0x558c57566e00 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57566e00 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57881d50 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574e0690 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574e0690) get_ref 0x0~2000 Blob(0x558c574e0690 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574e0690) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c574e0690 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57881ce0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574d30a0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) get_ref 0x0~2000 Blob(0x558c574d30a0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57881c70 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c54cf82a0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf82a0) get_ref 0x0~2000 Blob(0x558c54cf82a0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf82a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c54cf82a0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x18000~2000: 0x0~2000 Blob(0x558c57881c00 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574d3030 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) get_ref 0x0~2000 Blob(0x558c574d3030 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c574d3030 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57881b90 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c575bd6c0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) get_ref 0x0~2000 Blob(0x558c575bd6c0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57881b20 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c5742ad90 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) get_ref 0x0~2000 Blob(0x558c5742ad90 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c57881ab0 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c574d3a40 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3a40) get_ref 0x0~2000 Blob(0x558c574d3a40 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3a40) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c574d3a40 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x20000~2000: 0x0~2000 Blob(0x558c57881a40 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c5742ae70 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) get_ref 0x0~2000 Blob(0x558c5742ae70 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x22000~2000: 0x0~2000 Blob(0x558c578819d0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c57396cb0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57396cb0) get_ref 0x0~2000 Blob(0x558c57396cb0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57396cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57396cb0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x24000~2000: 0x0~2000 Blob(0x558c57881960 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c575bc700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) get_ref 0x0~2000 Blob(0x558c575bc700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c575bc700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x26000~2000: 0x0~2000 Blob(0x558c578818f0 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c575bc620 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) get_ref 0x0~2000 Blob(0x558c575bc620 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c575bc620 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x28000~2000: 0x0~2000 Blob(0x558c57880a80 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c57396d20 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57396d20) get_ref 0x0~2000 Blob(0x558c57396d20 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.blob(0x558c57396d20) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c57396d20 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c57880a10 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1)))) 2019-12-07T19:34:14.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c575bc540 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc540) get_ref 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc540) get_ref init 0x2000, 1000 2019-12-07T19:34:14.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 97 ==== MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=93, reqid=client.4540.0:295, at_version=37'27, trim_to=0'0, roll_forward_to=29'24)) v2 ==== 2132+0+0 (unknown 0 0 0) 0x558c551dd800 con 0x558c54c5bc00 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c57efe160 prio 127 cost 0 latency 0.000027 epoch 37 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=93, reqid=client.4540.0:295, at_version=37'27, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c57881ea0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1)))) 2019-12-07T19:34:14.619+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=93, reqid=client.4540.0:295, at_version=37'27, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c57396fc0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57396fc0) get_ref 0x0~2000 Blob(0x558c57396fc0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57396fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c57396fc0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c57881f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup new Blob(0x558c57397030 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397030) get_ref 0x0~2000 Blob(0x558c57397030 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397030) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c57397030 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dirty_range mark inline shard dirty 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:e# = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# snapset 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# snapset = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# 1 keys 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# 1 keys = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# hinfo_key (42 bytes) 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- SNA_1_000000000000000E_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000037.00000000000000000025' <- 0000000037.00000000000000000025 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000037.00000000000000000026' <- 0000000037.00000000000000000026 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20900 cost 8530 (1 ios * 4000 + 4530 bytes) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20900 onodes 0x558c54ce3400,0x558c54e12280 shared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1230 (573 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) update 2#1:6ee4dfe7:::smithi13140796-47:e# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) update inline shard 651 bytes from 24 extents 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:e# is 1069 (412 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20900 allocated 0x[] released 0x[] 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 prepare 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20900 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 io_done 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'26 (0'0,37'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=29'24 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] op_applied version 37'26 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1610 < min_flush_size 524288 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57efe000 finish 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5791 < min_flush_size 524288 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=93, reqid=client.4540.0:295, at_version=37'27, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=93, reqid=client.4540.0:295, at_version=37'27, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) pg 0x558c54d5c000 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57efe160 prio 127 cost 0 latency 0.000279 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=93, reqid=client.4540.0:295, at_version=37'27, trim_to=0'0, roll_forward_to=29'24)) v2 pg pg[1.6s2( v 37'26 (0'0,37'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'26 (0'0,37'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=93, reqid=client.4540.0:295, at_version=37'27, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3400 had 1 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12280 had 1 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'26 (0'0,37'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] append_log log((0'0,37'26], crt=29'24) [37'27 (37'26) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:295 2019-12-07T19:34:14.617498+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 0, new_object: 0] 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5814 < min_flush_size 524288 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5814 < min_flush_size 524288 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc79b6~16b6 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'27 (0'0,37'27] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'26 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] add_log_entry 37'27 (37'26) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:295 2019-12-07T19:34:14.617498+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 0, new_object: 0 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc79b6 0xc79b6~16b6 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'27 (0'0,37'27] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'26 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] append_log approx pg log length = 27 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x979b6 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9b6 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6c and padding block with 0xf94 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'27 (0'0,37'27] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'26 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.619+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'27, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d21b00 seq 55 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:head#1b key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0xfffffffffffffffe000000000000001b'o' 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:head#1b 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# -> 2#1:6ee4dfe7:::smithi13140796-47:head#1b = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# -> 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1161 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.onode(0x558c54e12280).flush flush done 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# -> 2#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 -> 0x0~30000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x0~2000: 0x0~2000 Blob(0x558c574d28c0 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c577f3b90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc906c 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3b90) get_ref 0x0~2000 Blob(0x558c577f3b90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3b90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x0~2000: 0x0~2000 Blob(0x558c577f3b90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d750a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c577f38f0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000216215s (0.000000129s flush + 0.000216086s kv commit) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f38f0) get_ref 0x0~2000 Blob(0x558c577f38f0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f38f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c577f38f0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20900> 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 kv_submitted 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x4000~2000: 0x0~2000 Blob(0x558c577f39d0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20900 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c577f3960 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20900 finishing 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20900 onodes 0x558c54ce3400,0x558c54e12280 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3960) get_ref 0x0~2000 Blob(0x558c577f3960 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20900 done 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3960) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 prepare 2019-12-07T19:34:14.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20900 [] 2019-12-07T19:34:14.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c577f3960 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57314310 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397110 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397110) get_ref 0x0~2000 Blob(0x558c57397110 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397110) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57397110 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x8000~2000: 0x0~2000 Blob(0x558c574d39d0 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397180 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397180) get_ref 0x0~2000 Blob(0x558c57397180 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397180) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57397180 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0xa000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c573971f0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573971f0) get_ref 0x0~2000 Blob(0x558c573971f0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573971f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c573971f0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0xc000~2000: 0x0~2000 Blob(0x558c574d2770 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397260 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397260) get_ref 0x0~2000 Blob(0x558c57397260 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397260) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57397260 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0xe000~2000: 0x0~2000 Blob(0x558c573144d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c573972d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573972d0) get_ref 0x0~2000 Blob(0x558c573972d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573972d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c573972d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57566e00 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397340 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397340) get_ref 0x0~2000 Blob(0x558c57397340 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397340) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57397340 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574e0690 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c573973b0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573973b0) get_ref 0x0~2000 Blob(0x558c573973b0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573973b0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c573973b0 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x14000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397420 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397420) get_ref 0x0~2000 Blob(0x558c57397420 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397420) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57397420 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x16000~2000: 0x0~2000 Blob(0x558c54cf82a0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397490 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397490) get_ref 0x0~2000 Blob(0x558c57397490 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397490) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c57397490 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x18000~2000: 0x0~2000 Blob(0x558c574d3030 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397500 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397500) get_ref 0x0~2000 Blob(0x558c57397500 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397500) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57397500 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397570 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397570) get_ref 0x0~2000 Blob(0x558c57397570 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397570) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57397570 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c573975e0 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573975e0) get_ref 0x0~2000 Blob(0x558c573975e0 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573975e0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c573975e0 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c574d3a40 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397650 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397650) get_ref 0x0~2000 Blob(0x558c57397650 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397650) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c57397650 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x20000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c573976c0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573976c0) get_ref 0x0~2000 Blob(0x558c573976c0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573976c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c573976c0 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57396cb0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57397730 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397730) get_ref 0x0~2000 Blob(0x558c57397730 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57397730) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57397730 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x24000~2000: 0x0~2000 Blob(0x558c575bc700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c573977a0 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573977a0) get_ref 0x0~2000 Blob(0x558c573977a0 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573977a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c573977a0 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x26000~2000: 0x0~2000 Blob(0x558c575bc620 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57efcf50 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efcf50) get_ref 0x0~2000 Blob(0x558c57efcf50 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efcf50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c57efcf50 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x28000~2000: 0x0~2000 Blob(0x558c57396d20 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57efcfc0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efcfc0) get_ref 0x0~2000 Blob(0x558c57efcfc0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efcfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c57efcfc0 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57efd030 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd030) get_ref 0x0~2000 Blob(0x558c57efd030 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd030) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c57efd030 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c57396fc0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57efd0a0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd0a0) get_ref 0x0~2000 Blob(0x558c57efd0a0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c57efd0a0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c57397030 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup new Blob(0x558c57efd110 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd110) get_ref 0x0~2000 Blob(0x558c57efd110 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd110) get_ref init 0x2000, 1000 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c57efd110 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=3)))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12888) dirty_range mark inline shard dirty 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# -> 2#1:6ee4dfe7:::smithi13140796-47:head# = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000037.00000000000000000027' <- 0000000037.00000000000000000027 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5782 (1 ios * 4000 + 1782 bytes) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes 0x558c54ce3400,0x558c54e12780 shared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3508) update 2#1:6ee4dfe7:::smithi13140796-47:head#1b 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head#1b is 1230 (573 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12888) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12888) update inline shard 651 bytes from 24 extents 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1230 (573 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=3))) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'27 (0'0,37'27] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'26 crt=29'24 lcod 29'24 active mbc={} ps=[a~1]] op_applied version 37'27 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.619+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57efe160 finish 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3837 < min_flush_size 524288 2019-12-07T19:34:14.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 37/28 ECSubWriteReply(tid=92, last_complete=37'26, committed=1, applied=1)) v2 -- 0x558c575b8000 con 0x558c54c5bc00 2019-12-07T19:34:14.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3400 had 1 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12780 had 1 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3860 < min_flush_size 524288 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3860 < min_flush_size 524288 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc906c~f14 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc906c 0xc906c~f14 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9906c 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6c 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf80 and padding block with 0x80 2019-12-07T19:34:14.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 98 ==== MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=94, reqid=client.4540.0:296, at_version=37'28, trim_to=0'0, roll_forward_to=29'24)) v2 ==== 2341+0+0 (unknown 0 0 0) 0x558c54d85800 con 0x558c54c5bc00 2019-12-07T19:34:14.619+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c57efe9a0 prio 127 cost 0 latency 0.000025 epoch 37 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=94, reqid=client.4540.0:296, at_version=37'28, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:14.619+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=94, reqid=client.4540.0:296, at_version=37'28, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xc9f80 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000170511s (0.000000138s flush + 0.000170373s kv commit) 2019-12-07T19:34:14.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=94, reqid=client.4540.0:296, at_version=37'28, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=94, reqid=client.4540.0:296, at_version=37'28, trim_to=0'0, roll_forward_to=29'24)) v2) prio 127 cost 0 e37) pg 0x558c54d5c000 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 0x558c54ce3400,0x558c54e12780 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57efe9a0 prio 127 cost 0 latency 0.000077 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=94, reqid=client.4540.0:296, at_version=37'28, trim_to=0'0, roll_forward_to=29'24)) v2 pg pg[1.6s2( v 37'27 (0'0,37'27] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 lcod 37'26 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'27 (0'0,37'27] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 lcod 37'26 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=94, reqid=client.4540.0:296, at_version=37'28, trim_to=0'0, roll_forward_to=29'24)) v2 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'27 (0'0,37'27] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 lcod 37'26 active mbc={} ps=[a~1]] append_log log((0'0,37'27], crt=29'24) [37'28 (37'27) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:296 2019-12-07T19:34:14.617503+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'27 crt=29'24 lcod 37'26 active mbc={} ps=[a~1]] add_log_entry 37'28 (37'27) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:296 2019-12-07T19:34:14.617503+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'27 crt=29'24 lcod 37'26 active mbc={} ps=[a~1]] append_log approx pg log length = 28 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'27 crt=29'24 lcod 37'26 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.620+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'28, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730af00 seq 56 2019-12-07T19:34:14.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:14.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000037.00000000000000000028' <- 0000000037.00000000000000000028 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730af00 cost 5991 (1 ios * 4000 + 1991 bytes) 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730af00 onodes 0x558c54e12780 shared_blobs 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12888) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1230 (573 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730af00 allocated 0x[] released 0x[] 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 prepare 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730af00 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 io_done 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'27 crt=29'24 lcod 37'26 active mbc={} ps=[a~1]] op_applied version 37'28 2019-12-07T19:34:14.620+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57efe9a0 finish 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2361 < min_flush_size 524288 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12780 had 1 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2384 < min_flush_size 524288 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2384 < min_flush_size 524288 2019-12-07T19:34:14.620+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 37/28 ECSubWriteReply(tid=93, last_complete=37'27, committed=1, applied=1)) v2 -- 0x558c575b9e00 con 0x558c54c5bc00 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xc9f80~950 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xc9f80 0xc9f80~950 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x99f80 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf80 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8d0 and padding block with 0x730 2019-12-07T19:34:14.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xca8d0 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000136827s (0.000000114s flush + 0.000136713s kv commit) 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730af00> 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 kv_submitted 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730af00 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 finishing 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730af00 onodes 0x558c54e12780 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730af00 done 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730af00 [] 2019-12-07T19:34:14.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.620+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 37/28 ECSubWriteReply(tid=94, last_complete=37'28, committed=1, applied=1)) v2 -- 0x558c551aec00 con 0x558c54c5bc00 2019-12-07T19:34:14.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.626+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 161 ==== osd_map(37..37 src has 1..37) v4 ==== 633+0+0 (crc 0 0 0) 0x558c5524c480 con 0x558c54c64000 2019-12-07T19:34:14.626+0000 7f8f1199c700 20 osd.7 37 OSD::ms_dispatch: osd_map(37..37 src has 1..37) v4 2019-12-07T19:34:14.626+0000 7f8f1199c700 10 osd.7 37 do_waiters -- start 2019-12-07T19:34:14.626+0000 7f8f1199c700 10 osd.7 37 do_waiters -- finish 2019-12-07T19:34:14.626+0000 7f8f1199c700 20 osd.7 37 _dispatch 0x558c5524c480 osd_map(37..37 src has 1..37) v4 2019-12-07T19:34:14.626+0000 7f8f1199c700 3 osd.7 37 handle_osd_map epochs [37,37], i have 37, src has [1,37] 2019-12-07T19:34:14.626+0000 7f8f1199c700 10 osd.7 37 no new maps here, dropping 2019-12-07T19:34:14.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 162 ==== MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=162, reqid=client.4540.0:300, at_version=37'12, trim_to=0'0, roll_forward_to=28'10)) v2 ==== 113922+0+0 (crc 0 0 0) 0x558c54e40800 con 0x558c54c64000 2019-12-07T19:34:14.626+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57efedc0 prio 127 cost 0 latency 0.000036 epoch 37 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=162, reqid=client.4540.0:300, at_version=37'12, trim_to=0'0, roll_forward_to=28'10)) v2 2019-12-07T19:34:14.626+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=162, reqid=client.4540.0:300, at_version=37'12, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=162, reqid=client.4540.0:300, at_version=37'12, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=162, reqid=client.4540.0:300, at_version=37'12, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e37) pg 0x558c5500d000 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 37 dequeue_op 0x558c57efedc0 prio 127 cost 0 latency 0.000079 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=162, reqid=client.4540.0:300, at_version=37'12, trim_to=0'0, roll_forward_to=28'10)) v2 pg pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 37 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 36 -> 37 (as per caller) 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=162, reqid=client.4540.0:300, at_version=37'12, trim_to=0'0, roll_forward_to=28'10)) v2 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 snap_mapper.add_oid 1:0064fe1d:::smithi13140796-44:e c,d,e 2019-12-07T19:34:14.626+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:0064fe1d:::smithi13140796-44:e got.empty() 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 snap_mapper.set_snaps 1:0064fe1d:::smithi13140796-44:e c,d,e 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 snap_mapper.set_snaps set OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_000000000000000D_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 28'10 (0'0,28'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] append_log log((0'0,28'10], crt=28'10) [37'11 (28'10) clone 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:01.654004+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,37'12 (28'10) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:300 2019-12-07T19:34:14.625221+0000 0 ObjectCleanRegions clean_offsets: [0~737280,1179648~18446744073708371967], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'11 (0'0,37'11] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=28'10 crt=28'10 lcod 28'10 active mbc={} ps=[a~1]] add_log_entry 37'11 (28'10) clone 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:01.654004+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'12 (0'0,37'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=28'10 crt=28'10 lcod 28'10 active mbc={} ps=[a~1]] add_log_entry 37'12 (28'10) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:300 2019-12-07T19:34:14.625221+0000 0 ObjectCleanRegions clean_offsets: [0~737280,1179648~18446744073708371967], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'12 (0'0,37'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=28'10 crt=28'10 lcod 28'10 active mbc={} ps=[a~1]] append_log approx pg log length = 12 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'12 (0'0,37'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=28'10 crt=28'10 lcod 28'10 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.626+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.626+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c57ef8300 seq 26 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:0064fe1d:::smithi13140796-44:e# key 0x8380000000000000010064fe1d21736d'ithi13140796-44!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) r -2 v.len 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:e# 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1162 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.onode(0x558c575bb900).flush flush done 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# 0x0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:e# 0x0~2d000 -> 0x0~2d000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x0~2000: 0x0~2000 Blob(0x558c54cf9880 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c574fba40 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fba40) get_ref 0x0~2000 Blob(0x558c574fba40 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fba40) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574fba40 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c55279420 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c574fb9d0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb9d0) get_ref 0x0~2000 Blob(0x558c574fb9d0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c574fb9d0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c55279030 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c574fdd50 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fdd50) get_ref 0x0~2000 Blob(0x558c574fdd50 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fdd50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c574fdd50 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c55264310 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c574fb810 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb810) get_ref 0x0~2000 Blob(0x558c574fb810 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb810) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c574fb810 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c55264230 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c574fbab0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fbab0) get_ref 0x0~2000 Blob(0x558c574fbab0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c574fbab0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c574fbab0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c55264000 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57efdce0 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdce0) get_ref 0x0~2000 Blob(0x558c57efdce0 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdce0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57efdce0 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c55265260 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57efddc0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efddc0) get_ref 0x0~2000 Blob(0x558c57efddc0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efddc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57efddc0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57566000 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57efde30 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efde30) get_ref 0x0~2000 Blob(0x558c57efde30 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efde30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57efde30 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57566070 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57efdea0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdea0) get_ref 0x0~2000 Blob(0x558c57efdea0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdea0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57efdea0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c575660e0 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57efdf10 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdf10) get_ref 0x0~2000 Blob(0x558c57efdf10 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdf10) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57efdf10 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57566150 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57efdf80 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdf80) get_ref 0x0~2000 Blob(0x558c57efdf80 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57efdf80) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57efdf80 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c575661c0 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a000 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a000) get_ref 0x0~2000 Blob(0x558c57f3a000 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a000) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c57f3a000 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c57566230 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a070 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a070) get_ref 0x0~2000 Blob(0x558c57f3a070 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a070) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57f3a070 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c575662a0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a0e0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a0e0) get_ref 0x0~2000 Blob(0x558c57f3a0e0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57f3a0e0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57566310 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a150 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a150) get_ref 0x0~2000 Blob(0x558c57f3a150 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a150) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57f3a150 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c57566380 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a1c0 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a1c0) get_ref 0x0~2000 Blob(0x558c57f3a1c0 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c57f3a1c0 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x20000~2000: 0x0~2000 Blob(0x558c575663f0 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a230 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a230) get_ref 0x0~2000 Blob(0x558c57f3a230 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a230) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c57f3a230 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57566460 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a2a0 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a2a0) get_ref 0x0~2000 Blob(0x558c57f3a2a0 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57f3a2a0 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x24000~2000: 0x0~2000 Blob(0x558c575664d0 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a310 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a310) get_ref 0x0~2000 Blob(0x558c57f3a310 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a310) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c57f3a310 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x26000~2000: 0x0~2000 Blob(0x558c57566540 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a380 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a380) get_ref 0x0~2000 Blob(0x558c57f3a380 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a380) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c57f3a380 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x28000~2000: 0x0~2000 Blob(0x558c575665b0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a3f0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a3f0) get_ref 0x0~2000 Blob(0x558c57f3a3f0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c57f3a3f0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c57566620 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a460 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a460) get_ref 0x0~2000 Blob(0x558c57f3a460 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a460) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c57f3a460 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup src 0x2c000~1000: 0x0~1000 Blob(0x558c57566690 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=1)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup new Blob(0x558c57f3a4d0 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a4d0) get_ref 0x0~1000 Blob(0x558c57f3a4d0 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a4d0) get_ref init 0x1000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dup dst 0x2c000~1000: 0x0~1000 Blob(0x558c57f3a4d0 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57f34108) dirty_range mark inline shard dirty 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# -> 3#1:0064fe1d:::smithi13140796-44:e# = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# snapset 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# snapset = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# 1 keys 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# 1 keys = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# hinfo_key (42 bytes) 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x2d000~1b000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:0064fe1d:::smithi13140796-44:head# 0x2d000~1b000 - have 0x2d000 (184320) bytes fadvise_flags 0x0 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x2d000~1b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef8300 14 blobs 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1b000/1000,1b000,0 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x4fe000~1b000/1000,1b000,0 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x4fe000~1b000] 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a540 blob([0x4fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a5b0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a540) get_ref 0x0~2000 Blob(0x558c57f3a540 blob([0x4fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a5b0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a540) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c57f3a540 blob([0x4fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3a5b0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3a5c8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a620 blob([0x500000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a690 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a620) get_ref 0x0~2000 Blob(0x558c57f3a620 blob([0x500000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a690 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a620) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c57f3a620 blob([0x500000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3a690 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3a6a8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a700 blob([0x502000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a770 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a700) get_ref 0x0~2000 Blob(0x558c57f3a700 blob([0x502000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a770 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a700) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c57f3a700 blob([0x502000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3a770 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3a788 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a7e0 blob([0x504000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a850 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a7e0) get_ref 0x0~2000 Blob(0x558c57f3a7e0 blob([0x504000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a850 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c57f3a7e0 blob([0x504000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3a850 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3a868 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a8c0 blob([0x506000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a930 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a8c0) get_ref 0x0~2000 Blob(0x558c57f3a8c0 blob([0x506000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a930 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c57f3a8c0 blob([0x506000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3a930 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3a948 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3aa10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a9a0 blob([0x508000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3aa10 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a9a0) get_ref 0x0~2000 Blob(0x558c57f3a9a0 blob([0x508000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3aa10 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3a9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c57f3a9a0 blob([0x508000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3aa10 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3aa28 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3aa80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3aaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3aa80 blob([0x50a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3aaf0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3aa80) get_ref 0x0~2000 Blob(0x558c57f3aa80 blob([0x50a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3aaf0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3aa80) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c57f3aa80 blob([0x50a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3aaf0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3ab08 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3ab60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3abd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3ab60 blob([0x50c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3abd0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ab60) get_ref 0x0~2000 Blob(0x558c57f3ab60 blob([0x50c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3abd0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ab60) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c57f3ab60 blob([0x50c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3abd0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3abe8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3ac40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3acb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3ac40 blob([0x50e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3acb0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ac40) get_ref 0x0~2000 Blob(0x558c57f3ac40 blob([0x50e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3acb0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ac40) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c57f3ac40 blob([0x50e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3acb0 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3acc8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3ad20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3ad90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3ad20 blob([0x510000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3ad90 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ad20) get_ref 0x0~2000 Blob(0x558c57f3ad20 blob([0x510000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3ad90 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ad20) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c57f3ad20 blob([0x510000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3ad90 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3ada8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3ae00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3ae70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3ae00 blob([0x512000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3ae70 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ae00) get_ref 0x0~2000 Blob(0x558c57f3ae00 blob([0x512000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3ae70 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3ae00) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c57f3ae00 blob([0x512000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3ae70 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3ae88 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3aee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3af50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3aee0 blob([0x514000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3af50 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3aee0) get_ref 0x0~2000 Blob(0x558c57f3aee0 blob([0x514000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3af50 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3aee0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c57f3aee0 blob([0x514000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3af50 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3af68 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3afc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3afc0 blob([0x516000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b030 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3afc0) get_ref 0x0~2000 Blob(0x558c57f3afc0 blob([0x516000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b030 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3afc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c57f3afc0 blob([0x516000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3b030 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3b048 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3b0a0 blob([!~1000,0x518000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b110 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3b0a0) get_ref 0x1000~1000 Blob(0x558c57f3b0a0 blob([!~1000,0x518000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b110 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3b0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~1000: 0x1000~1000 Blob(0x558c57f3b0a0 blob([!~1000,0x518000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57f3b110 sbid 0x0)) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3b128 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x48000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dirty_range mark inline shard dirty 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x2d000~1b000 = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- SNA_1_000000000000000C_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- SNA_1_000000000000000D_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- SNA_1_000000000000000E_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000037.00000000000000000011' <- 0000000037.00000000000000000011 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000037.00000000000000000012' <- 0000000037.00000000000000000012 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8300 cost 175285 (15 ios * 4000 + 115285 bytes) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8300 onodes 0x558c575bb900,0x558c57f34000 shared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) update inline shard 901 bytes from 37 extents 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 1480 (573 bytes onode + 2 bytes spanning blobs + 905 bytes inline extents) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57f34108) update 3#1:0064fe1d:::smithi13140796-44:e# 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57f34108) update inline shard 621 bytes from 23 extents 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:e# is 1039 (412 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2))) 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8300 allocated 0x[4fe000~1b000] released 0x[] 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 freelist allocate 0x4fe000~1b000 2019-12-07T19:34:14.627+0000 7f8f0898a700 20 freelist _xor first_key 0x480000 last_key 0x500000 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 prepare 2019-12-07T19:34:14.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef8300 2019-12-07T19:34:14.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.628+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'12 (0'0,37'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=28'10 crt=28'10 lcod 28'10 active mbc={} ps=[a~1]] op_applied version 37'12 2019-12-07T19:34:14.628+0000 7f8f0898a700 10 osd.7 37 dequeue_op 0x558c57efedc0 finish 2019-12-07T19:34:14.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.628+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 aio_wait 2019-12-07T19:34:14.628+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8300 2019-12-07T19:34:14.628+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 io_done 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6049 < min_flush_size 524288 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34000 had 1 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6072 < min_flush_size 524288 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6072 < min_flush_size 524288 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xca8d0~17b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xca8d0 0xca8d0~17b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9a8d0 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8d0 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x88 and padding block with 0xf78 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xcc088 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000235763s (0.000049125s flush + 0.000186638s kv commit) 2019-12-07T19:34:14.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8300> 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 kv_submitted 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8300 2019-12-07T19:34:14.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 finishing 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8300 onodes 0x558c575bb900,0x558c57f34000 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3a5c8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dba240 space 0x558c57f3a5c8 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3a6a8 in 0x558c53a59340) _finish_write discard buffer(0x558c573d9680 space 0x558c57f3a6a8 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3a788 in 0x558c53a59340) _finish_write discard buffer(0x558c573d95f0 space 0x558c57f3a788 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3a868 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8990 space 0x558c57f3a868 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3a948 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8870 space 0x558c57f3a948 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3aa28 in 0x558c53a59340) _finish_write discard buffer(0x558c573d8a20 space 0x558c57f3aa28 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3ab08 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64bd0 space 0x558c57f3ab08 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3abe8 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64b40 space 0x558c57f3abe8 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3acc8 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64ab0 space 0x558c57f3acc8 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3ada8 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64a20 space 0x558c57f3ada8 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3ae88 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64990 space 0x558c57f3ae88 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3af68 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64900 space 0x558c57f3af68 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3b048 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64750 space 0x558c57f3b048 0x0~2000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3b128 in 0x558c53a59340) _finish_write discard buffer(0x558c57e650e0 space 0x558c57f3b128 0x1000~1000 writing nocache) 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8300 done 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:14.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8300 [] 2019-12-07T19:34:14.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.628+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 37/18 ECSubWriteReply(tid=162, last_complete=37'12, committed=1, applied=1)) v2 -- 0x558c57568600 con 0x558c54c64000 2019-12-07T19:34:14.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 106 ==== MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 ==== 158902+0+0 (unknown 0 0 0) 0x558c54693000 con 0x558c54c5b400 2019-12-07T19:34:14.629+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57efef20 prio 127 cost 0 latency 0.000051 epoch 37 MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:14.629+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e37) pg 0x558c5513e000 2019-12-07T19:34:14.629+0000 7f8f07187700 10 osd.7 37 dequeue_op 0x558c57efef20 prio 127 cost 0 latency 0.000809 MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 pg pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.629+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:14.629+0000 7f8f07187700 20 snap_mapper.add_oid 1:c8ce3553:::smithi13140796-29:e c,d,e 2019-12-07T19:34:14.629+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.629+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:14.629+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.629+0000 7f8f07187700 20 snap_mapper.get_snaps 1:c8ce3553:::smithi13140796-29:e got.empty() 2019-12-07T19:34:14.629+0000 7f8f07187700 20 snap_mapper.set_snaps 1:c8ce3553:::smithi13140796-29:e c,d,e 2019-12-07T19:34:14.629+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.629+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.629+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.629+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.629+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'11 (0'0,37'11] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] append_log log((0'0,37'11], crt=23'10) [37'12 (21'6) clone 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:33:58.727310+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,37'13 (21'6) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:292 0.000000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.629+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'12 (0'0,37'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] add_log_entry 37'12 (21'6) clone 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:33:58.727310+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.629+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] add_log_entry 37'13 (21'6) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:292 0.000000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.629+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] append_log approx pg log length = 13 2019-12-07T19:34:14.629+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.629+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.629+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.629+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:14.629+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c548f7b00 seq 37 2019-12-07T19:34:14.629+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:c8ce3553:::smithi13140796-29:e# key 0x848000000000000001c8ce35'S!smithi13140796-29!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) r -2 v.len 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# -> 4#1:c8ce3553:::smithi13140796-29:e# 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1163 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.onode(0x558c57454000).flush flush done 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# -> 4#1:c8ce3553:::smithi13140796-29:e# 0x0~1d000 -> 0x0~1d000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10330 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c54d3c070 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c54d3c070 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57971ce0) get_ref 0x0~2000 Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57971ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10331 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c576ac3f0) get_ref 0x0~2000 Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c576ac3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10332 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c54d3c150 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c54d3c150 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c54cf81c0) get_ref 0x0~2000 Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c54cf81c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x6000~2000: 0x0~2000 Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10333 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c55265e30 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c55265e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57971e30) get_ref 0x0~2000 Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57971e30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x8000~2000: 0x0~2000 Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10334 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2070 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2070 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c576ac540) get_ref 0x0~2000 Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c576ac540) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0xa000~2000: 0x0~2000 Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10335 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2150 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2150 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3ba40) get_ref 0x0~2000 Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3ba40) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0xc000~2000: 0x0~2000 Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10336 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2230 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2230 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bb20) get_ref 0x0~2000 Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bb20) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0xe000~2000: 0x0~2000 Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10337 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2310 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2310 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bb90) get_ref 0x0~2000 Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bb90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x10000~2000: 0x0~2000 Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10338 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e23f0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e23f0 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bc00) get_ref 0x0~2000 Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bc00) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10339 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e24d0 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e24d0 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bc70) get_ref 0x0~2000 Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bc70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x14000~2000: 0x0~2000 Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10340 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e25b0 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e25b0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bce0) get_ref 0x0~2000 Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bce0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x16000~2000: 0x0~2000 Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10341 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2690 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2690 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bd50) get_ref 0x0~2000 Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bd50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x18000~2000: 0x0~2000 Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10342 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2770 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2770 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bdc0) get_ref 0x0~2000 Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10343 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2850 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2850 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3be30) get_ref 0x0~2000 Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3be30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup src 0x1c000~1000: 0x0~1000 Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10344 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared Blob(0x558c574e2930 blob([0xfd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) make_blob_shared now Blob(0x558c574e2930 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup new Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bea0) get_ref 0x0~1000 Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bea0) get_ref init 0x1000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dup dst 0x1c000~1000: 0x0~1000 Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=2)))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) dirty_range mark inline shard dirty 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976608) dirty_range mark inline shard dirty 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# -> 4#1:c8ce3553:::smithi13140796-29:e# = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# snapset 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# snapset = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 1 keys 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 1 keys = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:c8ce3553:::smithi13140796-29:head#d key 0x848000000000000001c8ce35'S!smithi13140796-29!='0xfffffffffffffffe000000000000000d'o' 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) r -2 v.len 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# -> 4#1:c8ce3553:::smithi13140796-29:head#d 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# -> 4#1:c8ce3553:::smithi13140796-29:head#d = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1164 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 3 keys 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 3 keys = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x0~26000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:c8ce3553:::smithi13140796-29:head# 0x0~26000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~26000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7b00 19 blobs 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.630+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x26000/1000,26000,0 2019-12-07T19:34:14.630+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x519000~26000/1000,26000,0 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x519000~26000] 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3bf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48000 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bf80) get_ref 0x0~2000 Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48000 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bf80) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48018 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f480e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f480e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48070) get_ref 0x0~2000 Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f480e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48070) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f480f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f481c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f481c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48150) get_ref 0x0~2000 Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f481c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48150) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f481d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f482a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f482a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48230) get_ref 0x0~2000 Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f482a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48230) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f482b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48380 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48310) get_ref 0x0~2000 Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48380 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48310) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48398 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f483f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48460 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f483f0) get_ref 0x0~2000 Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48460 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f483f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48478 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f484d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48540 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f484d0) get_ref 0x0~2000 Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48540 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f484d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48558 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f485b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48620 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f485b0) get_ref 0x0~2000 Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48620 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f485b0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48638 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48700 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48690) get_ref 0x0~2000 Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48700 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48690) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48718 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f487e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f487e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48770) get_ref 0x0~2000 Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f487e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48770) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f487f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f488c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f488c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48850) get_ref 0x0~2000 Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f488c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48850) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f488d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f489a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f489a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48930) get_ref 0x0~2000 Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f489a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48930) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f489b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48a80 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 16 ==== osd_op(client.4540.0:305 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e37) v8 ==== 250+0+54 (crc 0 0 0) 0x558c54ce98c0 con 0x558c54c9f000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48a10) get_ref 0x0~2000 Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48a80 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48a10) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48a98 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57eff1e0 prio 63 cost 54 latency 0.000030 epoch 37 osd_op(client.4540.0:305 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48b60 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:305 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48af0) get_ref 0x0~2000 Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48b60 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48af0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48b78 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48c40 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48bd0) get_ref 0x0~2000 Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48c40 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48c58 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48d20 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 17 ==== osd_op(client.4540.0:306 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 ==== 226+0+0 (crc 0 0 0) 0x558c57f12580 con 0x558c54c9f000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48cb0) get_ref 0x0~2000 Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48d20 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57eff4a0 prio 63 cost 0 latency 0.000012 epoch 37 osd_op(client.4540.0:306 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48d38 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48e00 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48d90) get_ref 0x0~2000 Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48e00 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48d90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48e18 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48ee0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48e70) get_ref 0x0~2000 Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48ee0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48e70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48ef8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f48f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f48fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48fc0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48f50) get_ref 0x0~2000 Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f48fc0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.blob(0x558c57f48f50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48fd8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x26000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976888) dirty_range mark inline shard dirty 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x0~26000 = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000C_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000E_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:305 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) queued 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000037.00000000000000000012' <- 0000000037.00000000000000000012 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000037.00000000000000000013' <- 0000000037.00000000000000000013 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:305 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e37) v8) prio 63 cost 54 e37) pg 0x558c548ab000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 240265 (20 ios * 4000 + 160265 bytes) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes 0x558c57454000,0x558c57976500,0x558c57976780 shared_blobs 0x558c54d3c310,0x558c54d3c4d0,0x558c574e0c40,0x558c574e2000,0x558c574e20e0,0x558c574e21c0,0x558c574e22a0,0x558c574e2380,0x558c574e2460,0x558c574e2540,0x558c574e2620,0x558c574e2700,0x558c574e27e0,0x558c574e28c0,0x558c574e29a0 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) update 4#1:c8ce3553:::smithi13140796-29:head#d 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57454108) update inline shard 405 bytes from 15 extents 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57eff1e0 prio 63 cost 54 latency 0.000149 osd_op(client.4540.0:305 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e37) v8 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:head#d is 896 (485 bytes onode + 2 bytes spanning blobs + 409 bytes inline extents) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976608) update 4#1:c8ce3553:::smithi13140796-29:e# 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976608) update inline shard 405 bytes from 15 extents 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:e# is 823 (412 bytes onode + 2 bytes spanning blobs + 409 bytes inline extents) 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:305 1.4s0 1.268ab6e4 (undecoded) ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976888) update 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976888) update inline shard 364 bytes from 19 extents 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:head# is 927 (557 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285b is 11 SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285c is 11 SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285a is 11 SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285d is 11 SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285e is 11 SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285f is 11 SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2860 is 11 SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2861 is 11 SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2862 is 11 SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2863 is 11 SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2864 is 11 SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2865 is 11 SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2866 is 11 SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2867 is 11 SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2868 is 11 SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=2))) 2019-12-07T19:34:14.630+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[519000~26000] released 0x[] 2019-12-07T19:34:14.630+0000 7f8f07187700 10 freelist allocate 0x519000~26000 2019-12-07T19:34:14.630+0000 7f8f07187700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:14.630+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f7b00 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: 0=[]:{} 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:34:14.630+0000 7f8f0297e700 25 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op oi 1:276d5164:::smithi13140796-22:head(21'2 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op obc obc(1:276d5164:::smithi13140796-22:head rwstate(write n=1 w=0)) 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] execute_ctx 0x558c576e0a00 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] execute_ctx 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] ov 21'2 av 37'3 snapc e=[e,d,c] snapset 0=[]:{} 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_osd_op 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_osd_op copy-from ver 4 in=54b 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] start_copy 1:276d5164:::smithi13140796-22:head from 1:6c205177:::smithi13140796-8:head @1 v4 flags 0 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _copy_some obc(1:276d5164:::smithi13140796-22:head rwstate(write n=1 w=0)) 0x558c5751cc10 2019-12-07T19:34:14.630+0000 7f8f0297e700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6810/37076,v1:172.21.15.131:6811/37076] conn(0x558c57d7f800 0x558c57600680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:14.630+0000 7f8f0297e700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6810/37076,v1:172.21.15.131:6811/37076] -- osd_op(unknown.0.13:2 1.6s0 1:6c205177:::smithi13140796-8:head [assert-version v4,copy-get max 8388608 in=29b] snapc 0=[] ondisk+read+known_if_redirected e37) v8 -- 0x558c57f13600 con 0x558c57d7f800 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:14.630+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57eff1e0 finish 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) queued 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.631+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57eff4a0 prio 63 cost 0 latency 0.000357 osd_op(client.4540.0:306 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 pg pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:306 1.4s0 1.268ab6e4 (undecoded) ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.631+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.631+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] op_applied version 37'13 2019-12-07T19:34:14.631+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.631+0000 7f8f07187700 10 osd.7 37 dequeue_op 0x558c57efef20 finish 2019-12-07T19:34:14.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:14.631+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] wait_for_blocked_object 1:276d5164:::smithi13140796-22:head 0x558c57eff4a0 2019-12-07T19:34:14.631+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57eff4a0 finish 2019-12-07T19:34:14.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.631+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 aio_wait 2019-12-07T19:34:14.631+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:14.631+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5719 < min_flush_size 524288 2019-12-07T19:34:14.631+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6810/37076,v1:172.21.15.131:6811/37076] conn(0x558c57d7f800 0x558c57600680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454000 had 1 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976500 had 1 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976780 had 1 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5742 < min_flush_size 524288 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5742 < min_flush_size 524288 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xcc088~166e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xcc088 0xcc088~166e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9c088 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x88 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6f6 and padding block with 0x90a 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xcd6f6 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000172308s (0.000041273s flush + 0.000131035s kv commit) 2019-12-07T19:34:14.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7b00> 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:14.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 0x558c57454000,0x558c57976500,0x558c57976780 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48018 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29680 space 0x558c57f48018 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f480f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29710 space 0x558c57f480f8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f481d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e297a0 space 0x558c57f481d8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f482b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29830 space 0x558c57f482b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48398 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e298c0 space 0x558c57f48398 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48478 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29950 space 0x558c57f48478 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48558 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e299e0 space 0x558c57f48558 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48638 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29a70 space 0x558c57f48638 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48718 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29b00 space 0x558c57f48718 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f487f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29b90 space 0x558c57f487f8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f488d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29c20 space 0x558c57f488d8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f489b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29cb0 space 0x558c57f489b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48a98 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29d40 space 0x558c57f48a98 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48b78 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29dd0 space 0x558c57f48b78 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48c58 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29e60 space 0x558c57f48c58 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 37/28 ECSubWriteReply(tid=60, last_complete=37'13, committed=1, applied=1)) v2 -- 0x558c54d76000 con 0x558c54c5b400 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48d38 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e29ef0 space 0x558c57f48d38 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48e18 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e65320 space 0x558c57f48e18 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48ef8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e653b0 space 0x558c57f48ef8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f48fd8 in 0x558c53a596c0) _finish_write discard buffer(0x558c57e65440 space 0x558c57f48fd8 0x0~2000 writing nocache) 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:14.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:14.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.631+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6810/37076,v1:172.21.15.131:6811/37076] conn(0x558c57d7f800 0x558c57600680 crc :-1 s=READY pgs=21 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:14.631+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 99 ==== MOSDECSubOpRead(1.6s2 37/28 ECSubRead(tid=95, to_read={1:6c205177:::smithi13140796-8:head=0,77824,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (unknown 0 0 0) 0x558c575bb400 con 0x558c54c5bc00 2019-12-07T19:34:14.632+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c57effe40 prio 127 cost 0 latency 0.000024 epoch 37 MOSDECSubOpRead(1.6s2 37/28 ECSubRead(tid=95, to_read={1:6c205177:::smithi13140796-8:head=0,77824,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.632+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 37/28 ECSubRead(tid=95, to_read={1:6c205177:::smithi13140796-8:head=0,77824,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 37/28 ECSubRead(tid=95, to_read={1:6c205177:::smithi13140796-8:head=0,77824,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 37/28 ECSubRead(tid=95, to_read={1:6c205177:::smithi13140796-8:head=0,77824,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) pg 0x558c54d5c000 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57effe40 prio 127 cost 0 latency 0.000071 MOSDECSubOpRead(1.6s2 37/28 ECSubRead(tid=95, to_read={1:6c205177:::smithi13140796-8:head=0,77824,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.6s2 37/28 ECSubRead(tid=95, to_read={1:6c205177:::smithi13140796-8:head=0,77824,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.632+0000 7f8f0197c700 25 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:14.632+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~13000 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~13000 size 0x13000 (77824) 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~13000 = 77824 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] handle_sub_read read request=77824 r=77824 len=77824 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] get_hash_info: not in cache 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:14.632+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key = 0 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:14.632+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 37/28 ECSubReadReply(tid=95, attrs_read=0)) v2 -- 0x558c5793bb80 con 0x558c54c5bc00 2019-12-07T19:34:14.632+0000 7f8f0197c700 10 osd.7 37 dequeue_op 0x558c57effe40 finish 2019-12-07T19:34:14.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 107 ==== MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c57dfe000 con 0x558c54c5b400 2019-12-07T19:34:14.632+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57eff600 prio 127 cost 0 latency 0.000018 epoch 37 MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 2019-12-07T19:34:14.632+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2) prio 127 cost 0 e37) pg 0x558c5513e000 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 osd.7 37 dequeue_op 0x558c57eff600 prio 127 cost 0 latency 0.000081 MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 pg pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.3s4 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] append_log log((0'0,37'13], crt=23'10) [] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] rollforward: entry=37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] rollforward: entry=37'12 (21'6) clone 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:33:58.727310+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] rollforward: entry=37'13 (21'6) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:292 0.000000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] append_log approx pg log length = 13 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.632+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c577fc000 seq 38 2019-12-07T19:34:14.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head#d onode 0x558c57454000 txc 0x558c577fc000 2019-12-07T19:34:14.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head#d 0x0 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c54d3c070) put_ref 0x0~2000 Blob(0x558c54d3c070 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c54d3c3f0) put_ref 0x0~2000 Blob(0x558c54d3c3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c54d3c150) put_ref 0x0~2000 Blob(0x558c54d3c150 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c55265e30) put_ref 0x0~2000 Blob(0x558c55265e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2070) put_ref 0x0~2000 Blob(0x558c574e2070 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2150) put_ref 0x0~2000 Blob(0x558c574e2150 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2230) put_ref 0x0~2000 Blob(0x558c574e2230 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2310) put_ref 0x0~2000 Blob(0x558c574e2310 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e23f0) put_ref 0x0~2000 Blob(0x558c574e23f0 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e24d0) put_ref 0x0~2000 Blob(0x558c574e24d0 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e25b0) put_ref 0x0~2000 Blob(0x558c574e25b0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2690) put_ref 0x0~2000 Blob(0x558c574e2690 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2770) put_ref 0x0~2000 Blob(0x558c574e2770 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2850) put_ref 0x0~2000 Blob(0x558c574e2850 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2930) put_ref 0x0~1000 Blob(0x558c574e2930 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57454108) dirty_range mark inline shard dirty 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d3c070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe1000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d3c3f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe3000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54d3c150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe5000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c55265e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe7000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c574e2070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe9000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c574e2150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xeb000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c574e2230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xed000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c574e2310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xef000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c574e23f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf1000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c574e24d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf3000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c574e25b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf5000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c574e2690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf7000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c574e2770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf9000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c574e2850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xfb000~2000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~1000: 0x0~1000 Blob(0x558c574e2930 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=2)))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xfd000~1000] 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3c310,0x558c54d3c4d0,0x558c574e0c40,0x558c574e2000,0x558c574e20e0,0x558c574e21c0,0x558c574e22a0,0x558c574e2380,0x558c574e2460,0x558c574e2540,0x558c574e2620,0x558c574e2700,0x558c574e27e0,0x558c574e28c0,0x558c574e29a0 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c57976780 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head#d = 0 2019-12-07T19:34:14.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc000 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc000 onodes shared_blobs 0x558c54d3c310,0x558c54d3c4d0,0x558c574e0c40,0x558c574e2000,0x558c574e20e0,0x558c574e21c0,0x558c574e22a0,0x558c574e2380,0x558c574e2460,0x558c574e2540,0x558c574e2620,0x558c574e2700,0x558c574e27e0,0x558c574e28c0,0x558c574e29a0 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285b is 11 SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285c is 11 SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285a is 11 SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285d is 11 SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285e is 11 SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285f is 11 SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2860 is 11 SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2861 is 11 SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2862 is 11 SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2863 is 11 SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2864 is 11 SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2865 is 11 SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2866 is 11 SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2867 is 11 SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2868 is 11 SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1))) 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc000 allocated 0x[] released 0x[] 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 prepare 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc000 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 io_done 2019-12-07T19:34:14.632+0000 7f8f0317f700 10 osd.7 37 dequeue_op 0x558c57eff600 finish 2019-12-07T19:34:14.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 163 ==== MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=163, reqid=client.4540.0:301, at_version=37'13, trim_to=0'0, roll_forward_to=28'10)) v2 ==== 182728+0+0 (crc 0 0 0) 0x558c575c0800 con 0x558c54c64000 2019-12-07T19:34:14.633+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57eff760 prio 127 cost 0 latency 0.000029 epoch 37 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=163, reqid=client.4540.0:301, at_version=37'13, trim_to=0'0, roll_forward_to=28'10)) v2 2019-12-07T19:34:14.633+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=163, reqid=client.4540.0:301, at_version=37'13, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.633+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.633+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=163, reqid=client.4540.0:301, at_version=37'13, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.633+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.633+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=163, reqid=client.4540.0:301, at_version=37'13, trim_to=0'0, roll_forward_to=28'10)) v2) prio 127 cost 0 e37) pg 0x558c5500d000 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1581 < min_flush_size 524288 2019-12-07T19:34:14.633+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c57eff760 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=163, reqid=client.4540.0:301, at_version=37'13, trim_to=0'0, roll_forward_to=28'10)) v2 pg pg[1.0s3( v 37'12 (0'0,37'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:14.633+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'12 (0'0,37'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=163, reqid=client.4540.0:301, at_version=37'13, trim_to=0'0, roll_forward_to=28'10)) v2 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454000 had 1 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xcd6f6~644 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xcd6f6 0xcd6f6~644 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9d6f6 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6f6 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd3a and padding block with 0x2c6 2019-12-07T19:34:14.633+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'12 (0'0,37'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] append_log log((0'0,37'12], crt=28'10) [37'13 (37'12) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:301 2019-12-07T19:34:14.630662+0000 0 ObjectCleanRegions clean_offsets: [0~1179648,1900544~18446744073707651071], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.633+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'13 (0'0,37'13] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'12 crt=28'10 lcod 37'12 active mbc={} ps=[a~1]] add_log_entry 37'13 (37'12) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:301 2019-12-07T19:34:14.630662+0000 0 ObjectCleanRegions clean_offsets: [0~1179648,1900544~18446744073707651071], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.633+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'13 (0'0,37'13] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'12 crt=28'10 lcod 37'12 active mbc={} ps=[a~1]] append_log approx pg log length = 13 2019-12-07T19:34:14.633+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'13 (0'0,37'13] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'12 crt=28'10 lcod 37'12 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.633+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.633+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xcdd3a 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000111670s (0.000000124s flush + 0.000111546s kv commit) 2019-12-07T19:34:14.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc000> 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d58f00 seq 27 2019-12-07T19:34:14.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 kv_submitted 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc000 2019-12-07T19:34:14.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 finishing 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc000 onodes 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc000 done 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:14.633+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:14.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc000 [] 2019-12-07T19:34:14.633+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:14.633+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.633+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 37/28 ECSubWriteReply(tid=61, last_complete=37'13, committed=1, applied=1)) v2 -- 0x558c575b8600 con 0x558c54c5b400 2019-12-07T19:34:14.633+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x48000~2c000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:0064fe1d:::smithi13140796-44:head# 0x48000~2c000 - have 0x48000 (294912) bytes fadvise_flags 0x0 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x48000~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.633+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.633+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58f00 22 blobs 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.633+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2c000/1000,2c000,0 2019-12-07T19:34:14.633+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x53f000~2c000/1000,2c000,0 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x53f000~2c000] 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bdea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bdea0 blob([0x53f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567c70 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bdea0) get_ref 0x0~2000 Blob(0x558c575bdea0 blob([0x53f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567c70 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bdea0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c575bdea0 blob([0x53f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567c70 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57567c88 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3dce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55264150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3dce0 blob([0x541000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c54d3dce0) get_ref 0x0~2000 Blob(0x558c54d3dce0 blob([0x541000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55264150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c54d3dce0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c54d3dce0 blob([0x541000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55264150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c55264168 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bdf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552652d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bdf10 blob([0x543000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552652d0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bdf10) get_ref 0x0~2000 Blob(0x558c575bdf10 blob([0x543000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552652d0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bdf10) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c575bdf10 blob([0x543000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552652d0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c552652e8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bddc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bddc0 blob([0x545000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3570 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bddc0) get_ref 0x0~2000 Blob(0x558c575bddc0 blob([0x545000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54ea3570 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bddc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c575bddc0 blob([0x545000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3570 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54ea3588 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bdce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5523a9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bdce0 blob([0x547000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a9a0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bdce0) get_ref 0x0~2000 Blob(0x558c575bdce0 blob([0x547000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5523a9a0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c575bdce0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c575bdce0 blob([0x547000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5523a9a0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5523a9b8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2460 blob([0x549000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1d50 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574d2460) get_ref 0x0~2000 Blob(0x558c574d2460 blob([0x549000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1d50 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574d2460) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c574d2460 blob([0x549000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1d50 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c574f1d68 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f28c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2930 blob([0x54b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f28c0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2930) get_ref 0x0~2000 Blob(0x558c577f2930 blob([0x54b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f28c0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2930) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c577f2930 blob([0x54b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f28c0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f28d8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f27e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2850 blob([0x54d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f27e0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2850) get_ref 0x0~2000 Blob(0x558c577f2850 blob([0x54d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f27e0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2850) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c577f2850 blob([0x54d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f27e0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f27f8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2770 blob([0x54f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2700 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2770) get_ref 0x0~2000 Blob(0x558c577f2770 blob([0x54f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2700 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2770) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c577f2770 blob([0x54f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2700 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f2718 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f21c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f21c0 blob([0x551000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2230 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f21c0) get_ref 0x0~2000 Blob(0x558c577f21c0 blob([0x551000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2230 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f21c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c577f21c0 blob([0x551000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2230 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f2248 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f22a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f22a0 blob([0x553000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2310 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f22a0) get_ref 0x0~2000 Blob(0x558c577f22a0 blob([0x553000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2310 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f22a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~2000: 0x0~2000 Blob(0x558c577f22a0 blob([0x553000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2310 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f2328 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f23f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2380 blob([0x555000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f23f0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2380) get_ref 0x0~2000 Blob(0x558c577f2380 blob([0x555000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f23f0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c577f2380) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5e000~2000: 0x0~2000 Blob(0x558c577f2380 blob([0x555000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f23f0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f2408 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2070 blob([0x557000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574e2070) get_ref 0x0~2000 Blob(0x558c574e2070 blob([0x557000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574e2070) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x60000~2000: 0x0~2000 Blob(0x558c574e2070 blob([0x557000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c574e2168 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2230 blob([0x559000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2310 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574e2230) get_ref 0x0~2000 Blob(0x558c574e2230 blob([0x559000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2310 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574e2230) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x62000~2000: 0x0~2000 Blob(0x558c574e2230 blob([0x559000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2310 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c574e2328 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e23f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e24d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e23f0 blob([0x55b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e24d0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574e23f0) get_ref 0x0~2000 Blob(0x558c574e23f0 blob([0x55b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e24d0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c574e23f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x64000~2000: 0x0~2000 Blob(0x558c574e23f0 blob([0x55b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e24d0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c574e24e8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55265e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55265e30 blob([0x55d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c070 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c55265e30) get_ref 0x0~2000 Blob(0x558c55265e30 blob([0x55d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c070 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c55265e30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x66000~2000: 0x0~2000 Blob(0x558c55265e30 blob([0x55d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c070 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54d3c088 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c3f0 blob([0x55f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c54d3c3f0) get_ref 0x0~2000 Blob(0x558c54d3c3f0 blob([0x55f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c54d3c3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x68000~2000: 0x0~2000 Blob(0x558c54d3c3f0 blob([0x55f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c150 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c54d3c168 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bea2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bea460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bea2a0 blob([0x561000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea460 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57bea2a0) get_ref 0x0~2000 Blob(0x558c57bea2a0 blob([0x561000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea460 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57bea2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6a000~2000: 0x0~2000 Blob(0x558c57bea2a0 blob([0x561000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea460 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bea478 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396540 blob([0x563000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49c00 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57396540) get_ref 0x0~2000 Blob(0x558c57396540 blob([0x563000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49c00 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57396540) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6c000~2000: 0x0~2000 Blob(0x558c57396540 blob([0x563000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49c00 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57f49c18 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f49c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f49c70 blob([0x565000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49ce0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57f49c70) get_ref 0x0~2000 Blob(0x558c57f49c70 blob([0x565000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49ce0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57f49c70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6e000~2000: 0x0~2000 Blob(0x558c57f49c70 blob([0x565000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49ce0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57f49cf8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f49d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f49d50 blob([0x567000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49dc0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57f49d50) get_ref 0x0~2000 Blob(0x558c57f49d50 blob([0x567000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49dc0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57f49d50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x70000~2000: 0x0~2000 Blob(0x558c57f49d50 blob([0x567000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49dc0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57f49dd8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f49e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f49e30 blob([0x569000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49ea0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57f49e30) get_ref 0x0~2000 Blob(0x558c57f49e30 blob([0x569000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49ea0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.blob(0x558c57f49e30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x72000~2000: 0x0~2000 Blob(0x558c57f49e30 blob([0x569000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49ea0 sbid 0x0)) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57f49eb8 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x74000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) dirty_range mark inline shard dirty 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x48000~2c000 = 0 2019-12-07T19:34:14.633+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.633+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000037.00000000000000000013' <- 0000000037.00000000000000000013 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 274378 (23 ios * 4000 + 182378 bytes) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes 0x558c575bb900 shared_blobs 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update inline shard 1320 bytes from 59 extents 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore.extentmap(0x558c575bba08) reshard 0x[0,ffffffff) of 0 shards on 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard extent_avg 22, target 500, slop 100 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard new shard 0x0 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard new shard 0x2c000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard new shard 0x56000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard new [0x0(0x0 bytes),0x2c000(0x0 bytes),0x56000(0x0 bytes)] 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard old [] 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard fin [0x0(0x0 bytes),0x2c000(0x0 bytes),0x56000(0x0 bytes)] 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# force 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update shard 0x0 is 597 bytes (was 0) from 22 extents 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update shard 0x2c000 is 442 bytes (was 0) from 22 extents 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update shard 0x56000 is 290 bytes (was 0) from 15 extents 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 588 (586 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:14.633+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[53f000~2c000] released 0x[] 2019-12-07T19:34:14.633+0000 7f8f04982700 10 freelist allocate 0x53f000~2c000 2019-12-07T19:34:14.633+0000 7f8f04982700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:34:14.633+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58f00 2019-12-07T19:34:14.634+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'13 (0'0,37'13] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'12 crt=28'10 lcod 37'12 active mbc={} ps=[a~1]] op_applied version 37'13 2019-12-07T19:34:14.634+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c57eff760 finish 2019-12-07T19:34:14.634+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.634+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 aio_wait 2019-12-07T19:34:14.634+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:34:14.634+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3328 < min_flush_size 524288 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3351 < min_flush_size 524288 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3351 < min_flush_size 524288 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xcdd3a~d17 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xcdd3a 0xcdd3a~d17 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9dd3a 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd3a 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa51 and padding block with 0x5af 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xcea51 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000154268s (0.000035068s flush + 0.000119200s kv commit) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00> 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:34:14.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 0x558c575bb900 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c088 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08360 space 0x558c54d3c088 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c168 in 0x558c53a59340) _finish_write discard buffer(0x558c57a083f0 space 0x558c54d3c168 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54ea3588 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb050 space 0x558c54ea3588 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5523a9b8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb0e0 space 0x558c5523a9b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55264168 in 0x558c53a59340) _finish_write discard buffer(0x558c548de240 space 0x558c55264168 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552652e8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbafc0 space 0x558c552652e8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2168 in 0x558c53a59340) _finish_write discard buffer(0x558c57a081b0 space 0x558c574e2168 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2328 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08240 space 0x558c574e2328 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 37/18 ECSubWriteReply(tid=163, last_complete=37'13, committed=1, applied=1)) v2 -- 0x558c57568800 con 0x558c54c64000 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e24e8 in 0x558c53a59340) _finish_write discard buffer(0x558c57a082d0 space 0x558c574e24e8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1d68 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb170 space 0x558c574f1d68 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567c88 in 0x558c53a59340) _finish_write discard buffer(0x558c54c9a090 space 0x558c57567c88 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2248 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08000 space 0x558c577f2248 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2328 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08090 space 0x558c577f2328 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2408 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08120 space 0x558c577f2408 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2718 in 0x558c53a59340) _finish_write discard buffer(0x558c57e65ef0 space 0x558c577f2718 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f27f8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb290 space 0x558c577f27f8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f28d8 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbb200 space 0x558c577f28d8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bea478 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08480 space 0x558c57bea478 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49c18 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08510 space 0x558c57f49c18 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49cf8 in 0x558c53a59340) _finish_write discard buffer(0x558c57a085a0 space 0x558c57f49cf8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49dd8 in 0x558c53a59340) _finish_write discard buffer(0x558c57a08630 space 0x558c57f49dd8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49eb8 in 0x558c53a59340) _finish_write discard buffer(0x558c57a086c0 space 0x558c57f49eb8 0x0~2000 writing nocache) 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:14.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:34:14.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.634+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 164 ==== MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=164, reqid=client.4540.0:302, at_version=37'14, trim_to=0'0, roll_forward_to=37'12)) v2 ==== 14792+0+0 (crc 0 0 0) 0x558c575c1800 con 0x558c54c64000 2019-12-07T19:34:14.635+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57eff8c0 prio 127 cost 0 latency 0.000023 epoch 37 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=164, reqid=client.4540.0:302, at_version=37'14, trim_to=0'0, roll_forward_to=37'12)) v2 2019-12-07T19:34:14.635+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=164, reqid=client.4540.0:302, at_version=37'14, trim_to=0'0, roll_forward_to=37'12)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=164, reqid=client.4540.0:302, at_version=37'14, trim_to=0'0, roll_forward_to=37'12)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=164, reqid=client.4540.0:302, at_version=37'14, trim_to=0'0, roll_forward_to=37'12)) v2) prio 127 cost 0 e37) pg 0x558c5500d000 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 37 dequeue_op 0x558c57eff8c0 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=164, reqid=client.4540.0:302, at_version=37'14, trim_to=0'0, roll_forward_to=37'12)) v2 pg pg[1.0s3( v 37'13 (0'0,37'13] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'13 (0'0,37'13] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=164, reqid=client.4540.0:302, at_version=37'14, trim_to=0'0, roll_forward_to=37'12)) v2 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'13 (0'0,37'13] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=28'10 active mbc={} ps=[a~1]] append_log log((0'0,37'13], crt=28'10) [37'14 (37'13) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:302 2019-12-07T19:34:14.631058+0000 0 ObjectCleanRegions clean_offsets: [0~1900544,1949696~18446744073707601919], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.635+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6819/37078 1 ==== osd_op_reply(1 smithi13140796-7 [assert-version v8,copy-get max 8388608 out=508362b] v0'0 uv8 ondisk = 507904) v8 ==== 202+0+508362 (crc 0 0 0) 0x558c5738d680 con 0x558c57d7f000 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'13 crt=28'10 lcod 37'13 active mbc={} ps=[a~1]] add_log_entry 37'14 (37'13) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:302 2019-12-07T19:34:14.631058+0000 0 ObjectCleanRegions clean_offsets: [0~1900544,1949696~18446744073707601919], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'13 crt=37'12 lcod 37'13 active mbc={} ps=[a~1]] rollforward: entry=37'11 (28'10) clone 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:01.654004+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'13 crt=37'12 lcod 37'13 active mbc={} ps=[a~1]] rollforward: entry=37'12 (28'10) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:300 2019-12-07T19:34:14.625221+0000 0 ObjectCleanRegions clean_offsets: [0~737280,1179648~18446744073708371967], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'13 crt=37'12 lcod 37'13 active mbc={} ps=[a~1]] append_log approx pg log length = 14 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'13 crt=37'12 lcod 37'13 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.635+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] process_copy_chunk 1:3a5a3f5b:::smithi13140796-23:head tid 1 (0) Success 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c57ef8600 seq 28 2019-12-07T19:34:14.635+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:14.635+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.635+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x74000~3000 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 3#1:0064fe1d:::smithi13140796-44:head# 0x74000~3000 - have 0x74000 (475136) bytes fadvise_flags 0x0 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x74000~3000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef8600 2 blobs 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0x3000/1000,3000,0 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x56b000~3000/1000,3000,0 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x56b000~3000] 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fb880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fb880 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1880 sbid 0x0)) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb880) get_ref 0x0~2000 Blob(0x558c574fb880 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1880 sbid 0x0)) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb880) get_ref init 0x2000, 1000 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] process_copy_chunk got digest: rx data 0xc13735ee omap 0xffffffff, source: data 0xc13735ee omap 0xffffffff flags 1 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x74000~2000: 0x0~2000 Blob(0x558c574fb880 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1880 sbid 0x0)) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c574f1898 in 0x558c53a59340) _discard 0x0~2000 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] process_copy_chunk success; committing 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fb8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bdd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fb8f0 blob([0x56d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bdd50 sbid 0x0)) 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] process_copy_chunk complete r = (0) Success 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb8f0) get_ref 0x0~1000 Blob(0x558c574fb8f0 blob([0x56d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bdd50 sbid 0x0)) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.blob(0x558c574fb8f0) get_ref init 0x1000, 1000 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x76000~1000: 0x0~1000 Blob(0x558c574fb8f0 blob([0x56d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575bdd50 sbid 0x0)) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c575bdd68 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x77000 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) dirty_range mark shard 0x56000 dirty 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x74000~3000 = 0 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] execute_ctx 0x558c576dee00 2019-12-07T19:34:14.635+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.635+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] ov 29'18 av 37'23 snapc e=[e,d,c] snapset 4=[]:{} 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000037.00000000000000000014' <- 0000000037.00000000000000000014 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8600 cost 26442 (3 ios * 4000 + 14442 bytes) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8600 onodes 0x558c575bb900 shared_blobs 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) update shard 0x56000 is 325 bytes (was 290) from 17 extents 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_osd_op copy-from ver 8 in=54b 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 588 (586 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8600 allocated 0x[56b000~3000] released 0x[] 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 freelist allocate 0x56b000~3000 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] finish_copyfrom on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] finish_copyfrom: exists, removing 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 prepare 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef8600 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] fill_in_final_tx: writing directly to final object 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _write_copy_chunk 0x558c57560c10 0 attrs 507904 bytes 0 omap header bytes 0 omap data bytes 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] finish_copyfrom clearing omap flag on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head snapset=4=[]:{} snapc=e=[e,d,c] 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'13 crt=37'12 lcod 37'13 active mbc={} ps=[a~1]] op_applied version 37'14 2019-12-07T19:34:14.635+0000 7f8f0898a700 10 osd.7 37 dequeue_op 0x558c57eff8c0 finish 2019-12-07T19:34:14.635+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] got greedy write on clone_obc obc(1:3a5a3f5b:::smithi13140796-23:e rwstate(write n=1 w=0)) 2019-12-07T19:34:14.635+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 aio_wait 2019-12-07T19:34:14.635+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8600 2019-12-07T19:34:14.635+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 io_done 2019-12-07T19:34:14.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] cloning v 29'18 to 1:3a5a3f5b:::smithi13140796-23:e v 37'23 snaps=[e,d,c] snapset=4=[]:{e=[e,d,c]} 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head done, snapset=e=[]:{e=[e,d,c]} 2019-12-07T19:34:14.635+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] finish_ctx 1:3a5a3f5b:::smithi13140796-23:head 0x558c576dee00 op modify 2019-12-07T19:34:14.635+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] mtime unchanged at 2019-12-07T19:33:58.699945+0000 2019-12-07T19:34:14.636+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6810/37076 1 ==== osd_op_reply(2 smithi13140796-8 [assert-version v4,copy-get max 8388608 out=311649b] v0'0 uv4 ondisk = 311296) v8 ==== 202+0+311649 (crc 0 0 0) 0x558c5738d680 con 0x558c57d7f800 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] final snapset e=[]:{e=[e,d,c]} in 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2202 < min_flush_size 524288 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2225 < min_flush_size 524288 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2225 < min_flush_size 524288 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] finish_ctx object 1:3a5a3f5b:::smithi13140796-23:head marks clean_regions clean_offsets: [1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xcea51~8b1 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xcea51 0xcea51~8b1 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] finish_ctx extra_reqids [client.4540.0:229,8,client.4540.0:92,8,client.4540.0:91,7,client.4540.0:181,6,client.4540.0:62,6,client.4540.0:61,5,client.4540.0:20,3,client.4540.0:19,1] {} 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9ea51 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa51 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x302 and padding block with 0xcfe 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] execute_ctx alloc reply 0x558c57f35680 result 0 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] calc_trim_to_aggressive limit = 35'22 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] op order client.4540 tid 286 (first) 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] new_repop rep_tid 18 on osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] new_repop: repgather(0x558c55159c20 0'0 rep_tid=18 committed?=0 r=0) 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xcf302 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.636+0000 7f8f0b18f700 7 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] issue_repop rep_tid 18 o 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000155558s (0.000044343s flush + 0.000111215s kv commit) 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8600> 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 kv_submitted 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8600 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 finishing 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8600 onodes 0x558c575bb900 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1898 in 0x558c53a59340) _finish_write discard buffer(0x558c54e15830 space 0x558c574f1898 0x0~2000 writing nocache) 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:head v=37'24 tt=0'0 tid=18 reqid=client.4540.0:286 client_op=osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bdd68 in 0x558c53a59340) _finish_write discard buffer(0x558c54dbaf30 space 0x558c575bdd68 0x0~1000 writing nocache) 2019-12-07T19:34:14.636+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8600 done 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8600 [] 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:14.636+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 37/18 ECSubWriteReply(tid=164, last_complete=37'14, committed=1, applied=1)) v2 -- 0x558c54d76000 con 0x558c54c64000 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:e# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.636+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:14.636+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:e projected size 1474560 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=37'24 tt=0'0 tid=18 reqid=client.4540.0:286 client_op=osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[0~507904]}) 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_state_to_reads: invalidating cache after this op 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:head v=37'24 tt=0'0 tid=18 reqid=client.4540.0:286 client_op=osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[0~507904]}) 2019-12-07T19:34:14.636+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=37'24 tt=0'0 tid=18 reqid=client.4540.0:286 client_op=osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[0~507904]}) 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( Extents(1:3a5a3f5b:::smithi13140796-23:head)[ Extent(0~507904:3) ) 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:e, current size is 0 buffers are {} 2019-12-07T19:34:14.636+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 165 ==== MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=165, reqid=client.4540.0:303, at_version=37'15, trim_to=0'0, roll_forward_to=37'12)) v2 ==== 2470+0+0 (crc 0 0 0) 0x558c575be800 con 0x558c54c64000 2019-12-07T19:34:14.636+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57effa20 prio 127 cost 0 latency 0.000020 epoch 37 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=165, reqid=client.4540.0:303, at_version=37'15, trim_to=0'0, roll_forward_to=37'12)) v2 2019-12-07T19:34:14.636+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=165, reqid=client.4540.0:303, at_version=37'15, trim_to=0'0, roll_forward_to=37'12)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.636+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.636+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=165, reqid=client.4540.0:303, at_version=37'15, trim_to=0'0, roll_forward_to=37'12)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.636+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.636+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=165, reqid=client.4540.0:303, at_version=37'15, trim_to=0'0, roll_forward_to=37'12)) v2) prio 127 cost 0 e37) pg 0x558c5500d000 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c57effa20 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=165, reqid=client.4540.0:303, at_version=37'15, trim_to=0'0, roll_forward_to=37'12)) v2 pg pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'12 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): new_size start 1474560 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'12 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=165, reqid=client.4540.0:303, at_version=37'15, trim_to=0'0, roll_forward_to=37'12)) v2 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): to_append: {} 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:e resetting hinfo to logical size 1474560 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'14 (0'0,37'14] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'12 active mbc={} ps=[a~1]] append_log log((0'0,37'14], crt=37'12) [37'15 (37'14) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:303 2019-12-07T19:34:14.631074+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 1474560 buffers are {0~507904(507904)} 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'14 crt=37'12 lcod 37'14 active mbc={} ps=[a~1]] add_log_entry 37'15 (37'14) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:303 2019-12-07T19:34:14.631074+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'14 crt=37'12 lcod 37'14 active mbc={} ps=[a~1]] append_log approx pg log length = 15 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'14 crt=37'12 lcod 37'14 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.636+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): new_size start 0 2019-12-07T19:34:14.636+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'15, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): adding buffer_update 0,507904 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): to_append: {0~507904(507904)} 2019-12-07T19:34:14.636+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): appending 0~507904 2019-12-07T19:34:14.636+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d58600 seq 29 2019-12-07T19:34:14.636+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 3 keys 2019-12-07T19:34:14.636+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 3 keys = 0 2019-12-07T19:34:14.636+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.636+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.636+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000037.00000000000000000015' <- 0000000037.00000000000000000015 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.636+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:14.636+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58600 cost 6120 (1 ios * 4000 + 2120 bytes) 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58600 onodes 0x558c575bb900 shared_blobs 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 588 (586 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58600 allocated 0x[] released 0x[] 2019-12-07T19:34:14.636+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 prepare 2019-12-07T19:34:14.636+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58600 2019-12-07T19:34:14.636+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 io_done 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'14 crt=37'12 lcod 37'14 active mbc={} ps=[a~1]] op_applied version 37'15 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.636+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c57effa20 finish 2019-12-07T19:34:14.636+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1776 < min_flush_size 524288 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1799 < min_flush_size 524288 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1799 < min_flush_size 524288 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xcf302~707 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xcf302 0xcf302~707 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9f302 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x302 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa09 and padding block with 0x5f7 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xcfa09 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131684s (0.000000116s flush + 0.000131568s kv commit) 2019-12-07T19:34:14.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58600> 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 kv_submitted 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58600 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 finishing 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58600 onodes 0x558c575bb900 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 done 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58600 [] 2019-12-07T19:34:14.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.636+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.636+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 37/18 ECSubWriteReply(tid=165, last_complete=37'15, committed=1, applied=1)) v2 -- 0x558c57568800 con 0x558c54c64000 2019-12-07T19:34:14.636+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.636+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 100 ==== MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=96, reqid=client.4540.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=37'28)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54d85000 con 0x558c54c5bc00 2019-12-07T19:34:14.636+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c5740c160 prio 127 cost 0 latency 0.000025 epoch 37 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=96, reqid=client.4540.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=37'28)) v2 2019-12-07T19:34:14.636+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=96, reqid=client.4540.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=37'28)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=96, reqid=client.4540.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=37'28)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=96, reqid=client.4540.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=37'28)) v2) prio 127 cost 0 e37) pg 0x558c54d5c000 2019-12-07T19:34:14.637+0000 7f8f05984700 10 osd.7 37 dequeue_op 0x558c5740c160 prio 127 cost 0 latency 0.000274 MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=96, reqid=client.4540.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=37'28)) v2 pg pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.637+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 37/28 ECSubWrite(tid=96, reqid=client.4540.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=37'28)) v2 2019-12-07T19:34:14.637+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=29'24 active mbc={} ps=[a~1]] append_log log((0'0,37'28], crt=29'24) [] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] rollforward: entry=37'25 (29'24) clone 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:04.726001+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] rollforward: entry=37'26 (29'24) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:294 2019-12-07T19:34:14.617472+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] rollforward: entry=37'27 (37'26) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:295 2019-12-07T19:34:14.617498+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 0, new_object: 0 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] rollforward: entry=37'28 (37'27) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:296 2019-12-07T19:34:14.617503+0000 0 ObjectCleanRegions clean_offsets: [0~786432,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.637+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] append_log approx pg log length = 28 2019-12-07T19:34:14.637+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.637+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.637+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730b800 seq 57 2019-12-07T19:34:14.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#1b onode 0x558c54ce3400 txc 0x558c5730b800 2019-12-07T19:34:14.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#1b 0x0 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57880d90) put_ref 0x0~2000 Blob(0x558c57880d90 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578dfe30) put_ref 0x0~2000 Blob(0x558c578dfe30 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578f1ea0) put_ref 0x0~2000 Blob(0x558c578f1ea0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578f1f80) put_ref 0x0~2000 Blob(0x558c578f1f80 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578f1e30) put_ref 0x0~2000 Blob(0x558c578f1e30 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578f1dc0) put_ref 0x0~2000 Blob(0x558c578f1dc0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578f1d50) put_ref 0x0~2000 Blob(0x558c578f1d50 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881e30) put_ref 0x0~2000 Blob(0x558c57881e30 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881dc0) put_ref 0x0~2000 Blob(0x558c57881dc0 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881d50) put_ref 0x0~2000 Blob(0x558c57881d50 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881ce0) put_ref 0x0~2000 Blob(0x558c57881ce0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881c70) put_ref 0x0~2000 Blob(0x558c57881c70 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881c00) put_ref 0x0~2000 Blob(0x558c57881c00 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881b90) put_ref 0x0~2000 Blob(0x558c57881b90 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881b20) put_ref 0x0~2000 Blob(0x558c57881b20 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881ab0) put_ref 0x0~2000 Blob(0x558c57881ab0 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881a40) put_ref 0x0~2000 Blob(0x558c57881a40 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578819d0) put_ref 0x0~2000 Blob(0x558c578819d0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881960) put_ref 0x0~2000 Blob(0x558c57881960 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c578818f0) put_ref 0x0~2000 Blob(0x558c578818f0 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57880a80) put_ref 0x0~2000 Blob(0x558c57880a80 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57880a10) put_ref 0x0~2000 Blob(0x558c57880a10 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881ea0) put_ref 0x0~2000 Blob(0x558c57881ea0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.blob(0x558c57881f10) put_ref 0x0~2000 Blob(0x558c57881f10 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3508) dirty_range mark inline shard dirty 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57880d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22b000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c578dfe30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22d000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c578f1ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22f000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c578f1f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x231000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c578f1e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x233000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c578f1dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x235000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c578f1d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x237000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57881e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x239000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57881dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23b000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57881d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23d000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57881ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23f000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57881c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x241000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57881c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x243000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57881b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x245000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57881b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x247000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c57881ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x249000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57881a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24b000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c578819d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24d000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c57881960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24f000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c578818f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x251000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c57880a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x253000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c57880a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x255000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c57881ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x257000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c57881f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=3)))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x259000~2000] 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head#1b = 0 2019-12-07T19:34:14.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:14.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b800 onodes shared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2))) 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b800 allocated 0x[] released 0x[] 2019-12-07T19:34:14.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 prepare 2019-12-07T19:34:14.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b800 2019-12-07T19:34:14.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 io_done 2019-12-07T19:34:14.637+0000 7f8f05984700 10 osd.7 37 dequeue_op 0x558c5740c160 finish 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1527 < min_flush_size 524288 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1808 < min_flush_size 524288 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3400 had 1 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1831 < min_flush_size 524288 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1831 < min_flush_size 524288 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xcfa09~727 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xcfa09 0xcfa09~727 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:12.628722+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _allocate len 0x130 from 0 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _allocate len 0x130 from 1 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd200000 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd200000~10000/10000,10000,acd200000 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0xe0000 to include allocated 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 35 (was clean) 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0x9fa09 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa09 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x130 and padding block with 0xed0 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0xe0000 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd0130 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (35) on file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]), flushing log 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 35 len 0x49 crc 0x7fa3bb80) 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x22000~1000 to file(ino 1 size 0x22000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x22000 0x22000~1000 to file(ino 1 size 0x22000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x23000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x22000 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x23000 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:14.637+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] encode_and_write: 1:3a5a3f5b:::smithi13140796-23:head new_size 507904 2019-12-07T19:34:14.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000023 2019-12-07T19:34:14.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.638+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 35 2019-12-07T19:34:14.638+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000430714s (0.000000140s flush + 0.000430574s kv commit) 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b800> 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 kv_submitted 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b800 2019-12-07T19:34:14.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 finishing 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b800 onodes 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 done 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:14.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b800 [] 2019-12-07T19:34:14.638+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.638+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.638+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 37/28 ECSubWriteReply(tid=96, last_complete=37'28, committed=1, applied=1)) v2 -- 0x558c576b5200 con 0x558c54c5bc00 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] operator(): marking append 0 2019-12-07T19:34:14.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( Extents(1:3a5a3f5b:::smithi13140796-23:head)[ Extent(0~507904:3) ) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:e={},1:3a5a3f5b:::smithi13140796-23:head={0~507904(507904)}} 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=37'24 tt=0'0 tid=18 reqid=client.4540.0:286 client_op=osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[0~507904]}) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[0~507904]} 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:e,{} 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{0~507904(507904)} 2019-12-07T19:34:14.638+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 37/28 ECSubWrite(tid=18, reqid=client.4540.0:286, at_version=37'24, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c551dc000 con 0x558c54c64000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 37 send_incremental_map 35 -> 37 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:14.638+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c2eb972f:::inc_osdmap.36:0# 0x0~0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~279 size 0x279 (633) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d24d0 blob([0x4f8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x279) SharedBlob(0x558c5794d260 sbid 0x0)) need 0x0~279 cache has 0x[0~279] 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c2eb972f:::inc_osdmap.36:0# 0x0~279 = 633 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 37 add_map_inc_bl 36 633 bytes 2019-12-07T19:34:14.638+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c82b972f:::inc_osdmap.37:0# 0x0~0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57397b20 blob([0x4fb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c54677340 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c82b972f:::inc_osdmap.37:0# 0x0~24d = 589 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 37 add_map_inc_bl 37 589 bytes 2019-12-07T19:34:14.638+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(36..37 src has 1..37) v4 -- 0x558c57945200 con 0x558c54c19800 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 37 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 35 -> 37 (shared) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 37/28 ECSubWrite(tid=18, reqid=client.4540.0:286, at_version=37'24, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c5748f000 con 0x558c54c19800 2019-12-07T19:34:14.638+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 37/28 ECSubWrite(tid=18, reqid=client.4540.0:286, at_version=37'24, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c5748d000 con 0x558c54c5bc00 2019-12-07T19:34:14.638+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 37/28 ECSubWrite(tid=18, reqid=client.4540.0:286, at_version=37'24, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c575c0800 con 0x558c54c5b400 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 37 send_incremental_map 35 -> 37 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:14.638+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(36..37 src has 1..37) v4 -- 0x558c57570480 con 0x558c54c19c00 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 37 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 35 -> 37 (shared) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 37/28 ECSubWrite(tid=18, reqid=client.4540.0:286, at_version=37'24, trim_to=0'0, roll_forward_to=35'22)) v2 -- 0x558c575c1800 con 0x558c54c19c00 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 snap_mapper.add_oid 1:3a5a3f5b:::smithi13140796-23:e c,d,e 2019-12-07T19:34:14.638+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:e got.empty() 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 snap_mapper.set_snaps 1:3a5a3f5b:::smithi13140796-23:e c,d,e 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 earliest_dup_version = 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 trim 37'23 (29'18) clone 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 trim 37'24 (29'18) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:286 0.000000 0 ObjectCleanRegions clean_offsets: [1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 35'22 (0'0,35'22] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] append_log log((0'0,35'22], crt=35'22) [37'23 (29'18) clone 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,37'24 (29'18) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:286 0.000000 0 ObjectCleanRegions clean_offsets: [1474560~18446744073708077055], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'23 (0'0,37'23] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 lua=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] add_log_entry 37'23 (29'18) clone 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 lua=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] add_log_entry 37'24 (29'18) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:286 0.000000 0 ObjectCleanRegions clean_offsets: [1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 lua=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] append_log approx pg log length = 24 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 lua=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'23, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57ef8000 seq 30 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:e# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1165 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.onode(0x558c574a1180).flush flush done 2019-12-07T19:34:14.638+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 0x0 2019-12-07T19:34:14.638+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:e# 0x0~5a000 -> 0x0~5a000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x0~2000: 0x0~2000 Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4a10 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4a10) get_ref 0x0~2000 Blob(0x558c57ff4a10 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4a10) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57ff4a10 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x2000~2000: 0x0~2000 Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff49a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff49a0) get_ref 0x0~2000 Blob(0x558c57ff49a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff49a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57ff49a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x4000~2000: 0x0~2000 Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff48c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff48c0) get_ref 0x0~2000 Blob(0x558c57ff48c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff48c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57ff48c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x6000~2000: 0x0~2000 Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4930 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4930) get_ref 0x0~2000 Blob(0x558c57ff4930 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4930) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57ff4930 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x8000~2000: 0x0~2000 Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4a80 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4a80) get_ref 0x0~2000 Blob(0x558c57ff4a80 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4a80) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57ff4a80 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0xa000~2000: 0x0~2000 Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4af0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4af0) get_ref 0x0~2000 Blob(0x558c57ff4af0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4af0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57ff4af0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0xc000~2000: 0x0~2000 Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4bd0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4bd0) get_ref 0x0~2000 Blob(0x558c57ff4bd0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57ff4bd0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0xe000~2000: 0x0~2000 Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4c40 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4c40) get_ref 0x0~2000 Blob(0x558c57ff4c40 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4c40) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57ff4c40 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4cb0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4cb0) get_ref 0x0~2000 Blob(0x558c57ff4cb0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57ff4cb0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4d20 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4d20) get_ref 0x0~2000 Blob(0x558c57ff4d20 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4d20) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57ff4d20 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x14000~2000: 0x0~2000 Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff4d90 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4d90) get_ref 0x0~2000 Blob(0x558c57ff4d90 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4d90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57ff4d90 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x16000~1000: 0x0~1000 Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff50a0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff50a0) get_ref 0x0~1000 Blob(0x558c57ff50a0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff50a0) get_ref init 0x1000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x16000~1000: 0x0~1000 Blob(0x558c57ff50a0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x17000~2000: 0x0~2000 Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10345 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d3b0 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d3b0 blob([0x377000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881ce0 blob([0x377000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881ce0) get_ref 0x0~2000 Blob(0x558c57881ce0 blob([0x377000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x17000~2000: 0x0~2000 Blob(0x558c57881ce0 blob([0x377000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x19000~2000: 0x0~2000 Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10346 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d490 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d490 blob([0x379000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881c70 blob([0x379000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881c70) get_ref 0x0~2000 Blob(0x558c57881c70 blob([0x379000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881c70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x19000~2000: 0x0~2000 Blob(0x558c57881c70 blob([0x379000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x1b000~2000: 0x0~2000 Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10347 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d570 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d570 blob([0x37b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881c00 blob([0x37b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881c00) get_ref 0x0~2000 Blob(0x558c57881c00 blob([0x37b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881c00) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x1b000~2000: 0x0~2000 Blob(0x558c57881c00 blob([0x37b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x1d000~2000: 0x0~2000 Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10348 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d650 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d650 blob([0x37d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881b90 blob([0x37d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881b90) get_ref 0x0~2000 Blob(0x558c57881b90 blob([0x37d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881b90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x1d000~2000: 0x0~2000 Blob(0x558c57881b90 blob([0x37d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x1f000~2000: 0x0~2000 Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10349 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d730 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d730 blob([0x37f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881b20 blob([0x37f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881b20) get_ref 0x0~2000 Blob(0x558c57881b20 blob([0x37f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881b20) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x1f000~2000: 0x0~2000 Blob(0x558c57881b20 blob([0x37f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x21000~2000: 0x0~2000 Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10350 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d810 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d810 blob([0x381000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881ab0 blob([0x381000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881ab0) get_ref 0x0~2000 Blob(0x558c57881ab0 blob([0x381000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x21000~2000: 0x0~2000 Blob(0x558c57881ab0 blob([0x381000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x23000~2000: 0x0~2000 Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10351 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d8f0 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d8f0 blob([0x383000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=1)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881a40 blob([0x383000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881a40) get_ref 0x0~2000 Blob(0x558c57881a40 blob([0x383000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881a40) get_ref init 0x2000, 1000 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c57881a40 blob([0x383000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=2)))) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x25000~2000: 0x0~2000 Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) 2019-12-07T19:34:14.638+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10352 2019-12-07T19:34:14.638+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d9d0 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d9d0 blob([0x385000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578819d0 blob([0x385000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578819d0) get_ref 0x0~2000 Blob(0x558c578819d0 blob([0x385000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578819d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c578819d0 blob([0x385000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x27000~2000: 0x0~2000 Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10353 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794dab0 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794dab0 blob([0x387000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881960 blob([0x387000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881960) get_ref 0x0~2000 Blob(0x558c57881960 blob([0x387000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881960) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c57881960 blob([0x387000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x29000~2000: 0x0~2000 Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10354 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794db90 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794db90 blob([0x389000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578818f0 blob([0x389000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578818f0) get_ref 0x0~2000 Blob(0x558c578818f0 blob([0x389000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578818f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c578818f0 blob([0x389000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10355 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794dc70 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794dc70 blob([0x38b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57880a80 blob([0x38b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880a80) get_ref 0x0~2000 Blob(0x558c57880a80 blob([0x38b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880a80) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c57880a80 blob([0x38b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10356 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794dd50 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794dd50 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57880a10 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880a10) get_ref 0x0~2000 Blob(0x558c57880a10 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880a10) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c57880a10 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x2f000~2000: 0x0~2000 Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10357 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794de30 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794de30 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881ea0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881ea0) get_ref 0x0~2000 Blob(0x558c57881ea0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x2f000~2000: 0x0~2000 Blob(0x558c57881ea0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x31000~2000: 0x0~2000 Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10358 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794df10 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794df10 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881f10 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881f10) get_ref 0x0~2000 Blob(0x558c57881f10 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881f10) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x31000~2000: 0x0~2000 Blob(0x558c57881f10 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x33000~2000: 0x0~2000 Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10359 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c000 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c000 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57880d90 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880d90) get_ref 0x0~2000 Blob(0x558c57880d90 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880d90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x33000~2000: 0x0~2000 Blob(0x558c57880d90 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x35000~2000: 0x0~2000 Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10360 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c0e0 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c0e0 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57880cb0 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880cb0) get_ref 0x0~2000 Blob(0x558c57880cb0 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57880cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x35000~2000: 0x0~2000 Blob(0x558c57880cb0 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x37000~2000: 0x0~2000 Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10361 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c1c0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c1c0 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881e30 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881e30) get_ref 0x0~2000 Blob(0x558c57881e30 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881e30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x37000~2000: 0x0~2000 Blob(0x558c57881e30 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x39000~2000: 0x0~2000 Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10362 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c2a0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c2a0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881dc0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881dc0) get_ref 0x0~2000 Blob(0x558c57881dc0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x39000~2000: 0x0~2000 Blob(0x558c57881dc0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x3b000~2000: 0x0~2000 Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10363 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c380 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c380 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57881d50 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881d50) get_ref 0x0~2000 Blob(0x558c57881d50 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57881d50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x3b000~2000: 0x0~2000 Blob(0x558c57881d50 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x3d000~2000: 0x0~2000 Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10364 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c460 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c460 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578f1ea0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1ea0) get_ref 0x0~2000 Blob(0x558c578f1ea0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x3d000~2000: 0x0~2000 Blob(0x558c578f1ea0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x3f000~2000: 0x0~2000 Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10365 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c540 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c540 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578f1f80 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1f80) get_ref 0x0~2000 Blob(0x558c578f1f80 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1f80) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x3f000~2000: 0x0~2000 Blob(0x558c578f1f80 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x41000~2000: 0x0~2000 Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10366 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792c620 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792c620 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578f1e30 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1e30) get_ref 0x0~2000 Blob(0x558c578f1e30 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1e30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x41000~2000: 0x0~2000 Blob(0x558c578f1e30 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x43000~2000: 0x0~2000 Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10367 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794d030 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794d030 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578f1dc0 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1dc0) get_ref 0x0~2000 Blob(0x558c578f1dc0 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x43000~2000: 0x0~2000 Blob(0x558c578f1dc0 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x45000~2000: 0x0~2000 Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10368 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57396770 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57396770 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578f1d50 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1d50) get_ref 0x0~2000 Blob(0x558c578f1d50 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578f1d50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x45000~2000: 0x0~2000 Blob(0x558c578f1d50 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x47000~2000: 0x0~2000 Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10369 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d110 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d110 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578dfdc0 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578dfdc0) get_ref 0x0~2000 Blob(0x558c578dfdc0 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578dfdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x47000~2000: 0x0~2000 Blob(0x558c578dfdc0 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x49000~2000: 0x0~2000 Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10370 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578dfe30 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578dfe30) get_ref 0x0~2000 Blob(0x558c578dfe30 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578dfe30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x49000~2000: 0x0~2000 Blob(0x558c578dfe30 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x4b000~2000: 0x0~2000 Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10371 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c578dff80 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578dff80) get_ref 0x0~2000 Blob(0x558c578dff80 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c578dff80) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x4b000~2000: 0x0~2000 Blob(0x558c578dff80 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x4d000~2000: 0x0~2000 Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10372 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c577f36c0 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c577f36c0) get_ref 0x0~2000 Blob(0x558c577f36c0 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c577f36c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x4d000~2000: 0x0~2000 Blob(0x558c577f36c0 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x4f000~2000: 0x0~2000 Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10373 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d490 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d490 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c575bdc70 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c575bdc70) get_ref 0x0~2000 Blob(0x558c575bdc70 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c575bdc70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x4f000~2000: 0x0~2000 Blob(0x558c575bdc70 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x51000~2000: 0x0~2000 Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10374 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d570 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d570 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57dcad20 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57dcad20) get_ref 0x0~2000 Blob(0x558c57dcad20 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57dcad20) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x51000~2000: 0x0~2000 Blob(0x558c57dcad20 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x53000~2000: 0x0~2000 Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10375 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d650 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d650 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c54d75810 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c54d75810) get_ref 0x0~2000 Blob(0x558c54d75810 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c54d75810) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x53000~2000: 0x0~2000 Blob(0x558c54d75810 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x55000~2000: 0x0~2000 Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10376 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d730 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d730 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff5110 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5110) get_ref 0x0~2000 Blob(0x558c57ff5110 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5110) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x55000~2000: 0x0~2000 Blob(0x558c57ff5110 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x57000~2000: 0x0~2000 Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10377 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d810 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d810 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff5180 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5180) get_ref 0x0~2000 Blob(0x558c57ff5180 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5180) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x57000~2000: 0x0~2000 Blob(0x558c57ff5180 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup src 0x59000~1000: 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10378 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=1)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup new Blob(0x558c57ff51f0 blob([!~1000,0x3b9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff51f0) get_ref 0x1000~1000 Blob(0x558c57ff51f0 blob([!~1000,0x3b9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff51f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dup dst 0x59000~1000: 0x1000~1000 Blob(0x558c57ff51f0 blob([!~1000,0x3b9000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=2)))) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark inline shard dirty 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:e# = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# snapset 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# snapset = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 1 keys 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 1 keys = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key (42 bytes) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:head#18 key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0xfffffffffffffffe0000000000000018'o' 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:head#18 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:head#18 = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1166 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 3 keys 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 3 keys = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~1f000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~1f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef8000 16 blobs 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1f000/1000,1f000,0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x56e000~1f000/1000,1f000,0 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x56e000~1f000] 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff52d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff52d0 blob([0x56e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5340 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff52d0) get_ref 0x0~2000 Blob(0x558c57ff52d0 blob([0x56e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5340 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff52d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57ff52d0 blob([0x56e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff53b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff53b0 blob([0x570000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5420 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff53b0) get_ref 0x0~2000 Blob(0x558c57ff53b0 blob([0x570000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5420 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff53b0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57ff53b0 blob([0x570000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5438 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5490 blob([0x572000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5500 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5490) get_ref 0x0~2000 Blob(0x558c57ff5490 blob([0x572000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5500 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5490) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57ff5490 blob([0x572000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5518 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff55e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5570 blob([0x574000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff55e0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5570) get_ref 0x0~2000 Blob(0x558c57ff5570 blob([0x574000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff55e0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5570) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57ff5570 blob([0x574000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff55f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff58f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff58f0 blob([0x576000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5880 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff58f0) get_ref 0x0~2000 Blob(0x558c57ff58f0 blob([0x576000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5880 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff58f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57ff58f0 blob([0x576000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff57a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5810 blob([0x578000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff57a0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5810) get_ref 0x0~2000 Blob(0x558c57ff5810 blob([0x578000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff57a0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5810) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57ff5810 blob([0x578000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff57b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff56c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5730 blob([0x57a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff56c0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5730) get_ref 0x0~2000 Blob(0x558c57ff5730 blob([0x57a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff56c0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5730) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57ff5730 blob([0x57a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff56d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5650 blob([0x57c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4e00 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5650) get_ref 0x0~2000 Blob(0x558c57ff5650 blob([0x57c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4e00 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5650) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57ff5650 blob([0x57c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff4e18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff4e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff4e70 blob([0x57e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4ee0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4e70) get_ref 0x0~2000 Blob(0x558c57ff4e70 blob([0x57e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4ee0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4e70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57ff4e70 blob([0x57e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff4ef8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff4f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff4f50 blob([0x580000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4fc0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4f50) get_ref 0x0~2000 Blob(0x558c57ff4f50 blob([0x580000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4fc0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4f50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57ff4f50 blob([0x580000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff4fd8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5030 blob([0x582000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5960 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5030) get_ref 0x0~2000 Blob(0x558c57ff5030 blob([0x582000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5960 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5030) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57ff5030 blob([0x582000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5978 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff59d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff59d0 blob([0x584000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5a40 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff59d0) get_ref 0x0~2000 Blob(0x558c57ff59d0 blob([0x584000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5a40 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff59d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57ff59d0 blob([0x584000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5a58 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5ab0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5b20 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5ab0) get_ref 0x0~2000 Blob(0x558c57ff5ab0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5b20 sbid 0x0)) 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.639+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c57ff5ab0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5b38 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5b90 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c00 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5b90) get_ref 0x0~2000 Blob(0x558c57ff5b90 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c00 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5b90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57ff5b90 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5c18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5c70 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5ce0 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5c70) get_ref 0x0~2000 Blob(0x558c57ff5c70 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5ce0 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5c70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57ff5c70 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5cf8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff5d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff5d50 blob([0x58c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5dc0 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5d50) get_ref 0x0~1000 Blob(0x558c57ff5d50 blob([0x58c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5dc0 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff5d50) get_ref init 0x1000, 1000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~1000: 0x0~1000 Blob(0x558c57ff5d50 blob([0x58c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 sbid 0x0)) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff5dd8 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1f000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58092108) dirty_range mark inline shard dirty 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~1f000 = 0 2019-12-07T19:34:14.640+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.640+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.640+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.640+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000037.00000000000000000023' <- 0000000037.00000000000000000023 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000037.00000000000000000024' <- 0000000037.00000000000000000024 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8000 cost 219803 (22 ios * 4000 + 131803 bytes) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8000 onodes 0x558c574a1180,0x558c57f35b80,0x558c58092000 shared_blobs 0x558c546777a0,0x558c54d75e30,0x558c57396230,0x558c573967e0,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70,0x558c5792c070,0x558c5792c150,0x558c5792c230,0x558c5792c310,0x558c5792c3f0,0x558c5792c4d0,0x558c5792c5b0,0x558c5792c690,0x558c5792d180,0x558c5792d260,0x558c5792d340,0x558c5792d420,0x558c5792d500,0x558c5792d5e0,0x558c5792d6c0,0x558c5792d7a0,0x558c5792d880,0x558c5792d960,0x558c5794d420,0x558c5794d500,0x558c5794d5e0,0x558c5794d6c0,0x558c5794d7a0,0x558c5794d880,0x558c5794d960,0x558c5794da40,0x558c5794db20,0x558c5794dc00,0x558c5794dce0,0x558c5794ddc0,0x558c5794dea0,0x558c5794df80 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) update 0#1:3a5a3f5b:::smithi13140796-23:head#18 2019-12-07T19:34:14.640+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 58 ==== MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=42, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c58092780 con 0x558c54c19800 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) update inline shard 1256 bytes from 46 extents 2019-12-07T19:34:14.640+0000 7f8f24c58700 15 osd.7 37 enqueue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000021 epoch 37 MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=42, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore.extentmap(0x558c574a1288) reshard 0x[0,ffffffff) of 0 shards on 0#1:3a5a3f5b:::smithi13140796-23:head#18 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard new shard 0x0 2019-12-07T19:34:14.640+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=42, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard new shard 0x23000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard new shard 0x47000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard old [] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) update 0#1:3a5a3f5b:::smithi13140796-23:head#18 force 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:14.640+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) update shard 0x23000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1288) update shard 0x47000 is 288 bytes (was 0) from 10 extents 2019-12-07T19:34:14.640+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=42, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.640+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=42, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) pg 0x558c55141000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head#18 is 500 (498 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) update 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) update inline shard 1256 bytes from 46 extents 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore.extentmap(0x558c57f35c88) reshard 0x[0,ffffffff) of 0 shards on 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard new shard 0x0 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard new shard 0x23000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard new shard 0x47000 2019-12-07T19:34:14.640+0000 7f8f06185700 10 osd.7 37 dequeue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000063 MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=42, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:14.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 166 ==== MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=166, reqid=client.4540.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=37'15)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c551dc000 con 0x558c54c64000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard old [] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:14.640+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57effb80 prio 127 cost 0 latency 0.000022 epoch 37 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=166, reqid=client.4540.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=37'15)) v2 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) update 0#1:3a5a3f5b:::smithi13140796-23:e# force 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:14.640+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=166, reqid=client.4540.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=37'15)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.640+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=42, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) update shard 0x23000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35c88) update shard 0x47000 is 288 bytes (was 0) from 10 extents 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:e# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:14.640+0000 7f8f06185700 25 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58092108) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:14.640+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58092108) update inline shard 304 bytes from 16 extents 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1e000 size 0x1e000 (122880) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 867 (557 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287f is 11 SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 167 ==== MOSDECSubOpRead(1.0s3 37/18 ECSubRead(tid=167, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c5738d680 con 0x558c54c64000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2851 is 11 SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284a is 11 SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2880 is 11 SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c578d3e40 prio 127 cost 0 latency 0.000015 epoch 37 MOSDECSubOpRead(1.0s3 37/18 ECSubRead(tid=167, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2853 is 11 SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2852 is 11 SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284b is 11 SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 37/18 ECSubRead(tid=167, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2854 is 11 SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2855 is 11 SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284c is 11 SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284d is 11 SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284e is 11 SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284f is 11 SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2850 is 11 SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2877 is 11 SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2878 is 11 SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2879 is 11 SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287a is 11 SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287b is 11 SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287c is 11 SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=166, reqid=client.4540.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=37'15)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287d is 11 SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287e is 11 SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2881 is 11 SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=166, reqid=client.4540.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=37'15)) v2) prio 127 cost 0 e37) pg 0x558c5500d000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2882 is 11 SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2883 is 11 SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2884 is 11 SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2885 is 11 SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c57effb80 prio 127 cost 0 latency 0.000102 MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=166, reqid=client.4540.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=37'15)) v2 pg pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'12 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2886 is 11 SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2887 is 11 SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2888 is 11 SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2889 is 11 SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288a is 11 SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=2))) 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'12 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.0s3 37/18 ECSubWrite(tid=166, reqid=client.4540.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=37'15)) v2 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2869 is 11 SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286a is 11 SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286b is 11 SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286c is 11 SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286d is 11 SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'12 active mbc={} ps=[a~1]] append_log log((0'0,37'15], crt=37'12) [] 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286e is 11 SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286f is 11 SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2870 is 11 SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2871 is 11 SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2872 is 11 SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2873 is 11 SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2874 is 11 SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] rollforward: entry=37'13 (37'12) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:301 2019-12-07T19:34:14.630662+0000 0 ObjectCleanRegions clean_offsets: [0~1179648,1900544~18446744073707651071], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2875 is 11 SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2876 is 11 SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=2))) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8000 allocated 0x[56e000~1f000] released 0x[] 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 freelist allocate 0x56e000~1f000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 freelist _xor first_key 0x500000 last_key 0x580000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] rollforward: entry=37'14 (37'13) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:302 2019-12-07T19:34:14.631058+0000 0 ObjectCleanRegions clean_offsets: [0~1900544,1949696~18446744073707601919], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 prepare 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef8000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] rollforward: entry=37'15 (37'14) modify 1:64fe1d00:::smithi13140796-44:head by client.4540.0:303 2019-12-07T19:34:14.631074+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] append_log approx pg log length = 15 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.640+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.640+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c54d58c00 seq 30 2019-12-07T19:34:14.640+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes shared_blobs 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[] released 0x[] 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c57effb80 finish 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 37/18 ECSubRead(tid=167, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpRead(1.0s3 37/18 ECSubRead(tid=167, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) pg 0x558c5500d000 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c578d3e40 prio 127 cost 0 latency 0.000241 MOSDECSubOpRead(1.0s3 37/18 ECSubRead(tid=167, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd0130~471 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd0130 0xd0130~471 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.0s3 37/18 ECSubRead(tid=167, to_read={1:0064fe1d:::smithi13140796-44:head=0,4096,0}, subchunks={1:0064fe1d:::smithi13140796-44:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa0130 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x130 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5a1 and padding block with 0xa5f 2019-12-07T19:34:14.640+0000 7f8f04982700 25 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:14.640+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x0~1000 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x77000 (487424) 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:14.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf9880 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 0x0~1000 = 4096 2019-12-07T19:34:14.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 51 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57568800 con 0x558c54c19c00 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:14.640+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000011 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.640+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:0064fe1d:::smithi13140796-44:head 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] get_hash_info: not in cache 1:0064fe1d:::smithi13140796-44:head 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:14.640+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] get_hash_info: found on disk, size 487424 2019-12-07T19:34:14.640+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key 2019-12-07T19:34:14.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key = 0 2019-12-07T19:34:14.640+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.0s0 37/18 ECSubReadReply(tid=167, attrs_read=0)) v2 -- 0x558c5738c280 con 0x558c54c64000 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd05a1 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.640+0000 7f8f04982700 10 osd.7 37 dequeue_op 0x558c578d3e40 finish 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000234698s (0.000000140s flush + 0.000234558s kv commit) 2019-12-07T19:34:14.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:34:14.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 finishing 2019-12-07T19:34:14.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:14.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:34:14.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.640+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 37/18 ECSubWriteReply(tid=166, last_complete=37'15, committed=1, applied=1)) v2 -- 0x558c54d77c00 con 0x558c54c64000 2019-12-07T19:34:14.640+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 lua=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] op_applied version 37'24 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c54d3d208 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c573d9ef0 space 0x558c54d3d208 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e32e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc000 space 0x558c574e32e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3438 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc090 space 0x558c574e3438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3588 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc120 space 0x558c574e3588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3668 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc1b0 space 0x558c574e3668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e37b8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc240 space 0x558c574e37b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3898 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc2d0 space 0x558c574e3898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3978 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc360 space 0x558c574e3978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3a58 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc3f0 space 0x558c574e3a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3b38 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc480 space 0x558c574e3b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3c18 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc510 space 0x558c574e3c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3cf8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc5a0 space 0x558c574e3cf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3dd8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc630 space 0x558c574e3dd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3eb8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc6c0 space 0x558c574e3eb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] eval_repop repgather(0x558c55159c20 37'24 rep_tid=18 committed?=0 r=0) 2019-12-07T19:34:14.640+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c574e3f98 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:14.640+0000 7f8f06185700 20 _add level 1 near 0 on buffer(0x558c580cc750 space 0x558c574e3f98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:14.640+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 = 122880 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] kick_object_context_blocked 1:3a5a3f5b:::smithi13140796-23:head requeuing 1 requests 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] requeue_op 0x558c57efeb00 2019-12-07T19:34:14.640+0000 7f8f06185700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read read request=122880 r=122880 len=122880 2019-12-07T19:34:14.640+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.640+0000 7f8f0b18f700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.640+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: not in cache 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.640+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:14.640+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: found on disk, size 122880 2019-12-07T19:34:14.640+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key 2019-12-07T19:34:14.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 108 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d77c00 con 0x558c54c5b400 2019-12-07T19:34:14.640+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key = 0 2019-12-07T19:34:14.640+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c578d2dc0 prio 196 cost 0 latency 0.000014 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f06185700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.640+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.640+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] process_copy_chunk 1:276d5164:::smithi13140796-22:head tid 2 (0) Success 2019-12-07T19:34:14.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.640+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.640+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 37/18 ECSubReadReply(tid=42, attrs_read=0)) v2 -- 0x558c57454f00 con 0x558c54c19800 2019-12-07T19:34:14.640+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.640+0000 7f8f06185700 10 osd.7 37 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:14.640+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efeb00 prio 63 cost 0 latency 0.026923 osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] process_copy_chunk got digest: rx data 0x98d51239 omap 0xffffffff, source: data 0x98d51239 omap 0xffffffff flags 1 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] process_copy_chunk success; committing 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] process_copy_chunk complete r = (0) Success 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] execute_ctx 0x558c576e0a00 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] execute_ctx 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] ov 21'2 av 37'3 snapc e=[e,d,c] snapset 0=[]:{} 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_osd_op 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_osd_op copy-from ver 4 in=54b 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] finish_copyfrom on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] finish_copyfrom: exists, removing 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] fill_in_final_tx: writing directly to final object 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _write_copy_chunk 0x558c5751cc10 0 attrs 311296 bytes 0 omap header bytes 0 omap data bytes 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=1 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] finish_copyfrom clearing omap flag on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] make_writeable 1:276d5164:::smithi13140796-22:head snapset=0=[]:{} snapc=e=[e,d,c] 2019-12-07T19:34:14.641+0000 7f8f06986700 25 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] got greedy write on clone_obc obc(1:276d5164:::smithi13140796-22:e rwstate(write n=1 w=0)) 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] do_op waiting for rw locks 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] cloning v 21'2 to 1:276d5164:::smithi13140796-22:e v 37'3 snaps=[e,d,c] snapset=0=[]:{e=[e,d,c]} 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] make_writeable 1:276d5164:::smithi13140796-22:head done, snapset=e=[]:{e=[e,d,c]} 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] finish_ctx 1:276d5164:::smithi13140796-22:head 0x558c576e0a00 op modify 2019-12-07T19:34:14.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] mtime unchanged at 2019-12-07T19:33:58.661803+0000 2019-12-07T19:34:14.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 aio_wait 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] final snapset e=[]:{e=[e,d,c]} in 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:14.641+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8000 2019-12-07T19:34:14.641+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000642 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] finish_ctx object 1:276d5164:::smithi13140796-22:head marks clean_regions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 io_done 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] finish_ctx extra_reqids [client.4540.0:187,4,client.4540.0:56,4,client.4540.0:55,3,client.4540.0:23,5,client.4540.0:22,2] {} 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] execute_ctx alloc reply 0x558c58092500 result 0 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.641+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d3a20 finish 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] calc_trim_to_aggressive limit = 21'2 2019-12-07T19:34:14.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] op order client.4540 tid 305 last was 65 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c578d2dc0 prio 196 cost 0 latency 0.000271 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] new_repop rep_tid 19 on osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c578d2dc0 finish 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] new_repop: repgather(0x558c576b6c60 0'0 rep_tid=19 committed?=0 r=0) 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.641+0000 7f8f0b18f700 7 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] issue_repop rep_tid 19 o 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8748 < min_flush_size 524288 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] submit_transaction: op Op(1:276d5164:::smithi13140796-22:head v=37'4 tt=0'0 tid=19 reqid=client.4540.0:305 client_op=osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:276d5164:::smithi13140796-22:e# key 0x808000000000000001276d51'd!smithi13140796-22!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) r -2 v.len 0 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1180 had 1 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35b80 had 1 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092000 had 1 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8771 < min_flush_size 524288 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8771 < min_flush_size 524288 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_hash_info: found on disk, size 90112 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd05a1~2243 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd05a1 0xd05a1~2243 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa05a1 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5a1 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7e4 and padding block with 0x81c 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:e projected size 360448 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:head projected size 311296 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] start_rmw: Op(1:276d5164:::smithi13140796-22:head v=37'4 tt=0'0 tid=19 reqid=client.4540.0:305 client_op=osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[0~311296]}) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_state_to_reads: invalidating cache after this op 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_state_to_reads: Op(1:276d5164:::smithi13140796-22:head v=37'4 tt=0'0 tid=19 reqid=client.4540.0:305 client_op=osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[0~311296]}) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:head v=37'4 tt=0'0 tid=19 reqid=client.4540.0:305 client_op=osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[0~311296]}) 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd27e4 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( Extents(1:276d5164:::smithi13140796-22:head)[ Extent(0~311296:3) ) 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] generate_transactions: 1:276d5164:::smithi13140796-22:e, current size is 0 buffers are {} 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8000> 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000193267s (0.000051121s flush + 0.000142146s kv commit) 2019-12-07T19:34:14.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 kv_submitted 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8000 2019-12-07T19:34:14.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 finishing 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8000 onodes 0x558c574a1180,0x558c57f35b80,0x558c58092000 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff4e18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09950 space 0x558c57ff4e18 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff4ef8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a099e0 space 0x558c57ff4ef8 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): new_size start 360448 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff4fd8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09a70 space 0x558c57ff4fd8 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5358 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09560 space 0x558c57ff5358 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5438 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a095f0 space 0x558c57ff5438 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): to_append: {} 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5518 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09680 space 0x558c57ff5518 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff55f8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09710 space 0x558c57ff55f8 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff56d8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a098c0 space 0x558c57ff56d8 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:e resetting hinfo to logical size 360448 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff57b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09830 space 0x558c57ff57b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5898 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a097a0 space 0x558c57ff5898 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5978 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09b00 space 0x558c57ff5978 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5a58 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09b90 space 0x558c57ff5a58 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5b38 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09c20 space 0x558c57ff5b38 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5c18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09cb0 space 0x558c57ff5c18 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] generate_transactions: 1:276d5164:::smithi13140796-22:head, current size is 360448 buffers are {0~311296(311296)} 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5cf8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09d40 space 0x558c57ff5cf8 0x0~2000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff5dd8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c57a09dd0 space 0x558c57ff5dd8 0x0~1000 writing nocache) 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8000 done 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:14.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8000 [] 2019-12-07T19:34:14.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): new_size start 0 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): adding buffer_update 0,311296 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): to_append: {0~311296(311296)} 2019-12-07T19:34:14.641+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): appending 0~311296 2019-12-07T19:34:14.641+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 59 ==== MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=43, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c58092a00 con 0x558c54c19800 2019-12-07T19:34:14.641+0000 7f8f24c58700 15 osd.7 37 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000015 epoch 37 MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=43, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.641+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=43, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) 2019-12-07T19:34:14.641+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 60 ==== MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=44, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57454f00 con 0x558c54c19800 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f24c58700 15 osd.7 37 enqueue_op 0x558c573354a0 prio 127 cost 0 latency 0.000011 epoch 37 MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=44, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.641+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=44, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=43, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=43, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) pg 0x558c55141000 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=44, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 osd.7 37 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000071 MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=43, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=43, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.641+0000 7f8f0217d700 25 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:14.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1e000 size 0x1e000 (122880) 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 = 122880 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read read request=122880 r=122880 len=122880 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: not in cache 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: found on disk, size 122880 2019-12-07T19:34:14.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key = 0 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.641+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 37/18 ECSubReadReply(tid=43, attrs_read=0)) v2 -- 0x558c575baf00 con 0x558c54c19800 2019-12-07T19:34:14.641+0000 7f8f0217d700 10 osd.7 37 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=44, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e37) pg 0x558c55141000 2019-12-07T19:34:14.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:14.641+0000 7f8f06185700 10 osd.7 37 dequeue_op 0x558c573354a0 prio 127 cost 0 latency 0.000311 MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=44, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.641+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.ds2 37/18 ECSubRead(tid=44, to_read={1:bca18876:::smithi13140796-30:head=0,122880,0}, subchunks={1:bca18876:::smithi13140796-30:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:14.641+0000 7f8f06185700 25 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:14.641+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1e000 size 0x1e000 (122880) 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:14.641+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~1e000 = 122880 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read read request=122880 r=122880 len=122880 2019-12-07T19:34:14.641+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.641+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: not in cache 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.641+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:14.641+0000 7f8f06185700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] get_hash_info: found on disk, size 122880 2019-12-07T19:34:14.641+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key 2019-12-07T19:34:14.641+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key = 0 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:14.641+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 37/18 ECSubReadReply(tid=44, attrs_read=0)) v2 -- 0x558c58092780 con 0x558c54c19800 2019-12-07T19:34:14.641+0000 7f8f06185700 10 osd.7 37 dequeue_op 0x558c573354a0 finish 2019-12-07T19:34:14.641+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:14.641+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 61 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59400 con 0x558c54c19800 2019-12-07T19:34:14.641+0000 7f8f24c58700 15 osd.7 37 enqueue_op 0x558c54e79a20 prio 196 cost 0 latency 0.000008 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.641+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.641+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c54e79a20 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.642+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.642+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:34:14.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.642+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 101 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c576b5200 con 0x558c54c5bc00 2019-12-07T19:34:14.642+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c5740d080 prio 196 cost 0 latency 0.000011 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.642+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.642+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c5740d080 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.642+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.642+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:14.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] encode_and_write: 1:276d5164:::smithi13140796-22:head new_size 311296 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:head resetting hinfo to logical size 311296 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] operator(): marking append 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( Extents(1:276d5164:::smithi13140796-22:head)[ Extent(0~311296:3) ) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: written: {1:276d5164:::smithi13140796-22:e={},1:276d5164:::smithi13140796-22:head={0~311296(311296)}} 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:head v=37'4 tt=0'0 tid=19 reqid=client.4540.0:305 client_op=osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[0~311296]}) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: written_set: {1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[0~311296]} 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:e,{} 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:head,{0~311296(311296)} 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 osd.7 37 send_incremental_map 33 -> 37 to 0x558c54c64400 v2:172.21.15.131:6822/37079 2019-12-07T19:34:14.642+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(34..37 src has 1..37) v4 -- 0x558c5738fd40 con 0x558c54c64400 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 osd.7 37 maybe_share_map con 0x558c54c64400 v2:172.21.15.131:6822/37079 map epoch 33 -> 37 (shared) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c575be000 con 0x558c54c64400 2019-12-07T19:34:14.642+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c575c0000 con 0x558c54c64000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c575bf000 con 0x558c54c19800 2019-12-07T19:34:14.642+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c54895800 con 0x558c54c5b400 2019-12-07T19:34:14.642+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558c54897000 con 0x558c54c19c00 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 snap_mapper.add_oid 1:276d5164:::smithi13140796-22:e c,d,e 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 snap_mapper.get_snaps 1:276d5164:::smithi13140796-22:e got.empty() 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 snap_mapper.set_snaps 1:276d5164:::smithi13140796-22:e c,d,e 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 earliest_dup_version = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 trim 37'3 (21'2) clone 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 trim 37'4 (21'2) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:305 0.000000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] append_log log((0'0,21'2], crt=21'2) [37'3 (21'2) clone 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,37'4 (21'2) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:305 0.000000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'3 (0'0,37'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 lua=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] add_log_entry 37'3 (21'2) clone 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 lua=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] add_log_entry 37'4 (21'2) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:305 0.000000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 lua=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] append_log approx pg log length = 4 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 lua=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c57ef8c00 seq 17 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:276d5164:::smithi13140796-22:e# key 0x808000000000000001276d51'd!smithi13140796-22!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) r -2 v.len 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# -> 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1167 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.onode(0x558c54e44a00).flush flush done 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# -> 0#1:276d5164:::smithi13140796-22:e# 0x0~16000 -> 0x0~16000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x0~2000: 0x0~2000 Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10379 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e56930 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e56930 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4ee0) get_ref 0x0~2000 Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54e56a80 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10380 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e56a80 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e56a80 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4e70) get_ref 0x0~2000 Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4e70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54e569a0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10381 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e569a0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e569a0 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4540) get_ref 0x0~2000 Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4540) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54e56540 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10382 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e56540 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e56540 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4e00) get_ref 0x0~2000 Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4e00) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c54e564d0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10383 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e564d0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e564d0 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4f50) get_ref 0x0~2000 Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4f50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c54e42c40 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10384 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e42c40 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e42c40 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4fc0) get_ref 0x0~2000 Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a4fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c54e43dc0 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10385 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e43dc0 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e43dc0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a50a0) get_ref 0x0~2000 Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a50a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c54e43ce0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10386 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e43ce0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e43ce0 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5110) get_ref 0x0~2000 Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5110) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c54e56bd0 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10387 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e56bd0 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e56bd0 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5180) get_ref 0x0~2000 Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5180) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c54e56cb0 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10388 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e56cb0 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e56cb0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a51f0) get_ref 0x0~2000 Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a51f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c54e56d90 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10389 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared Blob(0x558c54e56d90 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) make_blob_shared now Blob(0x558c54e56d90 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup new Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5260) get_ref 0x0~2000 Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5260) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=2)))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) dirty_range mark inline shard dirty 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58093508) dirty_range mark inline shard dirty 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# -> 0#1:276d5164:::smithi13140796-22:e# = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# snapset 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# snapset = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 1 keys 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 1 keys = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:276d5164:::smithi13140796-22:head#4 key 0x808000000000000001276d51'd!smithi13140796-22!='0xfffffffffffffffe0000000000000004'o' 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) r -2 v.len 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# -> 0#1:276d5164:::smithi13140796-22:head#4 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# -> 0#1:276d5164:::smithi13140796-22:head#4 = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1168 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 3 keys 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 3 keys = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x0~13000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:276d5164:::smithi13140796-22:head# 0x0~13000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~13000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef8c00 10 blobs 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x13000/1000,13000,0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x58d000~13000/1000,13000,0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x58d000~13000] 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a53b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a53b0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5340) get_ref 0x0~2000 Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a53b0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5340) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a53b0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c580a53c8 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff4000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4b60 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4000) get_ref 0x0~2000 Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4b60 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4000) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4b60 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff4b78 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff4230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a07ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07ea0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4230) get_ref 0x0~2000 Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07ea0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c57ff4230) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07ea0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57a07eb8 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5dc0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5d50) get_ref 0x0~2000 Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5dc0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5d50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5dc0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c580a5dd8 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5ea0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5e30) get_ref 0x0~2000 Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5ea0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5e30) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5ea0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c580a5eb8 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5f80 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5f10) get_ref 0x0~2000 Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5f80 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c580a5f10) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5f80 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c580a5f98 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fd8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a000 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c574fd8f0) get_ref 0x0~2000 Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a000 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c574fd8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a000 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c5809a018 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809a070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a0e0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c5809a070) get_ref 0x0~2000 Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a0e0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c5809a070) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a0e0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c5809a0f8 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809a150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a1c0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c5809a150) get_ref 0x0~2000 Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a1c0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c5809a150) get_ref init 0x2000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a1c0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c5809a1d8 in 0x558c53a59260) _discard 0x0~2000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809a230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809a230 blob([0x59f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a2a0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c5809a230) get_ref 0x0~1000 Blob(0x558c5809a230 blob([0x59f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a2a0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.blob(0x558c5809a230) get_ref init 0x1000, 1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~1000: 0x0~1000 Blob(0x558c5809a230 blob([0x59f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5809a2a0 sbid 0x0)) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c5809a2b8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x13000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35a08) dirty_range mark inline shard dirty 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x0~13000 = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000037.00000000000000000003' <- 0000000037.00000000000000000003 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000037.00000000000000000004' <- 0000000037.00000000000000000004 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8c00 cost 142546 (15 ios * 4000 + 82546 bytes) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8c00 onodes 0x558c54e44a00,0x558c57f35900,0x558c58093400 shared_blobs 0x558c54e42af0,0x558c54e42d20,0x558c54e43d50,0x558c54e563f0,0x558c54e568c0,0x558c54e56a10,0x558c54e56af0,0x558c54e56b60,0x558c54e56c40,0x558c54e56d20,0x558c54e56e00 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) update 0#1:276d5164:::smithi13140796-22:head#4 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c54e44b08) update inline shard 300 bytes from 11 extents 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:head#4 is 791 (485 bytes onode + 2 bytes spanning blobs + 304 bytes inline extents) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35a08) update 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c57f35a08) update inline shard 190 bytes from 10 extents 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:head# is 753 (557 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58093508) update 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58093508) update inline shard 300 bytes from 11 extents 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:e# is 718 (412 bytes onode + 2 bytes spanning blobs + 304 bytes inline extents) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2890 is 11 SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2892 is 11 SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2891 is 11 SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288f is 11 SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288b is 11 SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288d is 11 SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288e is 11 SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288c is 11 SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2893 is 11 SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2894 is 11 SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2895 is 11 SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=2))) 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8c00 allocated 0x[58d000~13000] released 0x[] 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 freelist allocate 0x58d000~13000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 prepare 2019-12-07T19:34:14.642+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef8c00 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.643+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 lua=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] op_applied version 37'4 2019-12-07T19:34:14.643+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] eval_repop repgather(0x558c576b6c60 37'4 rep_tid=19 committed?=0 r=0) 2019-12-07T19:34:14.643+0000 7f8f0b18f700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] kick_object_context_blocked 1:276d5164:::smithi13140796-22:head requeuing 1 requests 2019-12-07T19:34:14.643+0000 7f8f0b18f700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] requeue_op 0x558c57eff4a0 2019-12-07T19:34:14.643+0000 7f8f0b18f700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) queued 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57eff4a0 prio 63 cost 0 latency 0.012505 osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.643+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 aio_wait 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.643+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8c00 2019-12-07T19:34:14.643+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 io_done 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(write n=1 w=0) oi: 1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) 2019-12-07T19:34:14.643+0000 7f8f06986700 25 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op oi 1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] do_op waiting for rw locks 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57eff4a0 finish 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5344 < min_flush_size 524288 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44a00 had 1 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35900 had 1 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093400 had 1 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5367 < min_flush_size 524288 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5367 < min_flush_size 524288 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd27e4~14f7 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd27e4 0xd27e4~14f7 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa27e4 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7e4 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcdb and padding block with 0x325 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd3cdb 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000187540s (0.000041559s flush + 0.000145981s kv commit) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8c00> 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 kv_submitted 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8c00 2019-12-07T19:34:14.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 finishing 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8c00 onodes 0x558c54e44a00,0x558c57f35900,0x558c58093400 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a07eb8 in 0x558c53a59260) _finish_write discard buffer(0x558c580cccf0 space 0x558c57a07eb8 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff4b78 in 0x558c53a59260) _finish_write discard buffer(0x558c580ccc60 space 0x558c57ff4b78 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a018 in 0x558c53a59260) _finish_write discard buffer(0x558c580ccf30 space 0x558c5809a018 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a0f8 in 0x558c53a59260) _finish_write discard buffer(0x558c580ccfc0 space 0x558c5809a0f8 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a1d8 in 0x558c53a59260) _finish_write discard buffer(0x558c580cd050 space 0x558c5809a1d8 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a2b8 in 0x558c53a59260) _finish_write discard buffer(0x558c580cd0e0 space 0x558c5809a2b8 0x0~1000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a53c8 in 0x558c53a59260) _finish_write discard buffer(0x558c580ccbd0 space 0x558c580a53c8 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5dd8 in 0x558c53a59260) _finish_write discard buffer(0x558c580ccd80 space 0x558c580a5dd8 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5eb8 in 0x558c53a59260) _finish_write discard buffer(0x558c580cce10 space 0x558c580a5eb8 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5f98 in 0x558c53a59260) _finish_write discard buffer(0x558c580ccea0 space 0x558c580a5f98 0x0~2000 writing nocache) 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8c00 done 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:14.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8c00 [] 2019-12-07T19:34:14.643+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.643+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 43 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c575b9e00 con 0x558c54c64400 2019-12-07T19:34:14.643+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c57335760 prio 196 cost 0 latency 0.000019 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.643+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.643+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57335760 prio 196 cost 0 latency 0.000052 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.643+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.643+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57335760 finish 2019-12-07T19:34:14.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.643+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 62 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59e00 con 0x558c54c19800 2019-12-07T19:34:14.643+0000 7f8f24c58700 15 osd.7 37 enqueue_op 0x558c57efedc0 prio 196 cost 0 latency 0.000010 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.643+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.643+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efedc0 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.644+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.644+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efedc0 finish 2019-12-07T19:34:14.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 52 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57568600 con 0x558c54c19c00 2019-12-07T19:34:14.644+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000015 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.644+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.644+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 109 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d76000 con 0x558c54c5b400 2019-12-07T19:34:14.644+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57335340 prio 196 cost 0 latency 0.000009 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.644+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.644+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:14.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.644+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57335340 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.644+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.644+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57335340 finish 2019-12-07T19:34:14.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.645+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 168 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c575b8600 con 0x558c54c64000 2019-12-07T19:34:14.645+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c5740db80 prio 196 cost 0 latency 0.000013 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.645+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c5740db80 prio 196 cost 0 latency 0.000030 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=18, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=37'24 tt=0'0 tid=18 reqid=client.4540.0:286 client_op=osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[0~507904]}) 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=35'22 crt=35'22 lcod 35'21 mlcod 35'21 active+clean ps=[a~1]] repop_all_committed: repop tid 18 all committed 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] eval_repop repgather(0x558c55159c20 37'24 rep_tid=18 committed?=1 r=0) 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] commit: repgather(0x558c55159c20 37'24 rep_tid=18 committed?=1 r=0) 2019-12-07T19:34:14.645+0000 7f8f0297e700 15 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] log_op_stats osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 inb 507904 outb 0 lat 0.031145 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] sending reply on osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 0x558c57f35680 2019-12-07T19:34:14.645+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(286 smithi13140796-23 [copy-from ver 8] v37'24 uv8 ondisk = 0) v8 -- 0x558c57f35680 con 0x558c54c9f000 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:14.645+0000 7f8f0297e700 15 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] publish_stats_to_osd 36:111 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] removing repgather(0x558c55159c20 37'24 rep_tid=18 committed?=1 r=0) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] q front is repgather(0x558c55159c20 37'24 rep_tid=18 committed?=1 r=0) 2019-12-07T19:34:14.645+0000 7f8f0297e700 15 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] remove_repop repgather(0x558c55159c20 37'24 rep_tid=18 committed?=1 r=0) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] requeue_op 0x558c57efeb00 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) 2019-12-07T19:34:14.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) queued 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=37'24 tt=0'0 tid=18 reqid=client.4540.0:286 client_op=osd_op(client.4540.0:286 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [copy-from ver 8 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=35'22 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[0~507904]}) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_finish_rmw: ExtentCache( Extents(1:3a5a3f5b:::smithi13140796-23:head)[ Extent(0~507904:3) ) 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=20 reqid=client.4540.0:286 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=20 reqid=client.4540.0:286 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:14.645+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 37/28 ECSubWrite(tid=20, reqid=client.4540.0:286, at_version=0'0, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c579e0000 con 0x558c54c64000 2019-12-07T19:34:14.645+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 37/28 ECSubWrite(tid=20, reqid=client.4540.0:286, at_version=0'0, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c579e0800 con 0x558c54c19800 2019-12-07T19:34:14.645+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 37/28 ECSubWrite(tid=20, reqid=client.4540.0:286, at_version=0'0, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c579e1000 con 0x558c54c5bc00 2019-12-07T19:34:14.645+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 37/28 ECSubWrite(tid=20, reqid=client.4540.0:286, at_version=0'0, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c579e1800 con 0x558c54c5b400 2019-12-07T19:34:14.645+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 37/28 ECSubWrite(tid=20, reqid=client.4540.0:286, at_version=0'0, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c579e2000 con 0x558c54c19c00 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=35'22 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] append_log log((0'0,37'24], crt=35'22) [] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] rollforward: entry=37'23 (29'18) clone 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] rollforward: entry=37'24 (29'18) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:286 0.000000 0 ObjectCleanRegions clean_offsets: [1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] append_log approx pg log length = 24 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.645+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c577fd800 seq 31 2019-12-07T19:34:14.645+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head#18 onode 0x558c574a1180 txc 0x558c577fd800 2019-12-07T19:34:14.645+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head#18 0x0 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c573961c0) put_ref 0x0~2000 Blob(0x558c573961c0 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c573962a0) put_ref 0x0~2000 Blob(0x558c573962a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c574f1f80) put_ref 0x0~2000 Blob(0x558c574f1f80 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1960) put_ref 0x0~2000 Blob(0x558c578f1960 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1a40) put_ref 0x0~2000 Blob(0x558c578f1a40 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1b20) put_ref 0x0~2000 Blob(0x558c578f1b20 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1c00) put_ref 0x0~2000 Blob(0x558c578f1c00 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1ce0) put_ref 0x0~2000 Blob(0x558c578f1ce0 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c57314000) put_ref 0x0~2000 Blob(0x558c57314000 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c574fa310) put_ref 0x0~2000 Blob(0x558c574fa310 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf96c0) put_ref 0x0~2000 Blob(0x558c54cf96c0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c574e05b0) put_ref 0x0~1000 Blob(0x558c574e05b0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d3b0) put_ref 0x0~2000 Blob(0x558c5794d3b0 blob([0x377000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d490) put_ref 0x0~2000 Blob(0x558c5794d490 blob([0x379000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d570) put_ref 0x0~2000 Blob(0x558c5794d570 blob([0x37b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d650) put_ref 0x0~2000 Blob(0x558c5794d650 blob([0x37d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d730) put_ref 0x0~2000 Blob(0x558c5794d730 blob([0x37f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d810) put_ref 0x0~2000 Blob(0x558c5794d810 blob([0x381000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d8f0) put_ref 0x0~2000 Blob(0x558c5794d8f0 blob([0x383000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d9d0) put_ref 0x0~2000 Blob(0x558c5794d9d0 blob([0x385000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dab0) put_ref 0x0~2000 Blob(0x558c5794dab0 blob([0x387000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794db90) put_ref 0x0~2000 Blob(0x558c5794db90 blob([0x389000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dc70) put_ref 0x0~2000 Blob(0x558c5794dc70 blob([0x38b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794dd50) put_ref 0x0~2000 Blob(0x558c5794dd50 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794de30) put_ref 0x0~2000 Blob(0x558c5794de30 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794df10) put_ref 0x0~2000 Blob(0x558c5794df10 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c000) put_ref 0x0~2000 Blob(0x558c5792c000 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c0e0) put_ref 0x0~2000 Blob(0x558c5792c0e0 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c1c0) put_ref 0x0~2000 Blob(0x558c5792c1c0 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c2a0) put_ref 0x0~2000 Blob(0x558c5792c2a0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c380) put_ref 0x0~2000 Blob(0x558c5792c380 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c460) put_ref 0x0~2000 Blob(0x558c5792c460 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c540) put_ref 0x0~2000 Blob(0x558c5792c540 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c620) put_ref 0x0~2000 Blob(0x558c5792c620 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5794d030) put_ref 0x0~2000 Blob(0x558c5794d030 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c57396770) put_ref 0x0~2000 Blob(0x558c57396770 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d110) put_ref 0x0~2000 Blob(0x558c5792d110 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d1f0) put_ref 0x0~2000 Blob(0x558c5792d1f0 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d2d0) put_ref 0x0~2000 Blob(0x558c5792d2d0 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d3b0) put_ref 0x0~2000 Blob(0x558c5792d3b0 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d490) put_ref 0x0~2000 Blob(0x558c5792d490 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d570) put_ref 0x0~2000 Blob(0x558c5792d570 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d650) put_ref 0x0~2000 Blob(0x558c5792d650 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d730) put_ref 0x0~2000 Blob(0x558c5792d730 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d810) put_ref 0x0~2000 Blob(0x558c5792d810 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.blob(0x558c5792d8f0) put_ref 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x3b9000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark shard 0x23000 dirty 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1288) dirty_range mark shard 0x47000 dirty 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c573961c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x339000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c573962a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33b000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c574f1f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33d000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c578f1960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33f000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c578f1a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x341000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c578f1b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x343000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c578f1c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x345000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c578f1ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x347000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57314000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x349000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c574fa310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34b000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c54cf96c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34d000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~1000: 0x0~1000 Blob(0x558c574e05b0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34f000~1000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x17000~2000: 0x0~2000 Blob(0x558c5794d3b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x377000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x19000~2000: 0x0~2000 Blob(0x558c5794d490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x379000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1b000~2000: 0x0~2000 Blob(0x558c5794d570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x37b000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1d000~2000: 0x0~2000 Blob(0x558c5794d650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x37d000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1f000~2000: 0x0~2000 Blob(0x558c5794d730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x37f000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x21000~2000: 0x0~2000 Blob(0x558c5794d810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x381000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x23000~2000: 0x0~2000 Blob(0x558c5794d8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x383000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x25000~2000: 0x0~2000 Blob(0x558c5794d9d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x385000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x27000~2000: 0x0~2000 Blob(0x558c5794dab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x387000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x29000~2000: 0x0~2000 Blob(0x558c5794db90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x389000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2b000~2000: 0x0~2000 Blob(0x558c5794dc70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x38b000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2d000~2000: 0x0~2000 Blob(0x558c5794dd50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x38d000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2f000~2000: 0x0~2000 Blob(0x558c5794de30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x38f000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x31000~2000: 0x0~2000 Blob(0x558c5794df10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x391000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x33000~2000: 0x0~2000 Blob(0x558c5792c000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x393000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x35000~2000: 0x0~2000 Blob(0x558c5792c0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x395000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x37000~2000: 0x0~2000 Blob(0x558c5792c1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x397000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x39000~2000: 0x0~2000 Blob(0x558c5792c2a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x399000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3b000~2000: 0x0~2000 Blob(0x558c5792c380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x39b000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3d000~2000: 0x0~2000 Blob(0x558c5792c460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x39d000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3f000~2000: 0x0~2000 Blob(0x558c5792c540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x39f000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x41000~2000: 0x0~2000 Blob(0x558c5792c620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a1000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x43000~2000: 0x0~2000 Blob(0x558c5794d030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a3000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x45000~2000: 0x0~2000 Blob(0x558c57396770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a5000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x47000~2000: 0x0~2000 Blob(0x558c5792d110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a7000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x49000~2000: 0x0~2000 Blob(0x558c5792d1f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a9000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4b000~2000: 0x0~2000 Blob(0x558c5792d2d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3ab000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4d000~2000: 0x0~2000 Blob(0x558c5792d3b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3ad000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4f000~2000: 0x0~2000 Blob(0x558c5792d490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3af000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x51000~2000: 0x0~2000 Blob(0x558c5792d570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b1000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x53000~2000: 0x0~2000 Blob(0x558c5792d650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b3000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x55000~2000: 0x0~2000 Blob(0x558c5792d730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b5000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x57000~2000: 0x0~2000 Blob(0x558c5792d810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b7000~2000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x59000~1000: 0x1000~1000 Blob(0x558c5792d8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=2)))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b9000~1000] 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x23000 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x47000 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c546777a0,0x558c54d75e30,0x558c57396230,0x558c573967e0,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70,0x558c5792c070,0x558c5792c150,0x558c5792c230,0x558c5792c310,0x558c5792c3f0,0x558c5792c4d0,0x558c5792c5b0,0x558c5792c690,0x558c5792d180,0x558c5792d260,0x558c5792d340,0x558c5792d420,0x558c5792d500,0x558c5792d5e0,0x558c5792d6c0,0x558c5792d7a0,0x558c5792d880,0x558c5792d960,0x558c5794d420,0x558c5794d500,0x558c5794d5e0,0x558c5794d6c0,0x558c5794d7a0,0x558c5794d880,0x558c5794d960,0x558c5794da40,0x558c5794db20,0x558c5794dc00,0x558c5794dce0,0x558c5794ddc0,0x558c5794dea0,0x558c5794df80 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c58092000 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head#18 = 0 2019-12-07T19:34:14.645+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd800 onodes shared_blobs 0x558c546777a0,0x558c54d75e30,0x558c57396230,0x558c573967e0,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70,0x558c5792c070,0x558c5792c150,0x558c5792c230,0x558c5792c310,0x558c5792c3f0,0x558c5792c4d0,0x558c5792c5b0,0x558c5792c690,0x558c5792d180,0x558c5792d260,0x558c5792d340,0x558c5792d420,0x558c5792d500,0x558c5792d5e0,0x558c5792d6c0,0x558c5792d7a0,0x558c5792d880,0x558c5792d960,0x558c5794d420,0x558c5794d500,0x558c5794d5e0,0x558c5794d6c0,0x558c5794d7a0,0x558c5794d880,0x558c5794d960,0x558c5794da40,0x558c5794db20,0x558c5794dc00,0x558c5794dce0,0x558c5794ddc0,0x558c5794dea0,0x558c5794df80 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287f is 11 SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2851 is 11 SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284a is 11 SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2880 is 11 SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2853 is 11 SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2852 is 11 SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284b is 11 SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2854 is 11 SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2855 is 11 SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284c is 11 SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284d is 11 SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284e is 11 SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284f is 11 SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2850 is 11 SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2877 is 11 SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2878 is 11 SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2879 is 11 SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287a is 11 SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287b is 11 SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287c is 11 SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287d is 11 SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287e is 11 SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2881 is 11 SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2882 is 11 SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2883 is 11 SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2884 is 11 SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2885 is 11 SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2886 is 11 SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2887 is 11 SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2888 is 11 SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2889 is 11 SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288a is 11 SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2869 is 11 SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286a is 11 SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286b is 11 SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286c is 11 SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286d is 11 SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286e is 11 SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286f is 11 SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2870 is 11 SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2871 is 11 SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2872 is 11 SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2873 is 11 SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2874 is 11 SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2875 is 11 SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2876 is 11 SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=1))) 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd800 allocated 0x[] released 0x[] 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 prepare 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd800 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 io_done 2019-12-07T19:34:14.645+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c5740db80 finish 2019-12-07T19:34:14.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efeb00 prio 63 cost 0 latency 0.031956 osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2502 < min_flush_size 524288 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_op osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1180 had 1 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:14.646+0000 7f8f06986700 25 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2525 < min_flush_size 524288 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2525 < min_flush_size 524288 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(excl n=1 w=0)) 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd3cdb~9dd to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd3cdb 0xd3cdb~9dd to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] execute_ctx 0x558c57450000 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa3cdb 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcdb 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6b8 and padding block with 0x948 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [stat] ov 37'24 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [stat] 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_osd_op stat 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] stat oi has 507904 2019-12-07T19:33:58.699945+0000 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] execute_ctx alloc reply 0x558c5738d900 result 0 2019-12-07T19:34:14.646+0000 7f8f06986700 15 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:14.646+0000 7f8f06986700 15 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] log_op_stats osd_op(client.4540.0:287 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [stat out=16b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 inb 0 outb 16 lat 0.032124 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd46b8 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.646+0000 7f8f06986700 15 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] publish_stats_to_osd 37:112 2019-12-07T19:34:14.646+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(287 smithi13140796-23 [stat out=16b] v0'0 uv8 ondisk = 0) v8 -- 0x558c5738d900 con 0x558c54c9f000 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000146738s (0.000000129s flush + 0.000146609s kv commit) 2019-12-07T19:34:14.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd800> 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 kv_submitted 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd800 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 finishing 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd800 onodes 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd800 done 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:14.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd800 [] 2019-12-07T19:34:14.646+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 53 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfa600 con 0x558c54c19c00 2019-12-07T19:34:14.646+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000013 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c54e794a0 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] 2019-12-07T19:34:14.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 110 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c551afc00 con 0x558c54c5b400 2019-12-07T19:34:14.646+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57f3e420 prio 196 cost 0 latency 0.000014 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e420 prio 196 cost 0 latency 0.000042 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e420 finish 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 169 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c53a5e600 con 0x558c54c64000 2019-12-07T19:34:14.646+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57f3e2c0 prio 196 cost 0 latency 0.000011 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57f3e2c0 prio 196 cost 0 latency 0.000032 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57f3e2c0 finish 2019-12-07T19:34:14.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 170 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54dfa400 con 0x558c54c64000 2019-12-07T19:34:14.646+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000008 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000025 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:276d5164:::smithi13140796-22:head v=37'4 tt=0'0 tid=19 reqid=client.4540.0:305 client_op=osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[0~311296]}) 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean ps=[a~1]] repop_all_committed: repop tid 19 all committed 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] eval_repop repgather(0x558c576b6c60 37'4 rep_tid=19 committed?=1 r=0) 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] commit: repgather(0x558c576b6c60 37'4 rep_tid=19 committed?=1 r=0) 2019-12-07T19:34:14.646+0000 7f8f0297e700 15 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] log_op_stats osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 inb 311296 outb 0 lat 0.016323 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] sending reply on osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) v8 0x558c58092500 2019-12-07T19:34:14.646+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(305 smithi13140796-22 [copy-from ver 4] v37'4 uv4 ondisk = 0) v8 -- 0x558c58092500 con 0x558c54c9f000 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:14.646+0000 7f8f0297e700 15 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] publish_stats_to_osd 36:54 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] removing repgather(0x558c576b6c60 37'4 rep_tid=19 committed?=1 r=0) 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] q front is repgather(0x558c576b6c60 37'4 rep_tid=19 committed?=1 r=0) 2019-12-07T19:34:14.646+0000 7f8f0297e700 15 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] remove_repop repgather(0x558c576b6c60 37'4 rep_tid=19 committed?=1 r=0) 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] requeue_op 0x558c57eff4a0 2019-12-07T19:34:14.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:head v=37'4 tt=0'0 tid=19 reqid=client.4540.0:305 client_op=osd_op(client.4540.0:305 1.4s0 1:276d5164:::smithi13140796-22:head [copy-from ver 4 in=54b] snapc e=[e,d,c] ondisk+write+known_if_redirected e37) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[0~311296]}) 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) queued 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_finish_rmw: ExtentCache( Extents(1:276d5164:::smithi13140796-22:head)[ Extent(0~311296:3) ) 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=21 reqid=client.4540.0:305 roll_forward_to=37'4 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=21 reqid=client.4540.0:305 roll_forward_to=37'4 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 -- 0x558c579e2800 con 0x558c54c64400 2019-12-07T19:34:14.647+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 -- 0x558c579e3000 con 0x558c54c64000 2019-12-07T19:34:14.647+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 -- 0x558c579e3800 con 0x558c54c19800 2019-12-07T19:34:14.647+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 -- 0x558c57e7a000 con 0x558c54c5b400 2019-12-07T19:34:14.647+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 -- 0x558c57e7a800 con 0x558c54c19c00 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=21'2 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] append_log log((0'0,37'4], crt=21'2) [] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] rollforward: entry=37'3 (21'2) clone 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] rollforward: entry=37'4 (21'2) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:305 0.000000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] append_log approx pg log length = 4 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.647+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c577fdb00 seq 18 2019-12-07T19:34:14.647+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.4s0_head 0#1:276d5164:::smithi13140796-22:head#4 onode 0x558c54e44a00 txc 0x558c577fdb00 2019-12-07T19:34:14.647+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.4s0_head 0#1:276d5164:::smithi13140796-22:head#4 0x0 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e56930) put_ref 0x0~2000 Blob(0x558c54e56930 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e56a80) put_ref 0x0~2000 Blob(0x558c54e56a80 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e569a0) put_ref 0x0~2000 Blob(0x558c54e569a0 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e56540) put_ref 0x0~2000 Blob(0x558c54e56540 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e564d0) put_ref 0x0~2000 Blob(0x558c54e564d0 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e42c40) put_ref 0x0~2000 Blob(0x558c54e42c40 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e43dc0) put_ref 0x0~2000 Blob(0x558c54e43dc0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e43ce0) put_ref 0x0~2000 Blob(0x558c54e43ce0 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e56bd0) put_ref 0x0~2000 Blob(0x558c54e56bd0 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e56cb0) put_ref 0x0~2000 Blob(0x558c54e56cb0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.blob(0x558c54e56d90) put_ref 0x0~2000 Blob(0x558c54e56d90 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore.extentmap(0x558c54e44b08) dirty_range mark inline shard dirty 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54e56930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x97000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54e56a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x99000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54e569a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x9b000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c54e56540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x9d000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c54e564d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x9f000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c54e42c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa1000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c54e43dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa3000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c54e43ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa5000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c54e56bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa7000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c54e56cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa9000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c54e56d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=2)))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xab000~2000] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54e42af0,0x558c54e42d20,0x558c54e43d50,0x558c54e563f0,0x558c54e568c0,0x558c54e56a10,0x558c54e56af0,0x558c54e56b60,0x558c54e56c40,0x558c54e56d20,0x558c54e56e00 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c57f35900 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.4s0_head 0#1:276d5164:::smithi13140796-22:head#4 = 0 2019-12-07T19:34:14.647+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fdb00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fdb00 onodes shared_blobs 0x558c54e42af0,0x558c54e42d20,0x558c54e43d50,0x558c54e563f0,0x558c54e568c0,0x558c54e56a10,0x558c54e56af0,0x558c54e56b60,0x558c54e56c40,0x558c54e56d20,0x558c54e56e00 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2890 is 11 SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2892 is 11 SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2891 is 11 SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288f is 11 SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288b is 11 SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288d is 11 SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288e is 11 SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288c is 11 SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2893 is 11 SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2894 is 11 SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2895 is 11 SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1))) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fdb00 allocated 0x[] released 0x[] 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 prepare 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fdb00 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 io_done 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1485 < min_flush_size 524288 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8) prio 63 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44a00 had 1 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1508 < min_flush_size 524288 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57eff4a0 prio 63 cost 0 latency 0.016755 osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1508 < min_flush_size 524288 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd46b8~5e4 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd46b8 0xd46b8~5e4 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa46b8 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6b8 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc9c and padding block with 0x364 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_op osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558c54d2efc0 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159440 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] find_object_context 1:276d5164:::smithi13140796-22:head @head oi=1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) 2019-12-07T19:34:14.647+0000 7f8f06986700 25 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_op oi 1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd4c9c 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_op obc obc(1:276d5164:::smithi13140796-22:head rwstate(excl n=1 w=0)) 2019-12-07T19:34:14.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] execute_ctx 0x558c57450000 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000151115s (0.000000092s flush + 0.000151023s kv commit) 2019-12-07T19:34:14.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] execute_ctx 1:276d5164:::smithi13140796-22:head [stat] ov 37'4 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fdb00> 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 kv_submitted 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fdb00 2019-12-07T19:34:14.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 finishing 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_osd_op 1:276d5164:::smithi13140796-22:head [stat] 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fdb00 onodes 2019-12-07T19:34:14.647+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 102 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c551aec00 con 0x558c54c5bc00 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_osd_op stat 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fdb00 done 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:14.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fdb00 [] 2019-12-07T19:34:14.647+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000014 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] stat oi has 311296 2019-12-07T19:33:58.661803+0000 2019-12-07T19:34:14.647+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] execute_ctx alloc reply 0x558c57efa000 result 0 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f06986700 15 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.647+0000 7f8f06986700 15 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] log_op_stats osd_op(client.4540.0:306 1.4s0 1:276d5164:::smithi13140796-22:head [stat out=16b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e37) v8 inb 0 outb 16 lat 0.016927 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:14.647+0000 7f8f06986700 15 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] publish_stats_to_osd 37:55 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:14.647+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(306 smithi13140796-22 [stat out=16b] v0'0 uv4 ondisk = 0) v8 -- 0x558c57efa000 con 0x558c54c9f000 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57eff4a0 finish 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 54 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfa200 con 0x558c54c19c00 2019-12-07T19:34:14.647+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57f3e580 prio 196 cost 0 latency 0.000019 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e580 prio 196 cost 0 latency 0.000046 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e580 finish 2019-12-07T19:34:14.647+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 111 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c55271800 con 0x558c54c5b400 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57f3e6e0 prio 196 cost 0 latency 0.000013 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.647+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57f3e6e0 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57f3e6e0 finish 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 171 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55251400 con 0x558c54c64000 2019-12-07T19:34:14.647+0000 7f8f24457700 15 osd.7 37 enqueue_op 0x558c57f3e840 prio 196 cost 0 latency 0.000009 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e840 prio 196 cost 0 latency 0.000026 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57f3e840 finish 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 63 ==== MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c548cc600 con 0x558c54c19800 2019-12-07T19:34:14.647+0000 7f8f24c58700 15 osd.7 37 enqueue_op 0x558c57eff760 prio 196 cost 0 latency 0.000013 epoch 37 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.647+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 64 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c58600 con 0x558c54c19800 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f24c58700 15 osd.7 37 enqueue_op 0x558c57eff8c0 prio 196 cost 0 latency 0.000005 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c57676000 2019-12-07T19:34:14.647+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57eff760 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 pg pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.647+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 37/28 ECSubWriteReply(tid=20, last_complete=37'24, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57eff8c0 prio 196 cost 0 latency 0.000028 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] 2019-12-07T19:34:14.647+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.647+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=20 reqid=client.4540.0:286 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:14.648+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c57eff8c0 finish 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.648+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:14.648+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:14.648+0000 7f8f06986700 10 osd.7 37 dequeue_op 0x558c57eff760 finish 2019-12-07T19:34:14.648+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.648+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 44 ==== MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c53aa8c00 con 0x558c54c64400 2019-12-07T19:34:14.648+0000 7f8f23c56700 15 osd.7 37 enqueue_op 0x558c578d3600 prio 196 cost 0 latency 0.000013 epoch 37 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.648+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) queued 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2) prio 196 cost 0 e37) pg 0x558c548ab000 2019-12-07T19:34:14.648+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d3600 prio 196 cost 0 latency 0.000033 MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] 2019-12-07T19:34:14.648+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 2019-12-07T19:34:14.648+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:head v=0'0 tt=0'0 tid=21 reqid=client.4540.0:305 roll_forward_to=37'4 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:14.648+0000 7f8f0297e700 10 osd.7 37 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:14.648+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 112 ==== pg_lease(1.bs3 pg_lease(ru 46.104938842s ub 54.105088836s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558c55271c00 con 0x558c54c5b400 2019-12-07T19:34:14.654+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.bs3 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s) 2019-12-07T19:34:14.654+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s)) prio 255 cost 10 e37) pg 0x558c54d9b000 2019-12-07T19:34:14.654+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s) 2019-12-07T19:34:14.654+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.104938842s ub 54.105088836s int 16s) 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.013986430s -> ru 46.090952412s 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] proc_lease readable_until now 46.090952412s 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.013511698s -> ruub 54.091577138s 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.091577138s 2019-12-07T19:34:14.654+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.bs0 pg_lease_ack(ruub 54.105088836s) e37/37) v1 -- 0x558c576dc000 con 0x558c54c5b400 2019-12-07T19:34:14.654+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:14.654+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 113 ==== pg_lease(1.3s4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558c55271c00 con 0x558c54c5b400 2019-12-07T19:34:14.655+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.3s4 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s) 2019-12-07T19:34:14.655+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s)) prio 255 cost 10 e37) pg 0x558c5513e000 2019-12-07T19:34:14.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s) 2019-12-07T19:34:14.655+0000 7f8f0317f700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.105685108s ub 54.105801978s int 16s) 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.013986430s -> ru 46.091698678s 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] proc_lease readable_until now 46.091698678s 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.013511698s -> ruub 54.092290280s 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.092290280s 2019-12-07T19:34:14.655+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.3s0 pg_lease_ack(ruub 54.105801978s) e37/37) v1 -- 0x558c54d38d20 con 0x558c54c5b400 2019-12-07T19:34:14.655+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:14.655+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.655+0000 7f8f0998c700 15 osd.7 37 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:14.655+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) queued 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:14.655+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 46.092358474s 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 46.092358474s 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 46.092358474s 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.2 ruub 46.092358474s 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 46.092358474s 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.092358474s (sent 54.092497800s) 2019-12-07T19:34:14.655+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.cs3 pg_lease(ru 46.092358474s ub 54.092497800s int 16s) e37/37) v1 -- 0x558c548cc600 con 0x558c54c64000 2019-12-07T19:34:14.655+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.cs4 pg_lease(ru 46.092358474s ub 54.092497800s int 16s) e37/37) v1 -- 0x558c54c59400 con 0x558c54c19800 2019-12-07T19:34:14.655+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.cs5 pg_lease(ru 46.092358474s ub 54.092497800s int 16s) e37/37) v1 -- 0x558c54c59e00 con 0x558c54c5bc00 2019-12-07T19:34:14.655+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.cs2 pg_lease(ru 46.092358474s ub 54.092497800s int 16s) e37/37) v1 -- 0x558c54c1ee00 con 0x558c54c5b400 2019-12-07T19:34:14.655+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.cs1 pg_lease(ru 46.092358474s ub 54.092497800s int 16s) e37/37) v1 -- 0x558c54c1fe00 con 0x558c54c19c00 2019-12-07T19:34:14.655+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:14.655+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 114 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 54.092497800s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c54d38d20 con 0x558c54c5b400 2019-12-07T19:34:14.656+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.cs0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) pg 0x558c57676000 2019-12-07T19:34:14.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 55 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 54.092497800s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c54d38d20 con 0x558c54c19c00 2019-12-07T19:34:14.656+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.cs0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 172 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 54.092497800s) e37/37) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54d38d20 con 0x558c54c64000 2019-12-07T19:34:14.656+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.cs0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.2 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 173 ==== pg_lease(1.9s4 pg_lease(ru 46.114035089s ub 54.114343743s int 16s) e37/37) v1 ==== 58+0+0 (crc 0 0 0) 0x558c548cc600 con 0x558c54c64000 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.092358474s (sent 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.9s4 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s) 2019-12-07T19:34:14.656+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) pg 0x558c57676000 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s)) prio 255 cost 10 e37) pg 0x558c5500a000 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f04181700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f04181700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.114035089s ub 54.114343743s int 16s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.2 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.092358474s (sent 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.021674644s -> ru 46.092360445s 2019-12-07T19:34:14.656+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 65 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 54.092497800s) e37/37) v1 ==== 42+0+0 (crc 0 0 0) 0x558c57f10780 con 0x558c54c19800 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] proc_lease readable_until now 46.092360445s 2019-12-07T19:34:14.656+0000 7f8f24c58700 15 osd.7 37 enqueue_peering_evt 1.cs0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.2 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.2 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.021170857s -> ruub 54.093172886s 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.093172886s 2019-12-07T19:34:14.656+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.9s0 pg_lease_ack(ruub 54.114343743s) e37/37) v1 -- 0x558c57f10d20 con 0x558c54c64000 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.2 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) pg 0x558c57676000 2019-12-07T19:34:14.656+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.2 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.092358474s (sent 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.2 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) pg 0x558c57676000 2019-12-07T19:34:14.656+0000 7f8f06986700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.2 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.2 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 46.092358474s 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.092358474s (sent 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 103 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 54.092497800s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c576dde00 con 0x558c54c5bc00 2019-12-07T19:34:14.656+0000 7f8f23c56700 15 osd.7 37 enqueue_peering_evt 1.cs0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.092497800s)) prio 255 cost 10 e37) pg 0x558c57676000 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.2 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 54.092497800s 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 54.092497800s (sent 54.092497800s) 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.657+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 104 ==== pg_lease(1.6s2 pg_lease(ru 46.115942144s ub 54.116153259s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558c575b8000 con 0x558c54c5bc00 2019-12-07T19:34:14.657+0000 7f8f23c56700 15 osd.7 37 enqueue_peering_evt 1.6s2 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115942144s ub 54.116153259s int 16s) 2019-12-07T19:34:14.657+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115942144s ub 54.116153259s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115942144s ub 54.116153259s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115942144s ub 54.116153259s int 16s)) prio 255 cost 10 e37) pg 0x558c54d5c000 2019-12-07T19:34:14.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115942144s ub 54.116153259s int 16s) 2019-12-07T19:34:14.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.115942144s ub 54.116153259s int 16s) 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.022333763s -> ru 46.093608381s 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] proc_lease readable_until now 46.093608381s 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.021974879s -> ruub 54.094178380s 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.094178380s 2019-12-07T19:34:14.657+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.6s0 pg_lease_ack(ruub 54.116153259s) e37/37) v1 -- 0x558c54d381e0 con 0x558c54c5bc00 2019-12-07T19:34:14.657+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:14.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.658+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 174 ==== pg_lease(1.7s4 pg_lease(ru 46.116162602s ub 54.116297009s int 16s) e37/37) v1 ==== 58+0+0 (crc 0 0 0) 0x558c548cc600 con 0x558c54c64000 2019-12-07T19:34:14.658+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.7s4 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s) 2019-12-07T19:34:14.658+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s)) prio 255 cost 10 e37) pg 0x558c54d5f000 2019-12-07T19:34:14.658+0000 7f8f0117b700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s) 2019-12-07T19:34:14.658+0000 7f8f0117b700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.116162602s ub 54.116297009s int 16s) 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.021674644s -> ru 46.094487958s 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] proc_lease readable_until now 46.094487958s 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.021170857s -> ruub 54.095126152s 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.095126152s 2019-12-07T19:34:14.658+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.7s0 pg_lease_ack(ruub 54.116297009s) e37/37) v1 -- 0x558c57f101e0 con 0x558c54c64000 2019-12-07T19:34:14.658+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:14.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:14.663+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.0 v2:172.21.15.131:6822/37079 45 ==== pg_lease(1.2s4 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) e37/37) v1 ==== 58+0+0 (crc 0 0 0) 0x558c575b8000 con 0x558c54c64400 2019-12-07T19:34:14.663+0000 7f8f23c56700 15 osd.7 37 enqueue_peering_evt 1.2s4 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.0 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) 2019-12-07T19:34:14.663+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.0 pg_lease(ru 46.119144108s ub 54.119305575s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.0 pg_lease(ru 46.119144108s ub 54.119305575s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.0 pg_lease(ru 46.119144108s ub 54.119305575s int 16s)) prio 255 cost 10 e37) pg 0x558c5731a000 2019-12-07T19:34:14.663+0000 7f8f07988700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.0 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) 2019-12-07T19:34:14.663+0000 7f8f07988700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.119144108s ub 54.119305575s int 16s) 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.019556338s -> ru 46.099587770s 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] proc_lease readable_until now 46.099587770s 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.019152845s -> ruub 54.100152730s 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.100152730s 2019-12-07T19:34:14.663+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_lease_ack(1.2s0 pg_lease_ack(ruub 54.119305575s) e37/37) v1 -- 0x558c575cde00 con 0x558c54c64400 2019-12-07T19:34:14.663+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:14.663+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.697+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 10 osd.7 37 tick_without_osd_lock 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fa60000/0x40000000/0x1680000000, data 0x5c0d62/0x59e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 25 osd.7 37 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574337+0000 last_rx_front 2019-12-07T19:34:11.574305+0000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 25 osd.7 37 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:11.574325+0000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 25 osd.7 37 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574316+0000 last_rx_front 2019-12-07T19:34:11.574282+0000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 25 osd.7 37 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574261+0000 last_rx_front 2019-12-07T19:34:11.574293+0000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 25 osd.7 37 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574240+0000 last_rx_front 2019-12-07T19:34:11.574226+0000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 25 osd.7 37 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574165+0000 last_rx_front 2019-12-07T19:34:11.574203+0000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 20 osd.7 37 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 20 osd.7 37 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 20 osd.7 37 scrub_load_below_threshold loadavg per cpu 0.04875 < max 5 = yes 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 20 osd.7 37 sched_scrub load_is_low=1 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 10 osd.7 37 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:14.701487+0000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 20 osd.7 37 sched_scrub done 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 10 osd.7 37 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 20 osd.7 37 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:14.700+0000 7f8f1fbbe700 10 osd.7 37 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:14.748+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.798+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.843+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 8 ==== osd_ping(ping e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.311047011s delta_ub 0.031069115s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c54c5a400 2019-12-07T19:34:14.843+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 8 ==== osd_ping(ping e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.311047011s delta_ub 0.031069115s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c575b8000 con 0x558c54c5ac00 2019-12-07T19:34:14.843+0000 7f8f24c58700 20 osd.7 37 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030690736s,0.031069115s]) 2019-12-07T19:34:14.843+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.280356275s delta_ub 18446744073.678860880s) v5 -- 0x558c54c58200 con 0x558c54c5a400 2019-12-07T19:34:14.843+0000 7f8f23c56700 20 osd.7 37 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030659769s,0.031069115s]) 2019-12-07T19:34:14.843+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.280387242s delta_ub 18446744073.678891847s) v5 -- 0x558c576b5a00 con 0x558c54c5ac00 2019-12-07T19:34:14.848+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.898+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.948+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.966+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 msgr2=0x558c53aa4580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 62 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c5476d400 msgr2=0x558c53aa6c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 63 2019-12-07T19:34:14.966+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 msgr2=0x558c53aa4580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c5476d400 msgr2=0x558c53aa6c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c5476d400 0x558c53aa6c00 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:14.966+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 0x558c53aa4580 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c5476d400 0x558c53aa6c00 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:14.966+0000 7f8f24c58700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 0x558c53aa4580 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18400 msgr2=0x558c552d9180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 69 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18400 msgr2=0x558c552d9180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18400 0x558c552d9180 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18400 0x558c552d9180 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:14.966+0000 7f8f0f998700 20 osd.7 37 heartbeat_reset con 0x558c5476d400 s 0x558c53a85860 2019-12-07T19:34:14.966+0000 7f8f0f998700 10 osd.7 37 heartbeat_reset failed hb con 0x558c5476d400 for osd.0, reopening 2019-12-07T19:34:14.966+0000 7f8f0f998700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 msgr2=0x558c53aa4580 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:34:14.966+0000 7f8f0f998700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c551b6800 0x558c53aa4580 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] reap_dead start 2019-12-07T19:34:14.966+0000 7f8f0f998700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:14.966+0000 7f8f0f998700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 0x558c54d02b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18c00 msgr2=0x558c54c36000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 86 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18c00 msgr2=0x558c54c36000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18c00 0x558c54c36000 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.131:0/37079 conn(0x558c54c18c00 0x558c54c36000 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:14.966+0000 7f8f1099a700 20 osd.7 37 heartbeat_reset con 0x558c551b6800 s 0 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 msgr2=0x558c54c6e000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 94 2019-12-07T19:34:14.966+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] reap_dead start 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 msgr2=0x558c54c6e000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 0x558c54c6e000 crc :-1 s=READY pgs=16 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:14.966+0000 7f8f0d994700 20 osd.7 37 heartbeat_reset con 0x558c54c18c00 s 0x558c54c0ab60 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 0x558c54c6e000 crc :-1 s=READY pgs=16 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:14.966+0000 7f8f0d994700 10 osd.7 37 heartbeat_reset closing (old) failed hb con 0x558c54c18c00 2019-12-07T19:34:14.966+0000 7f8f0e996700 20 osd.7 37 heartbeat_reset con 0x558c54c18400 s 0x558c54c0b040 2019-12-07T19:34:14.966+0000 7f8f0e996700 10 osd.7 37 heartbeat_reset closing (old) failed hb con 0x558c54c18400 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 msgr2=0x558c552b2580 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.131:6830/37079 2019-12-07T19:34:14.966+0000 7f8f0f998700 2 osd.7 37 ms_handle_refused con 0x558c551b7000 session 0x558c53a85860 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 msgr2=0x558c54d02b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.131:6826/37079 2019-12-07T19:34:14.966+0000 7f8f0f998700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.0 [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] for 21sec e37 v37) v4 -- 0x558c57580d80 con 0x558c5476d000 2019-12-07T19:34:14.966+0000 7f8f1099a700 2 osd.7 37 ms_handle_refused con 0x558c5526f000 session 0x558c53a85860 2019-12-07T19:34:14.966+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 0x558c54d02b00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:34:14.966+0000 7f8f1099a700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.0 [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] for 21sec e37 v37) v4 -- 0x558c575c6480 con 0x558c5476d000 2019-12-07T19:34:14.998+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.029+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c548d2c00 0x558c54c70680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:15.029+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c548d2c00 0x558c54c70680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:15.029+0000 7f8f23c56700 10 osd.7 37 ms_handle_authentication new session 0x558c54c5c340 con 0x558c548d2c00 entity client.admin addr 2019-12-07T19:34:15.029+0000 7f8f23c56700 10 osd.7 37 ms_handle_authentication session 0x558c54c5c340 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:15.030+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2098596510 conn(0x558c548d2c00 0x558c54c70680 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=client.4454 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:15.030+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4454 172.21.15.131:0/2098596510 1 ==== command(tid 14: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c575cde00 con 0x558c548d2c00 2019-12-07T19:34:15.030+0000 7f8f164bb700 20 osd.7 37 OSD::ms_dispatch: command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:15.030+0000 7f8f164bb700 10 osd.7 37 do_waiters -- start 2019-12-07T19:34:15.030+0000 7f8f164bb700 10 osd.7 37 do_waiters -- finish 2019-12-07T19:34:15.030+0000 7f8f164bb700 20 osd.7 37 _dispatch 0x558c575cde00 command(tid 14: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:15.030+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2098596510 -- command_reply(tid 14: 0 ) v1 -- 0x558c54ce0700 con 0x558c548d2c00 2019-12-07T19:34:15.048+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.071+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4454 172.21.15.131:0/2098596510 2 ==== command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c576dde00 con 0x558c548d2c00 2019-12-07T19:34:15.071+0000 7f8f164bb700 20 osd.7 37 OSD::ms_dispatch: command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:15.071+0000 7f8f164bb700 10 osd.7 37 do_waiters -- start 2019-12-07T19:34:15.071+0000 7f8f164bb700 10 osd.7 37 do_waiters -- finish 2019-12-07T19:34:15.071+0000 7f8f164bb700 20 osd.7 37 _dispatch 0x558c576dde00 command(tid 15: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:15.072+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2098596510 -- command_reply(tid 15: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce01c0 con 0x558c548d2c00 2019-12-07T19:34:15.075+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2098596510 conn(0x558c548d2c00 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 62 2019-12-07T19:34:15.075+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2098596510 conn(0x558c548d2c00 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:15.075+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2098596510 conn(0x558c548d2c00 0x558c54c70680 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:15.075+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2098596510 conn(0x558c548d2c00 0x558c54c70680 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:15.075+0000 7f8f164bb700 2 osd.7 37 ms_handle_reset con 0x558c548d2c00 session 0x558c54c5c340 2019-12-07T19:34:15.099+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.143+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 10 ==== osd_ping(ping e37 up_from 13 ping_stamp 2019-12-07T19:34:15.143990+0000/38.615709387s send_stamp 38.615709387s delta_ub 0.036485782s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c552d1800 2019-12-07T19:34:15.143+0000 7f8f24c58700 20 osd.7 37 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036039039s,0.036485782s]) 2019-12-07T19:34:15.143+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 10 ==== osd_ping(ping e37 up_from 13 ping_stamp 2019-12-07T19:34:15.143990+0000/38.615709387s send_stamp 38.615709387s delta_ub 0.036485782s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55271c00 con 0x558c552d1c00 2019-12-07T19:34:15.143+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e37 up_from 13 ping_stamp 2019-12-07T19:34:15.143990+0000/38.615709387s send_stamp 38.579670348s delta_ub 18446744073.673512577s) v5 -- 0x558c54c58200 con 0x558c552d1800 2019-12-07T19:34:15.143+0000 7f8f24457700 20 osd.7 37 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036013368s,0.036485782s]) 2019-12-07T19:34:15.143+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e37 up_from 13 ping_stamp 2019-12-07T19:34:15.143990+0000/38.615709387s send_stamp 38.579696019s delta_ub 18446744073.673538248s) v5 -- 0x558c54c1ee00 con 0x558c552d1c00 2019-12-07T19:34:15.149+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.167+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 msgr2=0x558c54d02b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.131:6826/37079 2019-12-07T19:34:15.167+0000 7f8f1099a700 2 osd.7 37 ms_handle_refused con 0x558c5526f000 session 0x558c53a85860 2019-12-07T19:34:15.167+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 0x558c54d02b00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.400000 2019-12-07T19:34:15.167+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 msgr2=0x558c552b2580 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.131:6830/37079 2019-12-07T19:34:15.167+0000 7f8f1099a700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.0 [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] for 21sec e37 v37) v4 -- 0x558c579e4000 con 0x558c5476d000 2019-12-07T19:34:15.167+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.400000 2019-12-07T19:34:15.167+0000 7f8f0f998700 2 osd.7 37 ms_handle_refused con 0x558c551b7000 session 0x558c53a85860 2019-12-07T19:34:15.167+0000 7f8f0f998700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.0 [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] for 21sec e37 v37) v4 -- 0x558c579e46c0 con 0x558c5476d000 2019-12-07T19:34:15.199+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.249+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.299+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.349+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.399+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.430+0000 7f8f21451700 10 osd.7 37 tick 2019-12-07T19:34:15.430+0000 7f8f21451700 10 osd.7 37 do_waiters -- start 2019-12-07T19:34:15.430+0000 7f8f21451700 10 osd.7 37 do_waiters -- finish 2019-12-07T19:34:15.430+0000 7f8f21451700 20 osd.7 37 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:15.449+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.499+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.521+0000 7f8f1219d700 20 osd.7 37 reports for 0 queries 2019-12-07T19:34:15.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c54e1e380 con 0x558c5476d800 2019-12-07T19:34:15.521+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:15.521+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x598000/0x539000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:15.521+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(2 pgs tid 0 v 0) v2 -- 0x558c54e1e700 con 0x558c5476d800 2019-12-07T19:34:15.549+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.568+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 msgr2=0x558c54d02b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.131:6826/37079 2019-12-07T19:34:15.568+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 0x558c54d02b00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.800000 2019-12-07T19:34:15.568+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 msgr2=0x558c552b2580 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.131:6830/37079 2019-12-07T19:34:15.568+0000 7f8f1099a700 2 osd.7 37 ms_handle_refused con 0x558c5526f000 session 0x558c53a85860 2019-12-07T19:34:15.568+0000 7f8f23c56700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.800000 2019-12-07T19:34:15.568+0000 7f8f1099a700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.0 [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] for 21sec e37 v37) v4 -- 0x558c579e4240 con 0x558c5476d000 2019-12-07T19:34:15.568+0000 7f8f0f998700 2 osd.7 37 ms_handle_refused con 0x558c551b7000 session 0x558c53a85860 2019-12-07T19:34:15.568+0000 7f8f0f998700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_failure(failed immediate osd.0 [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] for 21sec e37 v37) v4 -- 0x558c579e4900 con 0x558c5476d000 2019-12-07T19:34:15.600+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 585335 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 115 ==== pg_query2(1.bs1 1.bs1 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c58092500 con 0x558c54c5b400 2019-12-07T19:34:15.613+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.bs1 epoch_sent: 38 epoch_requested: 38 MQuery 1.bs1 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.613+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs1 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 116 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c58092500 con 0x558c54c5b400 2019-12-07T19:34:15.613+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.bs3 epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.613+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs1 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs1 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 map 38 > 37 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 38, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs1 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c54d9b000 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 38, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:15.613+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 117 ==== osd_map(38..38 src has 1..38) v4 ==== 1000+0+0 (unknown 0 0 0) 0x558c57570480 con 0x558c54c5b400 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 osd.7 37 OSD::ms_dispatch: osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 osd.7 37 do_waiters -- start 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 osd.7 37 do_waiters -- finish 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 osd.7 37 _dispatch 0x558c57570480 osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.613+0000 7f8f1199c700 3 osd.7 37 handle_osd_map epochs [38,38], i have 37, src has [1,38] 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 osd.7 37 handle_osd_map got inc map for epoch 38 2019-12-07T19:34:15.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:37b8e826:::osdmap.37:0# 0x0~0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1428 size 0x1428 (5160) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54ea3260 blob([0x4fc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57538770 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efc150 blob([!~1000,0x4fd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558c57efc1c0 sbid 0x0)) need 0x1000~428 cache has 0x[1000~428] 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:37b8e826:::osdmap.37:0# 0x0~1428 = 5160 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 osd.7 37 add_map_bl 37 5160 bytes 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 osd.7 37 got_full_map 38, nothing requested 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 osd.7 37 handle_osd_map pg_num_history pg_num_history(e38 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {38={}} 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 osd.7 37 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e38 [1,38] lci=[13,38]) 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54dde300 seq 29 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c5ab972f:::inc_osdmap.38:0# key 0x7f7fffffffffffffffc5ab97'/!inc_osdmap.38!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c5ab972f:::inc_osdmap.38:0# 0x0~3bc 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1169 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c5ab972f:::inc_osdmap.38:0# 0x0~3bc - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~3bc 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc44 on front/back, now 0x0~1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 1 blobs 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5a0000~1000/1000,1000,0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5a0000~1000] 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e63260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e63500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e63260 blob([0x5a0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63500 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57e63260) get_ref 0x0~3bc Blob(0x558c57e63260 blob([0x5a0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63500 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57e63260) get_ref init 0x1000, 1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~3bc: 0x0~3bc Blob(0x558c57e63260 blob([0x5a0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3bc) SharedBlob(0x558c57e63500 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57e63518 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3bc 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34b08) dirty_range mark inline shard dirty 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c5ab972f:::inc_osdmap.38:0# 0x0~3bc = 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3d48e826:::osdmap.38:0# key 0x7f7fffffffffffffff3d48e8'&!osdmap.38!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3d48e826:::osdmap.38:0# 0x0~1454 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1170 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3d48e826:::osdmap.38:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~454 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 2 blobs 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5a1000~2000/1000,2000,0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5a1000~2000] 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e63110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e63030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e63110 blob([0x5a1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63030 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57e63110) get_ref 0x0~1000 Blob(0x558c57e63110 blob([0x5a1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63030 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57e63110) get_ref init 0x1000, 1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57e63110 blob([0x5a1000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57e63030 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57e63048 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efc000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bea7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efc000 blob([!~1000,0x5a2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea7e0 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57efc000) get_ref 0x1000~454 Blob(0x558c57efc000 blob([!~1000,0x5a2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57bea7e0 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c57efc000) get_ref init 0x2000, 1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558c57efc000 blob([!~1000,0x5a2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c57bea7e0 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57bea7f8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1454 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34d88) dirty_range mark inline shard dirty 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3d48e826:::osdmap.38:0# 0x0~1454 = 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 0 blobs 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 118 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c58092500 con 0x558c54c5b400 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde300 0 blobs 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:15.613+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.3s4 epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:15.613+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:15.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde300 cost 23641 (4 ios * 4000 + 7641 bytes) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde300 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34a00,0x558c57a34c80 shared_blobs 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34b08) update #-1:c5ab972f:::inc_osdmap.38:0# 2019-12-07T19:34:15.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c5ab972f:::inc_osdmap.38:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34d88) update #-1:3d48e826:::osdmap.38:0# 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a34d88) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3d48e826:::osdmap.38:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde300 allocated 0x[5a0000~3000] released 0x[] 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 freelist allocate 0x5a0000~3000 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 prepare 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54dde300 2019-12-07T19:34:15.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 175 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c576e1800 con 0x558c54c64000 2019-12-07T19:34:15.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c5513e000 2019-12-07T19:34:15.614+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 38, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:15.614+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 176 ==== pg_query2(1.8s5 1.8s5 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (crc 0 0 0) 0x558c58092500 con 0x558c54c64000 2019-12-07T19:34:15.614+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.8s5 epoch_sent: 38 epoch_requested: 38 MQuery 1.8s5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.614+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.8s5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.614+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 38, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 178 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (crc 0 0 0) 0x558c58092500 con 0x558c54c64000 2019-12-07T19:34:15.614+0000 7f8f24457700 15 osd.7 37 enqueue_peering_evt 1.9s4 epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.614+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c5500a000 2019-12-07T19:34:15.614+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 177 ==== osd_map(38..38 src has 1..38) v4 ==== 1000+0+0 (crc 0 0 0) 0x558c57e03b00 con 0x558c54c64000 2019-12-07T19:34:15.614+0000 7f8f08189700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 38, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 osd.7 37 OSD::ms_dispatch: osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.614+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 osd.7 37 do_waiters -- start 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 osd.7 37 do_waiters -- finish 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 osd.7 37 _dispatch 0x558c57e03b00 osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.614+0000 7f8f1199c700 3 osd.7 37 handle_osd_map epochs [38,38], i have 38, src has [1,38] 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 osd.7 37 no new maps here, dropping 2019-12-07T19:34:15.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.8s5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.8s5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 map 38 > 37 2019-12-07T19:34:15.614+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 38, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.8s5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 aio_wait 2019-12-07T19:34:15.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde300 2019-12-07T19:34:15.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 io_done 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a34a00 had 1 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a34c80 had 1 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd4c9c~226c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd4c9c 0xd4c9c~226c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa4c9c 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc9c 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf08 and padding block with 0xf8 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd6f08 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000157076s (0.000043336s flush + 0.000113740s kv commit) 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde300> 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 kv_submitted 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde300 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54dde300 osr 0x558c54ea4b40 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) _discard drop 26 0x1b000~1000 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7 37 _committed_osd_maps 38..38 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7 37 advance to epoch 38 (<= last 38 <= newest_map 38) 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) prepare_write seq 27 0x1b000~1000 crc d342ec03 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) _discard drop 26 0x2000~1000 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57e672c0) prepare_write seq 27 0x2000~1000 crc cbf4ed54 2019-12-07T19:34:15.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.614+0000 7f8f141a1700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] mark_down_addrs [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- 0x558c54c64400 2019-12-07T19:34:15.614+0000 7f8f141a1700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x558c54c64400 0x558c54c6e000 unknown :-1 s=STANDBY pgs=16 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:34:15.614+0000 7f8f141a1700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 msgr2=0x558c552b2580 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2019-12-07T19:34:15.614+0000 7f8f141a1700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x558c551b7000 0x558c552b2580 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:15.614+0000 7f8f141a1700 1 -- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 msgr2=0x558c54d02b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2019-12-07T19:34:15.614+0000 7f8f141a1700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x558c5526f000 0x558c54d02b00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:15.614+0000 7f8f141a1700 7 osd.7 38 consume_map version 38 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:0.identify_splits_and_merges identify_splits_and_merges slot 1.8s5 has no pg and waiting_for_split 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.0s3 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.9s4 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.2s4 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.bs3 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs1 has no pg and waiting_for_split 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.3s4 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.cs0 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.4s0 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.ds2 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.6s2 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 identify_splits_and_merges 1.7s4 e37 to e38 pg_nums {18=16} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 38 prune_pg_created 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:0.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 pending_peering first epoch 38 <= 38, requeueing 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {38=} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:1.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 38 <= 38, requeueing 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {38=} 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:2.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:3.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 pending_peering first epoch 38 <= 38, requeueing 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {38=} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 pending_peering first epoch 38 <= 38, requeueing 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {38=} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 38 <= 38, requeueing 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {38=} 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:4.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 pending_peering first epoch 38 <= 38, requeueing 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c5500a000 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.4s0 to_process <> waiting <> waiting_peering {38=0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)>} 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:5.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:6.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c5513e000 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7:7.consume_map 38 (was 37) 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:15.614+0000 7f8f141a1700 10 osd.7 38 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.0s3 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.9s4 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 37 -> 38 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.2s4 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 37 -> 38 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 37 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.bs3 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.3s4 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.cs0 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.ds2 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] new interval newup [1,4,5,2,7,3] newacting [1,4,5,2,7,3] 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] new interval newup [4,1,2,5,7,3] newacting [4,1,2,5,7,3] 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.6s2 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c5731a000 2019-12-07T19:34:15.614+0000 7f8f141a1700 15 osd.7 38 enqueue_peering_evt 1.7s4 epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:15.615746+0000 35 0.000412 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:15.615748+0000 35 0.000296 2019-12-07T19:34:15.614+0000 7f8f141a1700 7 osd.7 38 activate_map version 38 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:15.615758+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:15.615758+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:15.615767+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c54d5c000 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 37 -> 38 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:15.615771+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 37 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 120 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c576e1800 con 0x558c54c5b400 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=38 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=38 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 37 -> 38 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [7,4,1,5,2,3] 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=38 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=38 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 37 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=38 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=38 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:15.614+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:15.615808+0000 35 0.000329 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=38 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] new interval newup [4,1,2,5,7,3] newacting [4,1,2,5,7,3] 2019-12-07T19:34:15.614+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=38 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] new interval newup [1,4,5,2,7,3] newacting [1,4,5,2,7,3] 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=38 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:15.615817+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=38 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:15.614+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=4 lpr=38 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=4 lpr=38 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:15.615825+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.614+0000 7f8f07187700 15 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,3]p4(0) r=4 lpr=38 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:15.614+0000 7f8f04181700 15 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,7,3]p1(0) r=4 lpr=38 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=28 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=38 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c55141000 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=38 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(28-37 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,0](4)) up_thru 28 up_from 13 last_epoch_clean 29 interval(28-37 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,0](4) maybe_went_rw) : primary up 13-28 includes interval 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 37 while current is 38 2019-12-07T19:34:15.614+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=38 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(28-37 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,3](1)) up_thru 28 up_from 13 last_epoch_clean 29 interval(28-37 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,3](1) maybe_went_rw) : primary up 13-28 includes interval 2019-12-07T19:34:15.614+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.614+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=38 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [7,4,1,5,2,3] 2019-12-07T19:34:15.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] noting past ([28,37] all_participants=0(5),1(1),2(2),4(0),5(3),7(4) intervals=([28,37] acting 0(5),1(1),2(2),4(0),5(3),7(4))) 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 37 -> 38 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=38 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] noting past ([28,37] all_participants=0(3),1(0),3(5),4(1),5(2),7(4) intervals=([28,37] acting 0(3),1(0),3(5),4(1),5(2),7(4))) 2019-12-07T19:34:15.614+0000 7f8f1199c700 2 osd.7 38 ms_handle_reset con 0x558c54c64400 session 0x558c54c0b520 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=4 lpr=38 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 37 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.614+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 119 ==== osd_map(38..38 src has 1..38) v4 ==== 1000+0+0 (unknown 0 0 0) 0x558c57e03d40 con 0x558c54c5b400 2019-12-07T19:34:15.614+0000 7f8f03980700 15 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,4,1,5,2,3]p7(0) r=4 lpr=38 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 osd.7 38 OSD::ms_dispatch: osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:15.614+0000 7f8f1199c700 20 osd.7 38 _dispatch 0x558c57e03d40 osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:15.614+0000 7f8f1199c700 3 osd.7 38 handle_osd_map epochs [38,38], i have 38, src has [1,38] 2019-12-07T19:34:15.614+0000 7f8f1199c700 10 osd.7 38 no new maps here, dropping 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] new interval newup [2,1,7,3,4,5] newacting [2,1,7,3,4,5] 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active pruub 54.092290280s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.092290280s (mnow 39.051335414s + 15.040954866s) 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active pruub 54.093172886s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.093172886s (mnow 39.051340809s + 15.041832077s) 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(28-37 up [0,4,1,5,7,3](0) acting [0,4,1,5,7,3](0)) up_thru 28 up_from 13 last_epoch_clean 29 interval(28-37 up [0,4,1,5,7,3](0) acting [0,4,1,5,7,3](0) maybe_went_rw) : primary up 13-28 includes interval 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active pruub 54.092290280s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:15.614+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:15.615912+0000 65 0.000664 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active pruub 54.093172886s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] noting past ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4))) 2019-12-07T19:34:15.614+0000 7f8f07187700 1 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=37'13 active pruub 54.092290280s@ mbc={} ps=[a~1]] start_peering_interval up [4,1,2,5,7,0] -> [4,1,2,5,7,3], acting [4,1,2,5,7,0] -> [4,1,2,5,7,3], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:15.614+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:15.615923+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:15.614+0000 7f8f04181700 1 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=34'17 active pruub 54.093172886s@ mbc={} ps=[a~1]] start_peering_interval up [1,4,5,0,7,3] -> [1,4,5,2,7,3], acting [1,4,5,0,7,3] -> [1,4,5,2,7,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:15.614+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:15.615931+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 37 -> 38 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] init_hb_stamps now [] 2019-12-07T19:34:15.614+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active pruub 54.100152730s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.100152730s (mnow 39.051386741s + 15.048765989s) 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,3,1,5,2,4] 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=38 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active pruub 54.100152730s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] state: Active advmap interval change, fast return 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active pruub 54.100152730s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=38 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=38 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,3,1,5,2,4] 2019-12-07T19:34:15.614+0000 7f8f03980700 1 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=29'15 active pruub 54.100152730s@ mbc={} ps=[a~1]] start_peering_interval up [0,4,1,5,7,3] -> [7,4,1,5,2,3], acting [0,4,1,5,7,3] -> [7,4,1,5,2,3], acting_primary 0(0) -> 7, up_primary 0(0) -> 7, role 4 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=38 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] new interval newup [2,1,7,3,4,5] newacting [2,1,7,3,4,5] 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=38 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:15.614+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active+clean ps=[a~1]] exit Started/Primary/Active/Clean 2019-12-07T19:34:15.615987+0000 65 0.002195 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=2 lpr=38 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] exit Started/Primary/Active 2019-12-07T19:34:15.615999+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f0217d700 15 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,5]p2(0) r=2 lpr=38 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown pruub 54.092290280s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown pruub 54.093172886s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:15.614+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] exit Started/Primary 2019-12-07T19:34:15.616014+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.614+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:15.614+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-37 up [2,1,7,3,4,0](2) acting [2,1,7,3,4,0](2)) up_thru 28 up_from 13 last_epoch_clean 19 interval(18-37 up [2,1,7,3,4,0](2) acting [2,1,7,3,4,0](2) maybe_went_rw) : primary up 13-28 includes interval 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] noting past ([18,37] all_participants=0(5),1(1),2(0),3(3),4(4),7(2) intervals=([18,37] acting 0(5),1(1),2(0),3(3),4(4),7(2))) 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.614+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:15.616042+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.614+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040954866s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=18 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041832077s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown pruub 54.100152730s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a150] 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=38 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active pruub 52.054860248s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 52.054860248s (mnow 39.051506688s + 13.003353560s) 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:15.616069+0000 1 0.000298 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=38 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:15.616070+0000 1 0.000303 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:15.614+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=38 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active pruub 52.054860248s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=38 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,3,1,5,2,4] 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.614+0000 7f8f07187700 1 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=38 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:15.614+0000 7f8f04181700 1 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.614+0000 7f8f0217d700 1 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 luod=0'0 crt=21'6 active pruub 52.054860248s@ mbc={} ps=[a~1]] start_peering_interval up [2,1,7,3,4,0] -> [2,1,7,3,4,5], acting [2,1,7,3,4,0] -> [2,1,7,3,4,5], acting_primary 2(0) -> 2, up_primary 2(0) -> 2, role 2 -> 2, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=0 lpr=38 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 38 clear_ready_to_merge 1.4s0 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:15.616099+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:15.616102+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:15.614+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:15.614+0000 7f8f0297e700 15 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,3,1,5,2,4]p7(0) r=0 lpr=38 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048765989s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.614+0000 7f8f04181700 5 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:15.614+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 37 while current is 38 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] release_backoffs [1:b0000000::::0,1:c0000000::::head) 2019-12-07T19:34:15.614+0000 7f8f04181700 20 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 38 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.614+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:15.616125+0000 1 0.000295 2019-12-07T19:34:15.614+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-37 up [7,0,1,5,2,4](7) acting [7,0,1,5,2,4](7)) up_thru 28 up_from 13 last_epoch_clean 19 interval(18-37 up [7,0,1,5,2,4](7) acting [7,0,1,5,2,4](7) maybe_went_rw) : primary up 13-28 includes interval 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MQuery 1.3s4 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MQuery 1.9s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] noting past ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0))) 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:15.614+0000 7f8f03980700 1 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:15.614+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:15.614+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 38 send_incremental_map 37 -> 38 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:15.616154+0000 0 0.000000 2019-12-07T19:34:15.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:15.614+0000 7f8f04181700 10 osd.7 38 send_incremental_map 37 -> 38 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:15.614+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c5ab972f:::inc_osdmap.38:0# 0x0~0 2019-12-07T19:34:15.614+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c54ea3ce0,0x558c5523a230,0x558c54677f10,0x558c5523a150,0x558c5523a3f0] 2019-12-07T19:34:15.614+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~3bc size 0x3bc (956) 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:15.614+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:15.614+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 38 2019-12-07T19:34:15.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active pruub 52.052753396s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 52.052753396s (mnow 39.051610738s + 13.001142658s) 2019-12-07T19:34:15.614+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57e63260 blob([0x5a0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3bc) SharedBlob(0x558c57e63500 sbid 0x0)) need 0x0~3bc cache has 0x[0~3bc] 2019-12-07T19:34:15.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.614+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c5ab972f:::inc_osdmap.38:0# 0x0~3bc = 956 2019-12-07T19:34:15.614+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c54d5f000 2019-12-07T19:34:15.614+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:15.614+0000 7f8f07187700 10 osd.7 38 add_map_inc_bl 38 956 bytes 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active pruub 52.052753396s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active pruub 52.052753396s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:15.614+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown pruub 52.054860248s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:15.614+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(38..38 src has 1..38) v4 -- 0x558c576b3200 con 0x558c54c5b400 2019-12-07T19:34:15.615+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(38..38 src has 1..38) v4 -- 0x558c57570d80 con 0x558c54c64000 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 38 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 37 -> 38 (shared) 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active pruub 52.052753396s@ mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:34:56.363661+0000, must = 0 2019-12-07T19:34:15.614+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:15.615+0000 7f8f04181700 10 osd.7 38 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 37 -> 38 (shared) 2019-12-07T19:34:15.615+0000 7f8f03980700 10 osd.7 38 send_incremental_map 35 -> 38 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 active pruub 52.052753396s@ mbc={} ps=[a~1]] start_peering_interval up [7,0,1,5,2,4] -> [7,3,1,5,2,4], acting [7,0,1,5,2,4] -> [7,3,1,5,2,4], acting_primary 7(0) -> 7, up_primary 7(0) -> 7, role 0 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 37 -> 38 2019-12-07T19:34:15.615+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:38 sent:38 1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040788460s) 4->0 ([28,37] all_participants=0(5),1(1),2(2),4(0),5(3),7(4) intervals=([28,37] acting 0(5),1(1),2(2),4(0),5(3),7(4)))) e38/38) v1 -- 0x558c548a3800 con 0x558c54c5b400 2019-12-07T19:34:15.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(36..38 src has 1..38) v4 -- 0x558c54dd1200 con 0x558c53ae5400 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 37 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.615+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:38 sent:38 1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041668129s) 4->0 ([28,37] all_participants=0(3),1(0),3(5),4(1),5(2),7(4) intervals=([28,37] acting 0(3),1(0),3(5),4(1),5(2),7(4)))) e38/38) v1 -- 0x558c576dfc00 con 0x558c54c64000 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 21'2 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:15.615+0000 7f8f03980700 10 osd.7 38 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 35 -> 38 (shared) 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] new interval newup [1,4,3,5,7,2] newacting [1,4,3,5,7,2] 2019-12-07T19:34:15.615+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:38 sent:38 1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) 4->0 ([28,37] all_participants=0(5),1(1),2(2),4(0),5(3),7(4) intervals=([28,37] acting 0(5),1(1),2(2),4(0),5(3),7(4)))) e38/38) v1 -- 0x558c57450700 con 0x558c54c5b400 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:15.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 -- 0x558c576b8000 con 0x558c53ae5400 2019-12-07T19:34:15.615+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:38 sent:38 1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) 4->0 ([28,37] all_participants=0(3),1(0),3(5),4(1),5(2),7(4) intervals=([28,37] acting 0(3),1(0),3(5),4(1),5(2),7(4)))) e38/38) v1 -- 0x558c548a1c00 con 0x558c54c64000 2019-12-07T19:34:15.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:15.616243+0000 35 0.000390 2019-12-07T19:34:15.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5740b200 seq 25 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c548f6300 seq 39 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003353560s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:15.615+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c548f6900 seq 40 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:15.616251+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:15.615+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:15.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:15.616256+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:15.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:15.615+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:15.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:15.615+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:15.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.615+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:15.615+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:15.616260+0000 1 0.000325 2019-12-07T19:34:15.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b200 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:15.615+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6900 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:15.615+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6900 onodes shared_blobs 2019-12-07T19:34:15.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b200 onodes shared_blobs 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=38 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.615+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6900 allocated 0x[] released 0x[] 2019-12-07T19:34:15.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b200 allocated 0x[] released 0x[] 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes shared_blobs 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:15.615+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=38 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=38 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:15.615+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 prepare 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=38 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] new interval newup [1,4,3,5,7,2] newacting [1,4,3,5,7,2] 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=38 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:15.615+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6900 2019-12-07T19:34:15.615+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 io_done 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=4 lpr=38 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 prepare 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b200 2019-12-07T19:34:15.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 io_done 2019-12-07T19:34:15.615+0000 7f8f05183700 15 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,5,7,2]p1(0) r=4 lpr=38 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs1 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] publish_stats_to_osd 37:56 2019-12-07T19:34:15.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs1 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 38 handle_pg_query_nopg 1.bs1 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 38 pg 1.bs1 dne 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:15.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(28-37 up [1,4,3,0,7,2](1) acting [1,4,3,0,7,2](1)) up_thru 28 up_from 13 last_epoch_clean 29 interval(28-37 up [1,4,3,0,7,2](1) acting [1,4,3,0,7,2](1) maybe_went_rw) : primary up 13-28 includes interval 2019-12-07T19:34:15.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c5500a000 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.615+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:38 sent:38 1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 38) v7 -- 0x558c575cd0e0 con 0x558c54c5b400 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] noting past ([28,37] all_participants=0(3),1(0),2(5),3(2),4(1),7(4) intervals=([28,37] acting 0(3),1(0),2(5),3(2),4(1),7(4))) 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:15.615+0000 7f8f04181700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.615+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c54d9b000 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active pruub 54.095126152s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.095126152s (mnow 39.051777734s + 15.043348418s) 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:15.615+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active pruub 54.095126152s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] [7,0,1,5,2,4] -> [7,3,1,5,2,4], replicas changed 2019-12-07T19:34:15.615+0000 7f8f05183700 1 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 luod=0'0 crt=27'10 active pruub 54.095126152s@ mbc={} ps=[a~1]] start_peering_interval up [1,4,3,0,7,2] -> [1,4,3,5,7,2], acting [1,4,3,0,7,2] -> [1,4,3,5,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:15.615+0000 7f8f0217d700 1 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 37 -> 38 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:15.615+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:15.616359+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 37 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.615+0000 7f8f0217d700 5 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 38 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,7,2,3,1,5] 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:15.616393+0000 36 0.000318 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown pruub 54.095126152s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:15.616402+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] update_heartbeat_peers 0,1,2,4,5,7 -> 1,2,3,4,5,7 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 38 need_heartbeat_peer_update 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:15.616410+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 osd.7 38 send_incremental_map 37 -> 38 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:15.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:15.616415+0000 1 0.000413 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.615+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(38..38 src has 1..38) v4 -- 0x558c575d4b40 con 0x558c54c19800 2019-12-07T19:34:15.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043348418s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=38 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 osd.7 38 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 37 -> 38 (shared) 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:15.616438+0000 1 0.000180 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=38 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] state: transitioning to Primary 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=38 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:15.615+0000 7f8f05183700 1 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:15.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:15.616447+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:38 sent:38 1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) 2->0 ([18,37] all_participants=0(5),1(1),2(0),3(3),4(4),7(2) intervals=([18,37] acting 0(5),1(1),2(0),3(3),4(4),7(2)))) e38/38) v1 -- 0x558c548a3100 con 0x558c54c19800 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:15.616454+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=38 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,7,2,3,1,5] 2019-12-07T19:34:15.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] enter Started/Primary 2019-12-07T19:34:15.615+0000 7f8f05183700 5 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=38 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d21500 seq 23 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 52.052753396s@ mbc={} ps=[a~1]] enter Started/Primary/Peering 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=3 lpr=38 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] set_last_peering_reset 38 2019-12-07T19:34:15.615+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:15.615+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0297e700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@ mbc={} ps=[a~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:15.615+0000 7f8f07187700 15 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,7,2,3,1,5]p4(0) r=3 lpr=38 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:15.615+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:15.615+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:15.615+0000 7f8f0918b700 20 osd.7 38 agent_entry empty queue 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@ mbc={} ps=[a~1]] build_prior all_probe 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes shared_blobs 2019-12-07T19:34:15.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[] released 0x[] 2019-12-07T19:34:15.615+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:38 sent:38 1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) 4->0 ([28,37] all_participants=0(3),1(0),2(5),3(2),4(1),7(4) intervals=([28,37] acting 0(3),1(0),2(5),3(2),4(1),7(4)))) e38/38) v1 -- 0x558c576b8700 con 0x558c54c64000 2019-12-07T19:34:15.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@ mbc={} ps=[a~1]] build_prior maybe_rw interval:18, acting: 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:34:15.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(28-37 up [4,0,2,7,1,5](4) acting [4,0,2,7,1,5](4)) up_thru 28 up_from 13 last_epoch_clean 29 interval(28-37 up [4,0,2,7,1,5](4) acting [4,0,2,7,1,5](4) maybe_went_rw) : primary up 13-28 includes interval 2019-12-07T19:34:15.615+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@ mbc={} ps=[a~1]] build_prior prior osd.0 is down 2019-12-07T19:34:15.615+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740a300 seq 39 2019-12-07T19:34:15.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] noting past ([28,37] all_participants=0(1),1(4),2(2),4(0),5(5),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) 2019-12-07T19:34:15.615+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@ mbc={} ps=[a~1]] build_prior final: probe 1(2),2(4),3(1),4(5),5(3),7(0) down 0 blocked_by {} 2019-12-07T19:34:15.615+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:15.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:15.615+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:15.615+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@ mbc={} ps=[a~1]] up_thru 28 < same_since 38, must notify monitor 2019-12-07T19:34:15.615+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.615+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes shared_blobs 2019-12-07T19:34:15.615+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@0 mbc={} ps=[a~1]] state: querying info from osd.1(2) 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] init_hb_stamps now [] 2019-12-07T19:34:15.615+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:34:15.615+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:34:15.615+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@0 mbc={} ps=[a~1]] state: querying info from osd.2(4) 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active pruub 54.091577138s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.091577138s (mnow 39.051973721s + 15.039603417s) 2019-12-07T19:34:15.615+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@0 mbc={} ps=[a~1]] state: querying info from osd.3(1) 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active pruub 54.091577138s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@0 mbc={} ps=[a~1]] state: querying info from osd.4(5) 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@0 mbc={} ps=[a~1]] state: querying info from osd.5(3) 2019-12-07T19:34:15.615+0000 7f8f07187700 1 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 luod=0'0 crt=21'12 active pruub 54.091577138s@ mbc={} ps=[a~1]] start_peering_interval up [4,0,2,7,1,5] -> [4,7,2,3,1,5], acting [4,0,2,7,1,5] -> [4,7,2,3,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 3 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@0 mbc={} ps=[a~1]] check_prior_readable_down_osds prior_readable_down_osds osd.0 is dead as of epoch 38 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 52.052753396s@ mbc={} ps=[a~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:15.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] publish_stats_to_osd 37:57 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 37 while current is 38 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.001142658s) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown pruub 54.091577138s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:02.338936+0000, must = 0 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.4s2 1.4s2 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c57efa280 con 0x558c54c64000 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.4s4 1.4s4 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c57efa500 con 0x558c54c19800 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039603417s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 send_incremental_map 37 -> 38 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(38..38 src has 1..38) v4 -- 0x558c54d2efc0 con 0x558c54c5bc00 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:15.616693+0000 1 0.000283 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 37 -> 38 (shared) 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.4s1 1.4s1 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c57efa780 con 0x558c54c5bc00 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.4s5 1.4s5 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c57efaa00 con 0x558c54c5b400 2019-12-07T19:34:15.615+0000 7f8f07187700 1 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 send_incremental_map 37 -> 38 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:15.616721+0000 0 0.000000 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(38..38 src has 1..38) v4 -- 0x558c57570900 con 0x558c54c19c00 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 37 -> 38 (shared) 2019-12-07T19:34:15.615+0000 7f8f07187700 5 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.4s3 1.4s3 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c57efac80 con 0x558c54c19c00 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d20600 seq 19 2019-12-07T19:34:15.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MQuery 1.bs3 from 4(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes shared_blobs 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:15.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru now 38 (was 28), currently 28 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 send_alive up_thru currently 28 want 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 send_alive want 38 2019-12-07T19:34:15.615+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:38 sent:38 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039448742s) 3->0 ([28,37] all_participants=0(1),1(4),2(2),4(0),5(5),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e38/38) v1 -- 0x558c57450e00 con 0x558c54c5b400 2019-12-07T19:34:15.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_alive(want up_thru 38 have 38) v1 -- 0x558c54d0e5a0 con 0x558c5476d000 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c57676000 2019-12-07T19:34:15.615+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:38 sent:38 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) 3->0 ([28,37] all_participants=0(1),1(4),2(2),4(0),5(5),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e38/38) v1 -- 0x558c57452a00 con 0x558c54c5b400 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c548f7800 seq 19 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 37 -> 38 2019-12-07T19:34:15.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 37 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes shared_blobs 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[] released 0x[] 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean ps=[a~1]] state: Active advmap 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:34:15.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:34:15.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] on_active_advmap new removed_snaps [d~1], snap_trimq now [d~1] 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c5513e000 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:15.615+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] agent_stop 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] activate_map 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] state: Active: handling ActMap 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] Active: kicking snap trim 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] exit NotTrimming 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] enter Trimming 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] publish_stats_to_osd 37:113 2019-12-07T19:34:15.615+0000 7f8f0297e700 7 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.615+0000 7f8f0297e700 15 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] publish_stats_to_osd 37:114 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] activate_map: Not dirtying info: last_persisted is 37 while current is 38 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] SnapTrimmer state: NotTrimming: trimming d 2019-12-07T19:34:15.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim_wait trimq=[d~1] ps=[a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 2019-12-07T19:34:15.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] publish_stats_to_osd 38:115 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:25.361074+0000, must = 0 2019-12-07T19:34:15.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:15.615+0000 7f8f0b990700 10 osd.7 38 queueing pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean trimq=[d~1] ps=[a~1]] for snaptrim 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.615+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=38) prio 5 cost 1048576 e38) 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=38) prio 5 cost 1048576 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] publish_stats_to_osd 38:116 2019-12-07T19:34:15.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=38) prio 5 cost 1048576 e38) pg 0x558c57676000 2019-12-07T19:34:15.615+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 0+0+0 (unknown 0 0 0) 0x558c576b8000 con 0x558c53ae5400 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] snap_trimmer posting 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap d 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:15.615+0000 7f8f1119b700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.615+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 7 submitting 7 deferred done 0 stable 0 2019-12-07T19:34:15.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54e89280) upper_bound after SNA_1_000000000000000D_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.C' 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.8s5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.8s5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:15.615+0000 7f8f0898a700 10 osd.7 38 handle_pg_query_nopg 1.8s5 2019-12-07T19:34:15.615+0000 7f8f0898a700 10 osd.7 38 pg 1.8s5 dne 2019-12-07T19:34:15.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2538 < min_flush_size 524288 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54e89280) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' 2019-12-07T19:34:15.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:15.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3807 < min_flush_size 524288 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000D_.0_0000000000000001.C) returns 0 SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:15.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:15.615+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:38 sent:38 1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 38) v7 -- 0x558c576dd680 con 0x558c54c64000 2019-12-07T19:34:15.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4344 < min_flush_size 524288 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:15.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:15.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5087 < min_flush_size 524288 2019-12-07T19:34:15.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:15.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6356 < min_flush_size 524288 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54e89280) upper_bound after SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. key 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c5500d000 2019-12-07T19:34:15.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:15.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7645 < min_flush_size 524288 2019-12-07T19:34:15.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54e89280) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..) returns 0 SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8914 < min_flush_size 524288 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] handle_advance_map: 38 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:15.616+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 37 -> 38 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3859939328 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 37 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_object_context: found obc in cache: 0x558c576b2b40 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998559) bluestore 86 GiB free (0.959938), bluefs_ratio 0.0399475 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8937 < min_flush_size 524288 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8937 < min_flush_size 524288 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_object_context: 0x558c576b2b40 1:3a5a3f5b:::smithi13140796-23:e rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:e(37'23 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:15.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:15.616+0000 7f8f0898a700 20 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd6f08~22e9 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd6f08 0xd6f08~22e9 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.616+0000 7f8f0898a700 20 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.616+0000 7f8f0898a700 20 osd.7 38 advance_pg new pool opts old pool opts 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa6f08 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf08 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1f1 and padding block with 0xe0f 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] handle_activate_map: 38 2019-12-07T19:34:15.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 NullEvt) prio 255 cost 10 e38) pg 0x558c54d9b000 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 1:3a5a3f5b:::smithi13140796-23:e old_snaps c,d,e old snapset e=[]:{e=[e,d,c]} 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:15.616+0000 7f8f0898a700 20 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 37 while current is 38 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] simple_opc_create 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:15.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 1:3a5a3f5b:::smithi13140796-23:e snaps c,d,e -> c,e 2019-12-07T19:34:15.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 NullEvt 2019-12-07T19:34:15.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:15.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 1:3a5a3f5b:::smithi13140796-23:e new snapset e=[]:{e=[e,c]} on 1:3a5a3f5b:::smithi13140796-23:head(37'24 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:15.616+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(36..38 src has 1..38) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c54dd1200 con 0x558c53ae5400 2019-12-07T19:34:15.616+0000 7f8f1199c700 20 osd.7 38 OSD::ms_dispatch: osd_map(36..38 src has 1..38) v4 2019-12-07T19:34:15.616+0000 7f8f1199c700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:15.616+0000 7f8f1199c700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:15.616+0000 7f8f1199c700 20 osd.7 38 _dispatch 0x558c54dd1200 osd_map(36..38 src has 1..38) v4 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 1:3a5a3f5b:::smithi13140796-23:e writing updated snapset on 1:3a5a3f5b:::smithi13140796-23:head, snapset is e=[]:{e=[e,c]} 2019-12-07T19:34:15.616+0000 7f8f1199c700 3 osd.7 38 handle_osd_map epochs [36,38], i have 38, src has [1,38] 2019-12-07T19:34:15.616+0000 7f8f1199c700 10 osd.7 38 no new maps here, dropping 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] new_repop rep_tid 22 (no op) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] new_repop: repgather(0x558c55159440 0'0 rep_tid=22 committed?=0 r=0) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] simple_opc_submit 0x558c55159440 2019-12-07T19:34:15.616+0000 7f8f0297e700 7 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] issue_repop rep_tid 22 o 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:e v=38'26 tt=0'0 tid=22 reqid=osd.7.0:22 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xd91f1 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 7 cleaned 0 in 0.000269830s (0.000000103s flush + 0.000269727s kv commit) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300,0x558c5740b200,0x558c548f6900,0x558c54d21500,0x558c5740a300,0x558c54d20600,0x558c548f7800> 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key = 0 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 kv_submitted 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b200 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:e projected size 1474560 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 finishing 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b200 onodes 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b200 done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b200 [] 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 kv_submitted 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6900 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 finishing 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6900 onodes 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6900 [] 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 finishing 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:34:15.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=38'26 tt=0'0 tid=22 reqid=osd.7.0:22 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:e v=38'26 tt=0'0 tid=22 reqid=osd.7.0:22 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 no pg, should create on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=38'26 tt=0'0 tid=22 reqid=osd.7.0:22 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:e, current size is 1474560 buffers are {} 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 38 _make_pg 1.2s0 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): new_size start 1474560 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2(unlocked)] enter Initial 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): to_append: {} 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:e resetting hinfo to logical size 1474560 2019-12-07T19:34:15.616+0000 7f8f07988700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:15.616+0000 7f8f07988700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:15.616+0000 7f8f07988700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): new_size start 507904 2019-12-07T19:34:15.616+0000 7f8f07988700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:15.616+0000 7f8f07988700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): to_append: {} 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2(unlocked)] enter NotTrimming 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.2s0_head fresh osr 0x558c54e10dc0 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 0 up [7,4,1,5,2,3] acting [7,4,1,5,2,3] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s past_intervals ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4))) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:e={},1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:e v=38'26 tt=0'0 tid=22 reqid=osd.7.0:22 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] init_hb_stamps now [0x558c5523a3f0,0x558c5523a230,0x558c54677f10,0x558c5523a150,0x558c54ea3ce0] 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 39.053104722s + 0s) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:e,{} 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:15.616+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:15.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:15.617708+0000 0 0.000000 2019-12-07T19:34:15.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 38/28 ECSubWrite(tid=22, reqid=osd.7.0:22, at_version=38'26, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c57e7b000 con 0x558c54c64000 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=0 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 38 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:15.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 38/28 ECSubWrite(tid=22, reqid=osd.7.0:22, at_version=38'26, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c57e7b800 con 0x558c54c19800 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_activate_map: 38 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] activate_map 2019-12-07T19:34:15.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 38/28 ECSubWrite(tid=22, reqid=osd.7.0:22, at_version=38'26, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c57e7c000 con 0x558c54c5bc00 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] update_heartbeat_peers -> 1,2,3,4,5,7 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 38 need_heartbeat_peer_update 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:15.617759+0000 1 0.000033 2019-12-07T19:34:15.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 38/28 ECSubWrite(tid=22, reqid=osd.7.0:22, at_version=38'26, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c57e7c800 con 0x558c54c5b400 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Start 2019-12-07T19:34:15.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 38/28 ECSubWrite(tid=22, reqid=osd.7.0:22, at_version=38'26, trim_to=0'0, roll_forward_to=37'24)) v2 -- 0x558c57e7d000 con 0x558c54c19c00 2019-12-07T19:34:15.616+0000 7f8f07988700 1 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: transitioning to Primary 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Start 2019-12-07T19:34:15.617782+0000 0 0.000000 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 snap_mapper.update_snaps 1:3a5a3f5b:::smithi13140796-23:e c,e was 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary/Peering 2019-12-07T19:34:15.616+0000 7f8f07988700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:e c,d,e 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 snap_mapper.set_snaps 1:3a5a3f5b:::smithi13140796-23:e c,e 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior all_probe 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 snap_mapper.update_snaps rm SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior maybe_rw interval:28, acting: 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior prior osd.0 is down 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 trim 38'25 (37'23) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 trim 38'26 (37'24) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] build_prior final: probe 1(2),2(4),3(5),4(1),5(3),7(0),7(4) down 0 blocked_by {} 2019-12-07T19:34:15.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] up_thru 28 < same_since 38, must notify monitor 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.1(2) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 37'24 (0'0,37'24] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] append_log log((0'0,37'24], crt=37'24) [38'25 (37'23) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,38'26 (37'24) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.2(4) 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.3(5) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'25 (0'0,38'25] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 lua=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] add_log_entry 38'25 (37'23) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.4(1) 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.5(3) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 lua=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] add_log_entry 38'26 (37'24) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: querying info from osd.7(4) 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 lua=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] append_log approx pg log length = 26 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] check_prior_readable_down_osds prior_readable_down_osds osd.0 is dead as of epoch 38 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 lua=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:15.616+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 38'25, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:15.616+0000 7f8f07988700 15 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] activate_map: Not dirtying info: last_persisted is 38 while current is 38 2019-12-07T19:34:15.616+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c574a0280 con 0x558c54c64000 2019-12-07T19:34:15.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c5793a500 con 0x558c54c19800 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea1200 seq 32 2019-12-07T19:34:15.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.2s5 1.2s5 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c58093680 con 0x558c54c5bc00 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 1 keys 2019-12-07T19:34:15.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.2s1 1.2s1 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c58093900 con 0x558c54c5b400 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 1 keys = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key (42 bytes) 2019-12-07T19:34:15.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s3 1.2s3 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c58093b80 con 0x558c54c19c00 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:15.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 38) e38/38) v1 -- 0x558c57e8e000 con 0x558c53ae5400 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c5730af00 seq 1 2019-12-07T19:34:15.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.2s0_head bits 4 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.2s0_head bits 4 = 0 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- SNA_1_000000000000000D_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:40000000::::head# key 0x8080000000000000014000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000038.00000000000000000025' <- 0000000038.00000000000000000025 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000038.00000000000000000026' <- 0000000038.00000000000000000026 2019-12-07T19:34:15.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1171 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:15.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 8373 (1 ios * 4000 + 4373 bytes) 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._infover' <- _infover 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes 0x558c57f35b80,0x558c58092000 shared_blobs 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) update 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:15.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:e# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58092108) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:15.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 859 (549 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730af00 cost 5923 (1 ios * 4000 + 1923 bytes) 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730af00 onodes 0x558c57e8e280 shared_blobs 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore.extentmap(0x558c57e8e388) update 0#1:40000000::::head# 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore.extentmap(0x558c57e8e388) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 lua=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] op_applied version 38'26 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:40000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730af00 allocated 0x[] released 0x[] 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 prepare 2019-12-07T19:34:15.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730af00 2019-12-07T19:34:15.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 io_done 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] eval_repop repgather(0x558c55159440 38'26 rep_tid=22 committed?=0 r=0) (no op) 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 38 handle_pg_create_info new pg pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] calc_trim_to_aggressive limit = 37'24 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7:2._attach_pg 1.2s0 0x558c57e88000 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 38 identify_splits_and_merges 1.2s0 e38 to e38 pg_nums {18=16} 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7:2._prime_splits 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] enter Trimming/WaitRepops 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] snap_trimmer complete 2019-12-07T19:34:15.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.7(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 2019-12-07T19:34:15.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:21.970064+0000, must = 0 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] osd.7(4) has stray content: 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 7 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.616+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.616+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.616+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 0+0+0 (unknown 0 0 0) 0x558c57e8e000 con 0x558c53ae5400 2019-12-07T19:34:15.616+0000 7f8f1119b700 15 osd.7 38 enqueue_peering_evt 1.2s4 epoch_sent: 38 epoch_requested: 38 MQuery 1.2s4 from 7(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.616+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.2s4 from 7(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:15.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4135 < min_flush_size 524288 2019-12-07T19:34:15.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 179 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efac80 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35b80 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092000 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5652 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.7s4 epoch_sent: 38 epoch_requested: 38 MQuery 1.7s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.7s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5675 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5675 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xd91f1~162b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xd91f1 0xd91f1~162b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa91f1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1f1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x81c and padding block with 0x7e4 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 180 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 181 ==== MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=168, reqid=osd.1.0:168, at_version=38'17, trim_to=0'0, roll_forward_to=37'15)) v2 ==== 3426+0+0 (crc 0 0 0) 0x558c57e7a800 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xda81c 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c57f3e9a0 prio 127 cost 0 latency 0.000027 epoch 38 MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=168, reqid=osd.1.0:168, at_version=38'17, trim_to=0'0, roll_forward_to=37'15)) v2 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=168, reqid=osd.1.0:168, at_version=38'17, trim_to=0'0, roll_forward_to=37'15)) v2) prio 127 cost 0 e38) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000161492s (0.000000100s flush + 0.000161392s kv commit) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200,0x558c5730af00> 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 182 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efac80 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 0x558c57f35b80,0x558c58092000 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.es4 epoch_sent: 38 epoch_requested: 38 MQuery 1.es4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.es4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 kv_submitted 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730af00 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 finishing 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730af00 onodes 0x558c57e8e280 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730af00 done 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730af00 [] 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.2s4 from 7(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.2s4 from 7(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c5731a000 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 183 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.000154971s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MQuery 1.2s4 from 7(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY pruub 54.100152730s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.000154971s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.000154971s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:15.617+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.617+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 -- 0x558c57ed2000 con 0x558c53ae5400 2019-12-07T19:34:15.617+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 56 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c576b8700 con 0x558c54c19c00 2019-12-07T19:34:15.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:15.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5730b800 seq 26 2019-12-07T19:34:15.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:15.617+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.1(2) 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2019-12-07T19:34:15.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:15.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b800 onodes shared_blobs 2019-12-07T19:34:15.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b800 allocated 0x[] released 0x[] 2019-12-07T19:34:15.617+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:05.854653+0000, must = 0 2019-12-07T19:34:15.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 prepare 2019-12-07T19:34:15.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b800 2019-12-07T19:34:15.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 io_done 2019-12-07T19:34:15.617+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.617+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f03980700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 57 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c576b8700 con 0x558c54c19c00 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.5(3) 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:04.434820+0000, must = 0 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 58 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999496436s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c576b8700 con 0x558c54c19c00 2019-12-07T19:34:15.617+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999496436s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999496436s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 0+0+0 (unknown 0 0 0) 0x558c57ed2000 con 0x558c53ae5400 2019-12-07T19:34:15.617+0000 7f8f1119b700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xda81c~416 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xda81c 0xda81c~416 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xaa81c 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x81c 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc32 and padding block with 0x3ce 2019-12-07T19:34:15.617+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 59 ==== osd_map(38..38 src has 1..38) v4 ==== 1000+0+0 (unknown 0 0 0) 0x558c57570d80 con 0x558c54c19c00 2019-12-07T19:34:15.617+0000 7f8f1199c700 20 osd.7 38 OSD::ms_dispatch: osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.617+0000 7f8f1199c700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:15.617+0000 7f8f1199c700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:15.617+0000 7f8f1199c700 20 osd.7 38 _dispatch 0x558c57570d80 osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 184 ==== pg_query2(1.as5 1.as5 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efac80 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.as5 epoch_sent: 38 epoch_requested: 38 MQuery 1.as5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.as5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f1199c700 3 osd.7 38 handle_osd_map epochs [38,38], i have 38, src has [1,38] 2019-12-07T19:34:15.617+0000 7f8f1199c700 10 osd.7 38 no new maps here, dropping 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xdac32 2019-12-07T19:34:15.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=168, reqid=osd.1.0:168, at_version=38'17, trim_to=0'0, roll_forward_to=37'15)) v2) prio 127 cost 0 e38) queued 2019-12-07T19:34:15.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=168, reqid=osd.1.0:168, at_version=38'17, trim_to=0'0, roll_forward_to=37'15)) v2) prio 127 cost 0 e38) pg 0x558c5500d000 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 38 dequeue_op 0x558c57f3e9a0 prio 127 cost 0 latency 0.000413 MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=168, reqid=osd.1.0:168, at_version=38'17, trim_to=0'0, roll_forward_to=37'15)) v2 pg pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129632s (0.000000109s flush + 0.000129523s kv commit) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b800> 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 kv_submitted 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b800 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=168, reqid=osd.1.0:168, at_version=38'17, trim_to=0'0, roll_forward_to=37'15)) v2 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 finishing 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b800 onodes 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 done 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b800 [] 2019-12-07T19:34:15.617+0000 7f8f04982700 20 snap_mapper.update_snaps 1:0064fe1d:::smithi13140796-44:e c,e was 2019-12-07T19:34:15.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.617+0000 7f8f04982700 20 snap_mapper.get_snaps 1:0064fe1d:::smithi13140796-44:e c,d,e 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 7(4) notify: (query:38 sent:38 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 4->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f04982700 20 snap_mapper.set_snaps 1:0064fe1d:::smithi13140796-44:e c,e 2019-12-07T19:34:15.617+0000 7f8f04982700 20 snap_mapper.set_snaps set OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.7(4) info 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38), identical to ours 2019-12-07T19:34:15.617+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.617+0000 7f8f04982700 20 snap_mapper.update_snaps rm SNA_1_000000000000000D_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.as5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.as5 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 38 handle_pg_query_nopg 1.as5 2019-12-07T19:34:15.617+0000 7f8f07988700 10 osd.7 38 pg 1.as5 dne 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 37'15 (0'0,37'15] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] append_log log((0'0,37'15], crt=37'15) [38'16 (37'11) modify 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:15.615867+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,38'17 (37'15) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:15.615867+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:15.617+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:38 sent:38 1.as5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 38) v7 -- 0x558c54d381e0 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'16 (0'0,38'16] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'15 crt=37'15 lcod 37'15 active mbc={} ps=[a~1]] add_log_entry 38'16 (37'11) modify 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:15.615867+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'15 crt=37'15 lcod 37'15 active mbc={} ps=[a~1]] add_log_entry 38'17 (37'15) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:15.615867+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'15 crt=37'15 lcod 37'15 active mbc={} ps=[a~1]] append_log approx pg log length = 17 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'15 crt=37'15 lcod 37'15 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:15.617+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:15.617+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 38'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:15.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.es4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.es4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.617+0000 7f8f0197c700 10 osd.7 38 handle_pg_query_nopg 1.es4 2019-12-07T19:34:15.617+0000 7f8f0197c700 10 osd.7 38 pg 1.es4 dne 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:15.617+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:38 sent:38 1.es4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 38) v7 -- 0x558c57f114a0 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c577fcc00 seq 31 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# 1 keys 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# 1 keys = 0 2019-12-07T19:34:15.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# hinfo_key (42 bytes) 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- SNA_1_000000000000000D_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000038.00000000000000000016' <- 0000000038.00000000000000000016 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000038.00000000000000000017' <- 0000000038.00000000000000000017 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcc00 cost 8405 (1 ios * 4000 + 4405 bytes) 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcc00 onodes 0x558c575bb900,0x558c57f34000 shared_blobs 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 580 (578 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore.extentmap(0x558c57f34108) update 3#1:0064fe1d:::smithi13140796-44:e# 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:e# is 1039 (412 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcc00 allocated 0x[] released 0x[] 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 121 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c576b8700 con 0x558c54c5b400 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 prepare 2019-12-07T19:34:15.617+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcc00 2019-12-07T19:34:15.617+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 io_done 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=37'15 crt=37'15 lcod 37'15 active mbc={} ps=[a~1]] op_applied version 38'17 2019-12-07T19:34:15.617+0000 7f8f04982700 10 osd.7 38 dequeue_op 0x558c57f3e9a0 finish 2019-12-07T19:34:15.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.7s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.7s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c54d5f000 2019-12-07T19:34:15.617+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.043253254s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MQuery 1.7s4 from 1(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.617+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:15.617+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:15.617+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:38 sent:38 1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) 4->0 ([28,37] all_participants=0(3),1(0),2(5),3(2),4(1),7(4) intervals=([28,37] acting 0(3),1(0),2(5),3(2),4(1),7(4)))) e38/38) v1 -- 0x558c576ba300 con 0x558c54c64000 2019-12-07T19:34:15.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 122 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998893807s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57452a00 con 0x558c54c5b400 2019-12-07T19:34:15.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:15.617+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998893807s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998893807s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4484 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34000 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4507 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4507 < min_flush_size 524288 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xdac32~119b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xdac32 0xdac32~119b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xaac32 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc32 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdcd and padding block with 0x233 2019-12-07T19:34:15.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 123 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57452a00 con 0x558c54c5b400 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 2019-12-07T19:34:15.618+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:34:56.688549+0000, must = 0 2019-12-07T19:34:15.618+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 4(1) notify: (query:38 sent:38 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 1->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.4(1) info 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38), identical to ours 2019-12-07T19:34:15.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 66 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c57393100 con 0x558c54c19800 2019-12-07T19:34:15.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xdbdcd 2019-12-07T19:34:15.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000209944s (0.000000081s flush + 0.000209863s kv commit) 2019-12-07T19:34:15.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.618+0000 7f8f24c58700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcc00> 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 kv_submitted 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcc00 2019-12-07T19:34:15.618+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 finishing 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcc00 onodes 0x558c575bb900,0x558c57f34000 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcc00 done 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:15.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcc00 [] 2019-12-07T19:34:15.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.000154971s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.000154971s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 1(2) notify: (query:38 sent:38 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.000154971s) 2->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.1(2) info 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.000154971s), identical to ours 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 68 ==== pg_notify2(1.4s0 1.4s0 (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998433433s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e38/38) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c548a3100 con 0x558c54c19800 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.618+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 67 ==== osd_map(38..38 src has 1..38) v4 ==== 1000+0+0 (crc 0 0 0) 0x558c575d4b40 con 0x558c54c19800 2019-12-07T19:34:15.618+0000 7f8f24c58700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998433433s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f1199c700 20 osd.7 38 OSD::ms_dispatch: osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.618+0000 7f8f1199c700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:15.618+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998433433s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.618+0000 7f8f1199c700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:15.618+0000 7f8f1199c700 20 osd.7 38 _dispatch 0x558c575d4b40 osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.618+0000 7f8f1199c700 3 osd.7 38 handle_osd_map epochs [38,38], i have 38, src has [1,38] 2019-12-07T19:34:15.618+0000 7f8f1199c700 10 osd.7 38 no new maps here, dropping 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:16.196240+0000, must = 0 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999496436s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999496436s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 5(3) notify: (query:38 sent:38 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999496436s) 3->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.5(3) info 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999496436s), identical to ours 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998893807s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998893807s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 4(5) notify: (query:38 sent:38 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998893807s) 5->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.4(5) info 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998893807s), identical to ours 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:34:59.969215+0000, must = 0 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998433433s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998433433s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 2(4) notify: (query:38 sent:38 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998433433s) 4->0 ([18,37] all_participants=0(1),1(2),2(4),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.2(4) info 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.998433433s), identical to ours 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f0297e700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.618+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 38/18 ECSubWriteReply(tid=168, last_complete=38'17, committed=1, applied=1)) v2 -- 0x558c54dfae00 con 0x558c54c64000 2019-12-07T19:34:15.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 60 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57452a00 con 0x558c54c19c00 2019-12-07T19:34:15.618+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 5(3) notify: (query:38 sent:38 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 3->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.5(3) info 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38), identical to ours 2019-12-07T19:34:15.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.618+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 61 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfae00 con 0x558c54c19c00 2019-12-07T19:34:15.619+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c57f3eb00 prio 196 cost 0 latency 0.000013 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.619+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c57f3eb00 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:15.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.619+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c57f3eb00 finish 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 124 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c548cc600 con 0x558c54c5b400 2019-12-07T19:34:15.619+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c57f3ec60 prio 196 cost 0 latency 0.000012 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.619+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c57f3ec60 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:15.619+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.619+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c57f3ec60 finish 2019-12-07T19:34:15.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 185 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c576ba300 con 0x558c54c64000 2019-12-07T19:34:15.619+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.619+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.619+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 1(2) notify: (query:38 sent:38 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 2->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.619+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got dup osd.1(2) info 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38), identical to ours 2019-12-07T19:34:15.619+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:15.619+0000 7f8f03980700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 186 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c551aec00 con 0x558c54c64000 2019-12-07T19:34:15.619+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c57f3edc0 prio 196 cost 0 latency 0.000010 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.619+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c57f3edc0 prio 196 cost 0 latency 0.000046 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:15.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.619+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c57f3edc0 finish 2019-12-07T19:34:15.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.619+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 69 ==== pg_query2(1.5s4 1.5s4 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54d97b80 con 0x558c54c19800 2019-12-07T19:34:15.619+0000 7f8f24c58700 15 osd.7 38 enqueue_peering_evt 1.5s4 epoch_sent: 38 epoch_requested: 38 MQuery 1.5s4 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.619+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.5s4 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.5s4 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.5s4 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.619+0000 7f8f0217d700 10 osd.7 38 handle_pg_query_nopg 1.5s4 2019-12-07T19:34:15.619+0000 7f8f0217d700 10 osd.7 38 pg 1.5s4 dne 2019-12-07T19:34:15.619+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify((query:38 sent:38 1.5s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 38) v7 -- 0x558c57f11a40 con 0x558c54c19800 2019-12-07T19:34:15.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:15.620+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 70 ==== pg_notify((query:38 sent:38 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 38) v7 ==== 985+0+0 (crc 0 0 0) 0x558c57f10780 con 0x558c54c19800 2019-12-07T19:34:15.620+0000 7f8f24c58700 7 osd.7 38 handle_fast_pg_notify pg_notify((query:38 sent:38 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 38) v7 from osd.2 2019-12-07T19:34:15.620+0000 7f8f24c58700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 2(4) notify: (query:38 sent:38 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.620+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 2(4) notify: (query:38 sent:38 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 2(4) notify: (query:38 sent:38 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 2(4) notify: (query:38 sent:38 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.620+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 2(4) notify: (query:38 sent:38 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.620+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.2(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:15.620+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:09.816353+0000, must = 0 2019-12-07T19:34:15.620+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.620+0000 7f8f07988700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.620+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 71 ==== pg_query2(1.ds2 1.ds2 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (crc 0 0 0) 0x558c54d97b80 con 0x558c54c19800 2019-12-07T19:34:15.621+0000 7f8f24c58700 15 osd.7 38 enqueue_peering_evt 1.ds2 epoch_sent: 38 epoch_requested: 38 MQuery 1.ds2 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.ds2 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.ds2 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.ds2 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0x558c55141000 2019-12-07T19:34:15.621+0000 7f8f06185700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003168428s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MQuery 1.ds2 from 2(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.621+0000 7f8f06185700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:15.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:15.621+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:38 sent:38 1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) 2->0 ([18,37] all_participants=0(5),1(1),2(0),3(3),4(4),7(2) intervals=([18,37] acting 0(5),1(1),2(0),3(3),4(4),7(2)))) e38/38) v1 -- 0x558c574e5500 con 0x558c54c19800 2019-12-07T19:34:15.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:15.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 105 ==== pg_query2(1.fs5 1.fs5 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c5793bb80 con 0x558c54c5bc00 2019-12-07T19:34:15.621+0000 7f8f23c56700 15 osd.7 38 enqueue_peering_evt 1.fs5 epoch_sent: 38 epoch_requested: 38 MQuery 1.fs5 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.621+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.fs5 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 106 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 38) e38/38) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c58093680 con 0x558c54c5bc00 2019-12-07T19:34:15.621+0000 7f8f23c56700 15 osd.7 38 enqueue_peering_evt 1.1s1 epoch_sent: 38 epoch_requested: 38 MQuery 1.1s1 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38) 2019-12-07T19:34:15.621+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.1s1 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) 2019-12-07T19:34:15.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 107 ==== pg_notify((query:38 sent:38 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 38) v7 ==== 985+0+0 (unknown 0 0 0) 0x558c54d0e5a0 con 0x558c54c5bc00 2019-12-07T19:34:15.621+0000 7f8f23c56700 7 osd.7 38 handle_fast_pg_notify pg_notify((query:38 sent:38 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 38) v7 from osd.3 2019-12-07T19:34:15.621+0000 7f8f23c56700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 3(1) notify: (query:38 sent:38 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.621+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 3(1) notify: (query:38 sent:38 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 3(1) notify: (query:38 sent:38 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 3(1) notify: (query:38 sent:38 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c548ab000 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.4s0 from 3(1) notify: (query:38 sent:38 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 109 ==== MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=97, reqid=osd.3.0:97, at_version=38'30, trim_to=0'0, roll_forward_to=37'28)) v2 ==== 3426+0+0 (unknown 0 0 0) 0x558c57e7c000 con 0x558c54c5bc00 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.3(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:15.621+0000 7f8f23c56700 15 osd.7 38 enqueue_op 0x558c578d2f20 prio 127 cost 0 latency 0.000026 epoch 38 MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=97, reqid=osd.3.0:97, at_version=38'30, trim_to=0'0, roll_forward_to=37'28)) v2 2019-12-07T19:34:15.621+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=97, reqid=osd.3.0:97, at_version=38'30, trim_to=0'0, roll_forward_to=37'28)) v2) prio 127 cost 0 e38) 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:09.696176+0000, must = 0 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.fs5 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.fs5 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.621+0000 7f8f0117b700 10 osd.7 38 handle_pg_query_nopg 1.fs5 2019-12-07T19:34:15.621+0000 7f8f0117b700 10 osd.7 38 pg 1.fs5 dne 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:15.621+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:15.621+0000 7f8f06986700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:15.623100+0000 10 0.000376 2019-12-07T19:34:15.621+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:38 sent:38 1.fs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 38) v7 -- 0x558c57ede000 con 0x558c54c5bc00 2019-12-07T19:34:15.621+0000 7f8f06986700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:15.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:15.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.002957799s) 2019-12-07T19:34:15.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=97, reqid=osd.3.0:97, at_version=38'30, trim_to=0'0, roll_forward_to=37'28)) v2) prio 127 cost 0 e38) queued 2019-12-07T19:34:15.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999497053s) 2019-12-07T19:34:15.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=97, reqid=osd.3.0:97, at_version=38'30, trim_to=0'0, roll_forward_to=37'28)) v2) prio 127 cost 0 e38) pg 0x558c54d5c000 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.3(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:15.621+0000 7f8f05984700 10 osd.7 38 dequeue_op 0x558c578d2f20 prio 127 cost 0 latency 0.000122 MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=97, reqid=osd.3.0:97, at_version=38'30, trim_to=0'0, roll_forward_to=37'28)) v2 pg pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=13.003469286s) 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.999677072s) 2019-12-07T19:34:15.621+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=97, reqid=osd.3.0:97, at_version=38'30, trim_to=0'0, roll_forward_to=37'28)) v2 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.7(0) 1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) 2019-12-07T19:34:15.621+0000 7f8f05984700 20 snap_mapper.update_snaps 1:6ee4dfe7:::smithi13140796-47:e c,e was 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] calc_acting prefer osd.7(0) because it is current primary 2019-12-07T19:34:15.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:15.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.1s1 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.621+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 3(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 4(5) 2019-12-07T19:34:15.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MQuery 1.1s1 from 3(0) query_epoch 38 query: query(info 0'0 epoch_sent 38)) prio 255 cost 10 e38) pg 0 2019-12-07T19:34:15.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.622+0000 7f8f04181700 10 osd.7 38 handle_pg_query_nopg 1.1s1 2019-12-07T19:34:15.622+0000 7f8f04181700 10 osd.7 38 pg 1.1s1 dne 2019-12-07T19:34:15.622+0000 7f8f05984700 20 snap_mapper.get_snaps 1:6ee4dfe7:::smithi13140796-47:e c,d,e 2019-12-07T19:34:15.622+0000 7f8f05984700 20 snap_mapper.set_snaps 1:6ee4dfe7:::smithi13140796-47:e c,e 2019-12-07T19:34:15.622+0000 7f8f05984700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:15.622+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:38 sent:38 1.1s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 38) v7 -- 0x558c57f112c0 con 0x558c54c5bc00 2019-12-07T19:34:15.622+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_async_recovery_ec candidates by cost are: 4,3(1) 2019-12-07T19:34:15.622+0000 7f8f05984700 20 snap_mapper.update_snaps rm SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:15.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:15.622+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_async_recovery_ec result want=[7,2147483647,1,5,2,4] async_recovery=3(1) 2019-12-07T19:34:15.622+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting want [7,2147483647,1,5,2,4] != acting [7,3,1,5,2,4], requesting pg_temp change 2019-12-07T19:34:15.622+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 108 ==== osd_map(38..38 src has 1..38) v4 ==== 1000+0+0 (unknown 0 0 0) 0x558c54d2efc0 con 0x558c54c5bc00 2019-12-07T19:34:15.622+0000 7f8f06986700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:15.623228+0000 0 0.000000 2019-12-07T19:34:15.622+0000 7f8f1199c700 20 osd.7 38 OSD::ms_dispatch: osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.622+0000 7f8f1199c700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:15.622+0000 7f8f1199c700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:15.622+0000 7f8f1199c700 20 osd.7 38 _dispatch 0x558c54d2efc0 osd_map(38..38 src has 1..38) v4 2019-12-07T19:34:15.622+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Leaving Peering 2019-12-07T19:34:15.622+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 37'28 (0'0,37'28] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] append_log log((0'0,37'28], crt=37'28) [38'29 (37'25) modify 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:15.617999+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,38'30 (37'28) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:15.617999+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:15.622+0000 7f8f1199c700 3 osd.7 38 handle_osd_map epochs [38,38], i have 38, src has [1,38] 2019-12-07T19:34:15.622+0000 7f8f06986700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering 2019-12-07T19:34:15.623237+0000 0 0.000000 2019-12-07T19:34:15.622+0000 7f8f1199c700 10 osd.7 38 no new maps here, dropping 2019-12-07T19:34:15.622+0000 7f8f06986700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] exit Started/Primary 2019-12-07T19:34:15.623243+0000 0 0.000000 2019-12-07T19:34:15.622+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'29 (0'0,38'29] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'28 crt=37'28 lcod 37'28 active mbc={} ps=[a~1]] add_log_entry 38'29 (37'25) modify 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:15.617999+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.622+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:15.622+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:15.622+0000 7f8f06986700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Started/Primary 2019-12-07T19:34:15.622+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'28 crt=37'28 lcod 37'28 active mbc={} ps=[a~1]] add_log_entry 38'30 (37'28) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:15.617999+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.622+0000 7f8f06986700 5 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Started/Primary/WaitActingChange 2019-12-07T19:34:15.622+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:15.622+0000 7f8f06986700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.622+0000 7f8f06986700 10 osd.7 38 send_pg_temp {1.4=[7,2147483647,1,5,2,4]} 2019-12-07T19:34:15.622+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'28 crt=37'28 lcod 37'28 active mbc={} ps=[a~1]] append_log approx pg log length = 30 2019-12-07T19:34:15.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_pgtemp(e38 {1.4=[7,2147483647,1,5,2,4]} v38) v2 -- 0x558c579e7b00 con 0x558c5476d000 2019-12-07T19:34:15.622+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'28 crt=37'28 lcod 37'28 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:15.622+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:15.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.622+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 38'29, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730bb00 seq 58 2019-12-07T19:34:15.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# 1 keys 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# 1 keys = 0 2019-12-07T19:34:15.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# hinfo_key (42 bytes) 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:15.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:15.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:15.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- SNA_1_000000000000000D_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:15.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000038.00000000000000000029' <- 0000000038.00000000000000000029 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000038.00000000000000000030' <- 0000000038.00000000000000000030 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730bb00 cost 8405 (1 ios * 4000 + 4405 bytes) 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730bb00 onodes 0x558c54e12280,0x558c54e12780 shared_blobs 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore.extentmap(0x558c54e12388) update 2#1:6ee4dfe7:::smithi13140796-47:e# 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:e# is 1069 (412 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore.extentmap(0x558c54e12888) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1222 (565 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 prepare 2019-12-07T19:34:15.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730bb00 2019-12-07T19:34:15.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 io_done 2019-12-07T19:34:15.622+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=37'28 crt=37'28 lcod 37'28 active mbc={} ps=[a~1]] op_applied version 38'30 2019-12-07T19:34:15.622+0000 7f8f05984700 10 osd.7 38 dequeue_op 0x558c578d2f20 finish 2019-12-07T19:34:15.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5156 < min_flush_size 524288 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12280 had 1 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12780 had 1 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5179 < min_flush_size 524288 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5179 < min_flush_size 524288 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xdbdcd~143b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xdbdcd 0xdbdcd~143b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xabdcd 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdcd 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x208 and padding block with 0xdf8 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xdd208 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000111718s (0.000000053s flush + 0.000111665s kv commit) 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730bb00> 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 kv_submitted 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730bb00 2019-12-07T19:34:15.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 finishing 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730bb00 onodes 0x558c54e12280,0x558c54e12780 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730bb00 done 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:15.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730bb00 [] 2019-12-07T19:34:15.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.622+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 38/28 ECSubWriteReply(tid=97, last_complete=38'30, committed=1, applied=1)) v2 -- 0x558c575b8200 con 0x558c54c5bc00 2019-12-07T19:34:15.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.622+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 72 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59400 con 0x558c54c19800 2019-12-07T19:34:15.623+0000 7f8f24c58700 15 osd.7 38 enqueue_op 0x558c578d3ce0 prio 196 cost 0 latency 0.000019 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.623+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.623+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c578d3ce0 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:15.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.623+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c578d3ce0 finish 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 110 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57392300 con 0x558c54c5bc00 2019-12-07T19:34:15.623+0000 7f8f23c56700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.623+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 111 ==== pg_notify2(1.2s0 1.2s0 (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e38/38) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57392300 con 0x558c54c5bc00 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] got osd.3(5) 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 2019-12-07T19:34:15.623+0000 7f8f23c56700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.623+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:14.363298+0000, must = 0 2019-12-07T19:34:15.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 112 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c575b8200 con 0x558c54c5bc00 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:15.623+0000 7f8f23c56700 15 osd.7 38 enqueue_op 0x558c578d38c0 prio 196 cost 0 latency 0.000010 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.623+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c578d38c0 prio 196 cost 0 latency 0.000030 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:15.623+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:15.624677+0000 10 0.000389 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=22, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.623+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=38'26 tt=0'0 tid=22 reqid=osd.7.0:22 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.1(2) 1.2s2( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047952826s) 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=37'24 crt=37'24 lcod 35'22 mlcod 35'22 active+clean+snaptrim trimq=[d~1] ps=[a~1]] repop_all_committed: repop tid 22 all committed 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.2(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.3(5) 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041390057s) 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] eval_repop repgather(0x558c55159440 38'26 rep_tid=22 committed?=1 r=0) (no op) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.049077703s) 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] commit: repgather(0x558c55159440 38'26 rep_tid=22 committed?=1 r=0) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.5(3) 1.2s3( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.047145398s) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.7(0) 1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.623+0000 7f8f06986700 15 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] publish_stats_to_osd 38:117 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting all_info osd.7(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.048594663s) 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] removing repgather(0x558c55159440 38'26 rep_tid=22 committed?=1 r=0) 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] q front is repgather(0x558c55159440 38'26 rep_tid=22 committed?=1 r=0) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] exit Trimming/WaitRepops 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_async_recovery_ec candidates by cost are: 15,2(4),15,7(0) 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 38 queueing pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] for snaptrim 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=38) prio 5 cost 1048576 e38) 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_async_recovery_ec result want=[2147483647,4,1,5,2147483647,3] async_recovery=2(4),7(0) 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] choose_acting want [2147483647,4,1,5,2147483647,3] != acting [7,4,1,5,2,3], requesting pg_temp change 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:15.624775+0000 0 0.000000 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=38) prio 5 cost 1048576 e38) queued 2019-12-07T19:34:15.623+0000 7f8f06986700 15 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] publish_stats_to_osd 38:118 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] state: Leaving Peering 2019-12-07T19:34:15.623+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 peering mbc={}] exit Started/Primary/Peering 2019-12-07T19:34:15.624783+0000 0 0.000000 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] remove_repop repgather(0x558c55159440 38'26 rep_tid=22 committed?=1 r=0) 2019-12-07T19:34:15.623+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started/Primary 2019-12-07T19:34:15.624788+0000 0 0.000000 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=38'26 tt=0'0 tid=22 reqid=osd.7.0:22 roll_forward_to=37'24 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:15.623+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary 2019-12-07T19:34:15.623+0000 7f8f03980700 5 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary/WaitActingChange 2019-12-07T19:34:15.623+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.623+0000 7f8f03980700 10 osd.7 38 send_pg_temp {1.2=[2147483647,4,1,5,2147483647,3]} 2019-12-07T19:34:15.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38)> waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e38) pg 0x558c57e88000 2019-12-07T19:34:15.623+0000 7f8f03980700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_pgtemp(e38 {1.2=[2147483647,4,1,5,2147483647,3]} v38) v2 -- 0x558c54dd1440 con 0x558c5476d000 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=0'0 tt=0'0 tid=23 reqid=osd.7.0:22 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:15.623+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] do_peering_event: epoch_sent: 38 epoch_requested: 38 MNotifyRec 1.2s0 from 3(5) notify: (query:38 sent:38 1.2s5( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) 5->0 ([28,37] all_participants=0(0),1(2),3(5),4(1),5(3),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:15.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.623+0000 7f8f07988700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: In WaitActingChange, ignoring MNotifyRec 2019-12-07T19:34:15.623+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:15.623+0000 7f8f07988700 10 osd.7 38 queue_want_up_thru want 38 <= queued 38, currently 28 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:15.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:e v=0'0 tt=0'0 tid=23 reqid=osd.7.0:22 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:15.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 38/28 ECSubWrite(tid=23, reqid=osd.7.0:22, at_version=0'0, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c54d86000 con 0x558c54c64000 2019-12-07T19:34:15.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 38/28 ECSubWrite(tid=23, reqid=osd.7.0:22, at_version=0'0, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c54d87000 con 0x558c54c19800 2019-12-07T19:34:15.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 38/28 ECSubWrite(tid=23, reqid=osd.7.0:22, at_version=0'0, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c54d87800 con 0x558c54c5bc00 2019-12-07T19:34:15.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 38/28 ECSubWrite(tid=23, reqid=osd.7.0:22, at_version=0'0, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c5748e800 con 0x558c54c5b400 2019-12-07T19:34:15.623+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 38/28 ECSubWrite(tid=23, reqid=osd.7.0:22, at_version=0'0, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c57a45000 con 0x558c54c19c00 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=37'24 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] append_log log((0'0,38'26], crt=37'24) [] 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] rollforward: entry=38'25 (37'23) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] rollforward: entry=38'26 (37'24) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:15.617287+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] append_log approx pg log length = 26 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:15.623+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:15.623+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:15.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:15.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea0c00 seq 33 2019-12-07T19:34:15.623+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:15.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:15.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:15.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:15.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:15.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:15.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes shared_blobs 2019-12-07T19:34:15.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:15.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:15.623+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:15.623+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.623+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c578d38c0 finish 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=38) prio 5 cost 1048576 e38) pg 0x558c57676000 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:15.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:15.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:15.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] snap_trimmer posting 2019-12-07T19:34:15.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xdd208~471 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xdd208 0xdd208~471 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xad208 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x208 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x679 and padding block with 0x987 2019-12-07T19:34:15.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap d 2019-12-07T19:34:15.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:15.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c577f6240) upper_bound after SNA_1_000000000000000D_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.C' 2019-12-07T19:34:15.623+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c577f6240) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000D_.0_0000000000000001.C) returns 0 SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[d~1] ps=[a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1]] SnapTrimmer state: adding snap d to purged_snaps 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1]] SnapTrimmer state: purged_snaps now [a~1,d~1], snap_trimq now [] 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea1500 seq 34 2019-12-07T19:34:15.624+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1]] share_pg_info 2019-12-07T19:34:15.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 187 ==== MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=169, reqid=osd.1.0:168, at_version=0'0, trim_to=0'0, roll_forward_to=38'17)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748e800 con 0x558c54c64000 2019-12-07T19:34:15.624+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000019 epoch 38 MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=169, reqid=osd.1.0:168, at_version=0'0, trim_to=0'0, roll_forward_to=38'17)) v2 2019-12-07T19:34:15.624+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=169, reqid=osd.1.0:168, at_version=0'0, trim_to=0'0, roll_forward_to=38'17)) v2) prio 127 cost 0 e38) 2019-12-07T19:34:15.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e38/38) v1 -- 0x558c576e0a00 con 0x558c54c64000 2019-12-07T19:34:15.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e38/38) v1 -- 0x558c576dee00 con 0x558c54c19800 2019-12-07T19:34:15.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e38/38) v1 -- 0x558c54e98e00 con 0x558c54c5bc00 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xdd679 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000257055s (0.000000085s flush + 0.000256970s kv commit) 2019-12-07T19:34:15.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e38/38) v1 -- 0x558c57390000 con 0x558c54c5b400 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e38/38) v1 -- 0x558c574e4700 con 0x558c54c19c00 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xdd679~46d to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xdd679 0xdd679~46d to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xad679 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x679 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xae6 and padding block with 0x51a 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=169, reqid=osd.1.0:168, at_version=0'0, trim_to=0'0, roll_forward_to=38'17)) v2) prio 127 cost 0 e38) queued 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1]] exit Trimming 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=169, reqid=osd.1.0:168, at_version=0'0, trim_to=0'0, roll_forward_to=38'17)) v2) prio 127 cost 0 e38) pg 0x558c5500d000 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:15.624+0000 7f8f0297e700 15 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] publish_stats_to_osd 38:119 2019-12-07T19:34:15.624+0000 7f8f04982700 10 osd.7 38 dequeue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000125 MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=169, reqid=osd.1.0:168, at_version=0'0, trim_to=0'0, roll_forward_to=38'17)) v2 pg pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] enter NotTrimming 2019-12-07T19:34:15.624+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.0s3 38/18 ECSubWrite(tid=169, reqid=osd.1.0:168, at_version=0'0, trim_to=0'0, roll_forward_to=38'17)) v2 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:15.624+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=37'15 active mbc={} ps=[a~1]] append_log log((0'0,38'17], crt=37'15) [] 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] snap_trimmer complete 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1]] rollforward: entry=38'16 (37'11) modify 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:15.615867+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1]] rollforward: entry=38'17 (37'15) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:15.615867+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.624+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1]] append_log approx pg log length = 17 2019-12-07T19:34:15.624+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:15.624+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:15.624+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:15.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:15.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c574fe900 seq 32 2019-12-07T19:34:15.624+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:15.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:15.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:15.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:15.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:15.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:15.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe900 onodes shared_blobs 2019-12-07T19:34:15.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe900 allocated 0x[] released 0x[] 2019-12-07T19:34:15.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 prepare 2019-12-07T19:34:15.624+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe900 2019-12-07T19:34:15.624+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 io_done 2019-12-07T19:34:15.624+0000 7f8f04982700 10 osd.7 38 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 113 ==== MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=98, reqid=osd.3.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=38'30)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54d87800 con 0x558c54c5bc00 2019-12-07T19:34:15.624+0000 7f8f23c56700 15 osd.7 38 enqueue_op 0x558c57f3e420 prio 127 cost 0 latency 0.000026 epoch 38 MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=98, reqid=osd.3.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=38'30)) v2 2019-12-07T19:34:15.624+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=98, reqid=osd.3.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e38) 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xddae6 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=98, reqid=osd.3.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e38) queued 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000315978s (0.000000092s flush + 0.000315886s kv commit) 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=98, reqid=osd.3.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e38) pg 0x558c54d5c000 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 osd.7 38 dequeue_op 0x558c57f3e420 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=98, reqid=osd.3.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=38'30)) v2 pg pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 38/28 ECSubWrite(tid=98, reqid=osd.3.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=38'30)) v2 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xddae6~471 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 188 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e38/38) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c576ba300 con 0x558c54c64000 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=37'28 active mbc={} ps=[a~1]] append_log log((0'0,38'30], crt=37'28) [] 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xddae6 0xddae6~471 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xadae6 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xae6 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf57 and padding block with 0xa9 2019-12-07T19:34:15.624+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.0s3 epoch_sent: 38 epoch_requested: 38 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s) 2019-12-07T19:34:15.624+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e38) 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1]] rollforward: entry=38'29 (37'25) modify 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:15.617999+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1]] rollforward: entry=38'30 (37'28) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:15.617999+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1]] append_log approx pg log length = 30 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e38) pg 0x558c5500d000 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s) 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730ac00 seq 59 2019-12-07T19:34:15.624+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c57ef8900 seq 33 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730ac00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730ac00 onodes shared_blobs 2019-12-07T19:34:15.624+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 prepare 2019-12-07T19:34:15.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 62 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfa400 con 0x558c54c19c00 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730ac00 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 io_done 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8900 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8900 onodes shared_blobs 2019-12-07T19:34:15.624+0000 7f8f0197c700 10 osd.7 38 dequeue_op 0x558c57f3e420 finish 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8900 allocated 0x[] released 0x[] 2019-12-07T19:34:15.624+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c57f3f080 prio 196 cost 0 latency 0.000016 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 prepare 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8900 2019-12-07T19:34:15.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 io_done 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 73 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c551afc00 con 0x558c54c19800 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xddf57 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.624+0000 7f8f24c58700 15 osd.7 38 enqueue_op 0x558c552b7b80 prio 196 cost 0 latency 0.000021 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.624+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000176248s (0.000000078s flush + 0.000176170s kv commit) 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe900> 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 kv_submitted 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe900 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 finishing 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2224 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2247 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2247 < min_flush_size 524288 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c57f3f080 prio 196 cost 0 latency 0.000078 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xddf57~8c7 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xddf57 0xddf57~8c7 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c57f3f080 finish 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xadf57 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf57 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x81e and padding block with 0x7e2 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c552b7b80 prio 196 cost 0 latency 0.000083 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.624+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c552b7b80 finish 2019-12-07T19:34:15.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe900 onodes 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe900 done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8900 kv_submitted 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe900 [] 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.624+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 38/18 ECSubWriteReply(tid=169, last_complete=38'17, committed=1, applied=1)) v2 -- 0x558c54dfac00 con 0x558c54c64000 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xde81e 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000222402s (0.000000066s flush + 0.000222336s kv commit) 2019-12-07T19:34:15.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730ac00,0x558c57ef8900> 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 kv_submitted 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730ac00 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 finishing 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730ac00 onodes 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730ac00 done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730ac00 [] 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 kv_submitted 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8900 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 finishing 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8900 onodes 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8900 done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8900 [] 2019-12-07T19:34:15.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 114 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e38/38) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c57392300 con 0x558c54c5bc00 2019-12-07T19:34:15.624+0000 7f8f23c56700 15 osd.7 38 enqueue_peering_evt 1.6s2 epoch_sent: 38 epoch_requested: 38 MInfoRec from 3(0) info: 1.6s0( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:15.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.624+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MInfoRec from 3(0) info: 1.6s0( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e38) 2019-12-07T19:34:15.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:15.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 115 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c59e00 con 0x558c54c5bc00 2019-12-07T19:34:15.625+0000 7f8f23c56700 15 osd.7 38 enqueue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000012 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.625+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 189 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55271c00 con 0x558c54c64000 2019-12-07T19:34:15.625+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c57f3f1e0 prio 196 cost 0 latency 0.000012 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.625+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.625+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000179 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] 2019-12-07T19:34:15.625+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.625+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.625+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c57f3f1e0 prio 196 cost 0 latency 0.000129 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] 2019-12-07T19:34:15.625+0000 7f8f06986700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.625+0000 7f8f06986700 10 osd.7 38 dequeue_op 0x558c57f3f1e0 finish 2019-12-07T19:34:15.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MInfoRec from 3(0) info: 1.6s0( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e38) queued 2019-12-07T19:34:15.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 38 epoch_requested: 38 MInfoRec from 3(0) info: 1.6s0( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e38) pg 0x558c54d5c000 2019-12-07T19:34:15.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.625+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 38 epoch_requested: 38 MInfoRec from 3(0) info: 1.6s0( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:15.625+0000 7f8f05984700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:15.625+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:15.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:15.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730a900 seq 60 2019-12-07T19:34:15.625+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:15.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:15.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:15.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:15.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a900 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:15.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a900 onodes shared_blobs 2019-12-07T19:34:15.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a900 allocated 0x[] released 0x[] 2019-12-07T19:34:15.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 prepare 2019-12-07T19:34:15.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a900 2019-12-07T19:34:15.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 io_done 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:15.625+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 38/28 ECSubWriteReply(tid=98, last_complete=38'30, committed=1, applied=1)) v2 -- 0x558c54c58800 con 0x558c54c5bc00 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:15.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xde81e~46d to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xde81e 0xde81e~46d to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xae81e 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x81e 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc8b and padding block with 0x375 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xdec8b 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:15.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000087092s (0.000000124s flush + 0.000086968s kv commit) 2019-12-07T19:34:15.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:15.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 125 ==== MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfa200 con 0x558c54c5b400 2019-12-07T19:34:15.625+0000 7f8f24457700 15 osd.7 38 enqueue_op 0x558c54e79e40 prio 196 cost 0 latency 0.000015 epoch 38 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.625+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) 2019-12-07T19:34:15.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) queued 2019-12-07T19:34:15.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:15.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2) prio 196 cost 0 e38) pg 0x558c57676000 2019-12-07T19:34:15.625+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c54e79e40 prio 196 cost 0 latency 0.000044 MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 pg pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] 2019-12-07T19:34:15.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 38/28 ECSubWriteReply(tid=23, last_complete=38'26, committed=1, applied=1)) v2 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a900> 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:15.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 kv_submitted 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a900 2019-12-07T19:34:15.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=0'0 tt=0'0 tid=23 reqid=osd.7.0:22 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:15.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 finishing 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a900 onodes 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a900 done 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:15.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a900 [] 2019-12-07T19:34:15.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:15.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:15.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:15.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:15.625+0000 7f8f0297e700 10 osd.7 38 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:34:15.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:15.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:15.649+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:15.649+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:15.649+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 27 0x2000~1000 2019-12-07T19:34:15.649+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc cbf4ed54 2019-12-07T19:34:15.649+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 27 0x1b000~1000 2019-12-07T19:34:15.649+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc d342ec03 2019-12-07T19:34:15.649+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.649+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:15.649+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 10 osd.7 38 tick_without_osd_lock 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fa5d000/0x40000000/0x1680000000, data 0x5c2572/0x5a1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 25 osd.7 38 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:11.574325+0000 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 25 osd.7 38 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574316+0000 last_rx_front 2019-12-07T19:34:11.574282+0000 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 25 osd.7 38 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574261+0000 last_rx_front 2019-12-07T19:34:11.574293+0000 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 25 osd.7 38 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574240+0000 last_rx_front 2019-12-07T19:34:11.574226+0000 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 25 osd.7 38 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:11.573934+0000 last_rx_back 2019-12-07T19:34:11.574165+0000 last_rx_front 2019-12-07T19:34:11.574203+0000 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 20 osd.7 38 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 10 osd.7 38 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 20 osd.7 38 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:15.652+0000 7f8f1fbbe700 10 osd.7 38 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:15.673+0000 7f8f0097a700 5 osd.7 38 heartbeat osd_stat(store_statfs(0x163fa5d000/0x40000000/0x1680000000, data 0x5c2572/0x5a1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5] op hist []) 2019-12-07T19:34:15.673+0000 7f8f0097a700 20 osd.7 38 check_full_status cur ratio 6.11623e-05, physical ratio 6.11623e-05, new state none 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.688380759s) v5 -- 0x558c576b4200 con 0x558c54c9e000 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.688380759s) v5 -- 0x558c5501d600 con 0x558c54c9e400 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.673538248s) v5 -- 0x558c54dfa600 con 0x558c551b7800 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.673538248s) v5 -- 0x558c54dfbc00 con 0x558c551b7c00 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.687576737s) v5 -- 0x558c54dfbe00 con 0x558c552c2000 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.687576737s) v5 -- 0x558c57612000 con 0x558c552c2400 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.696039918s) v5 -- 0x558c57612200 con 0x558c552c2800 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.696039918s) v5 -- 0x558c57612400 con 0x558c552c2c00 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.678891847s) v5 -- 0x558c57612600 con 0x558c54c65800 2019-12-07T19:34:15.673+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.109813093s delta_ub 18446744073.678891847s) v5 -- 0x558c57612800 con 0x558c54c65c00 2019-12-07T19:34:15.673+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 7 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.140890516s delta_ub 0.031077423s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af000 con 0x558c54c65800 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574165+0000 -> 2019-12-07T19:34:15.674643+0000 last_rx_front 2019-12-07T19:34:11.574203+0000 2019-12-07T19:34:15.673+0000 7f8f23c56700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030807757s,0.031077423s]) 2019-12-07T19:34:15.673+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 11 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.146370796s delta_ub 0.036557703s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c551b7c00 2019-12-07T19:34:15.673+0000 7f8f24c58700 25 osd.7 38 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574316+0000 last_rx_front 2019-12-07T19:34:11.574282+0000 -> 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:15.673+0000 7f8f24c58700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036261688s,0.036557703s]) 2019-12-07T19:34:15.673+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 7 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.140914678s delta_ub 0.031101585s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af000 con 0x558c54c65c00 2019-12-07T19:34:15.673+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 11 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.123844680s delta_ub 0.014031587s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54dfac00 con 0x558c552c2c00 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674643+0000 last_rx_front 2019-12-07T19:34:11.574203+0000 -> 2019-12-07T19:34:15.674689+0000 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:15.674373+0000) and older pending ping(s) 2019-12-07T19:34:15.673+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 7 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.131581031s delta_ub 0.021767938s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c54c9e400 2019-12-07T19:34:15.673+0000 7f8f23c56700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030785900s,0.031101585s]) 2019-12-07T19:34:15.673+0000 7f8f24457700 25 osd.7 38 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574240+0000 last_rx_front 2019-12-07T19:34:11.574226+0000 -> 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:15.673+0000 7f8f24457700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013697977s,0.014031587s]) 2019-12-07T19:34:15.673+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 11 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.132164572s delta_ub 0.022351479s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af000 con 0x558c552c2400 2019-12-07T19:34:15.673+0000 7f8f24c58700 25 osd.7 38 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 last_rx_front 2019-12-07T19:34:11.574325+0000 -> 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:15.673+0000 7f8f24c58700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021420037s,0.021767938s]) 2019-12-07T19:34:15.673+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 11 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.132197830s delta_ub 0.022384737s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54dfac00 con 0x558c552c2000 2019-12-07T19:34:15.673+0000 7f8f24457700 25 osd.7 38 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574261+0000 -> 2019-12-07T19:34:15.674735+0000 last_rx_front 2019-12-07T19:34:11.574293+0000 2019-12-07T19:34:15.673+0000 7f8f24457700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022022877s,0.022384737s]) 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674735+0000 last_rx_front 2019-12-07T19:34:11.574293+0000 -> 2019-12-07T19:34:15.674747+0000 2019-12-07T19:34:15.673+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 11 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.123804704s delta_ub 0.013991611s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c552c2800 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:15.674373+0000) and older pending ping(s) 2019-12-07T19:34:15.673+0000 7f8f23c56700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021978047s,0.022351479s]) 2019-12-07T19:34:15.673+0000 7f8f24c58700 25 osd.7 38 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574240+0000 -> 2019-12-07T19:34:15.674761+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:15.673+0000 7f8f24c58700 25 osd.7 38 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:15.674373+0000) and older pending ping(s) 2019-12-07T19:34:15.673+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 11 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.146412194s delta_ub 0.036599101s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af000 con 0x558c551b7800 2019-12-07T19:34:15.673+0000 7f8f24c58700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013603746s,0.013991611s]) 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574316+0000 -> 2019-12-07T19:34:15.674781+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:15.674373+0000) and older pending ping(s) 2019-12-07T19:34:15.673+0000 7f8f23c56700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036191528s,0.036599101s]) 2019-12-07T19:34:15.673+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 7 ==== osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:15.674373+0000/39.109813093s send_stamp 39.131657262s delta_ub 0.021844169s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af000 con 0x558c54c9e000 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:11.574273+0000 -> 2019-12-07T19:34:15.674802+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:15.673+0000 7f8f23c56700 25 osd.7 38 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:15.674373+0000) and older pending ping(s) 2019-12-07T19:34:15.673+0000 7f8f23c56700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021415354s,0.021844169s]) 2019-12-07T19:34:15.699+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.749+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.800+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.850+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.900+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:15.950+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.000+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.051+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 13 ==== osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:16.054115+0000/39.511627733s send_stamp 39.511627733s delta_ub 0.022384737s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551af000 con 0x558c54c18000 2019-12-07T19:34:16.053+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 13 ==== osd_ping(ping e38 up_from 13 ping_stamp 2019-12-07T19:34:16.054115+0000/39.511627733s send_stamp 39.511627733s delta_ub 0.022384737s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54dfac00 con 0x558c552d1400 2019-12-07T19:34:16.053+0000 7f8f23c56700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021901769s,0.022384737s]) 2019-12-07T19:34:16.053+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:16.054115+0000/39.511627733s send_stamp 39.489725964s delta_ub 18446744073.687649847s) v5 -- 0x558c57612000 con 0x558c54c18000 2019-12-07T19:34:16.053+0000 7f8f24457700 20 osd.7 38 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021880519s,0.022384737s]) 2019-12-07T19:34:16.053+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e38 up_from 13 ping_stamp 2019-12-07T19:34:16.054115+0000/39.511627733s send_stamp 39.489747214s delta_ub 18446744073.687671097s) v5 -- 0x558c57612400 con 0x558c552d1400 2019-12-07T19:34:16.101+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.150+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.201+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.251+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.301+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.351+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 tick 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 maybe_update_heartbeat_peers updating 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 adding neighbor peer osd.1 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 adding neighbor peer osd.3 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 adding neighbor peer osd.5 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 adding random peer osd.2 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 adding random peer osd.4 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 maybe_update_heartbeat_peers 5 peers, extras 1,2,3,4,5 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:16.398+0000 7f8f21451700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:16.398+0000 7f8f21451700 20 osd.7 38 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:16.401+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.451+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.502+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.551+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.602+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 589669 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 23 ==== osd_map(38..39 src has 1..39) v4 ==== 2222+0+0 (crc 0 0 0) 0x558c54dd1440 con 0x558c5476d000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 osd.7 38 OSD::ms_dispatch: osd_map(38..39 src has 1..39) v4 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 osd.7 38 _dispatch 0x558c54dd1440 osd_map(38..39 src has 1..39) v4 2019-12-07T19:34:16.614+0000 7f8f164bb700 3 osd.7 38 handle_osd_map epochs [38,39], i have 38, src has [1,39] 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 osd.7 38 handle_osd_map got inc map for epoch 39 2019-12-07T19:34:16.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3d48e826:::osdmap.38:0# 0x0~0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57e63110 blob([0x5a1000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57e63030 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efc000 blob([!~1000,0x5a2000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c57bea7e0 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3d48e826:::osdmap.38:0# 0x0~1454 = 5204 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 osd.7 38 add_map_bl 38 5204 bytes 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 osd.7 38 got_full_map 39, nothing requested 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 osd.7 38 handle_osd_map pg_num_history pg_num_history(e39 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {39={}} 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 osd.7 38 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e39 [1,39] lci=[13,39]) 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c577fd200 seq 30 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:ceab972f:::inc_osdmap.39:0# key 0x7f7fffffffffffffffceab97'/!inc_osdmap.39!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ceab972f:::inc_osdmap.39:0# 0x0~4be 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1172 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:ceab972f:::inc_osdmap.39:0# 0x0~4be - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~4be 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb42 on front/back, now 0x0~1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fd200 1 blobs 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5a3000~1000/1000,1000,0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5a3000~1000] 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bd500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57880 blob([0x5a3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd500 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54e57880) get_ref 0x0~4be Blob(0x558c54e57880 blob([0x5a3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd500 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54e57880) get_ref init 0x1000, 1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~4be: 0x0~4be Blob(0x558c54e57880 blob([0x5a3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4be) SharedBlob(0x558c575bd500 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c575bd518 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4be 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58093288) dirty_range mark inline shard dirty 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ceab972f:::inc_osdmap.39:0# 0x0~4be = 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3248e826:::osdmap.39:0# key 0x7f7fffffffffffffff3248e8'&!osdmap.39!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3248e826:::osdmap.39:0# 0x0~1681 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1173 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3248e826:::osdmap.39:0# 0x0~1681 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~681 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x97f on front/back, now 0x0~1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fd200 2 blobs 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5a4000~2000/1000,2000,0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5a4000~2000] 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3110 blob([0x5a4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314620 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c574d3110) get_ref 0x0~1000 Blob(0x558c574d3110 blob([0x5a4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314620 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c574d3110) get_ref init 0x1000, 1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c574d3110 blob([0x5a4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314620 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57314638 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efc310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efc380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efc310 blob([!~1000,0x5a5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc380 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc310) get_ref 0x1000~681 Blob(0x558c57efc310 blob([!~1000,0x5a5000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57efc380 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc310) get_ref init 0x2000, 1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~681: 0x1000~681 Blob(0x558c57efc310 blob([!~1000,0x5a5000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,681]) SharedBlob(0x558c57efc380 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efc398 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1681 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efa608) dirty_range mark inline shard dirty 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3248e826:::osdmap.39:0# 0x0~1681 = 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fd200 0 blobs 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fd200 0 blobs 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd200 cost 24456 (4 ios * 4000 + 8456 bytes) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd200 onodes 0x558c53a91900,0x558c55276c80,0x558c57efa500,0x558c58093180 shared_blobs 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efa608) update #-1:3248e826:::osdmap.39:0# 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efa608) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3248e826:::osdmap.39:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58093288) update #-1:ceab972f:::inc_osdmap.39:0# 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58093288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:ceab972f:::inc_osdmap.39:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd200 allocated 0x[5a3000~3000] released 0x[] 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 freelist allocate 0x5a3000~3000 2019-12-07T19:34:16.614+0000 7f8f164bb700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 prepare 2019-12-07T19:34:16.614+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fd200 2019-12-07T19:34:16.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 aio_wait 2019-12-07T19:34:16.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd200 2019-12-07T19:34:16.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 io_done 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4981 < min_flush_size 524288 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efa500 had 1 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093180 had 1 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8859 < min_flush_size 524288 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8859 < min_flush_size 524288 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xdec8b~229b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xdec8b 0xdec8b~229b to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:14.638871+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _allocate len 0xf26 from 0 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _allocate len 0xf26 from 1 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd210000 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd210000~10000/10000,10000,acd210000 2019-12-07T19:34:16.614+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0xf0000 to include allocated 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 36 (was clean) 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xaec8b 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc8b 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf26 and padding block with 0xda 2019-12-07T19:34:16.614+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0xf0000 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe0f26 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (36) on file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]), flushing log 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 36 len 0x49 crc 0x4502c87) 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x23000~1000 to file(ino 1 size 0x23000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x23000 0x23000~1000 to file(ino 1 size 0x23000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x24000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x23000 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x24000 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:16.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 127 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (unknown 0 0 0) 0x558c574e4700 con 0x558c54c5b400 2019-12-07T19:34:16.615+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 126 ==== osd_map(39..39 src has 1..39) v4 ==== 1258+0+0 (unknown 0 0 0) 0x558c576b3200 con 0x558c54c5b400 2019-12-07T19:34:16.615+0000 7f8f1199c700 20 osd.7 38 OSD::ms_dispatch: osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.615+0000 7f8f1199c700 10 osd.7 38 do_waiters -- start 2019-12-07T19:34:16.615+0000 7f8f1199c700 10 osd.7 38 do_waiters -- finish 2019-12-07T19:34:16.615+0000 7f8f1199c700 20 osd.7 38 _dispatch 0x558c576b3200 osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.615+0000 7f8f1199c700 3 osd.7 38 handle_osd_map epochs [39,39], i have 39, src has [1,39] 2019-12-07T19:34:16.615+0000 7f8f1199c700 10 osd.7 38 no new maps here, dropping 2019-12-07T19:34:16.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000703 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.615+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.615+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.615+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 39, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 36 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.001070789s (0.000041792s flush + 0.001028997s kv commit) 2019-12-07T19:34:16.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd200> 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c57e672c0> 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 kv_submitted 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd200 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c577fd200 osr 0x558c54ea4b40 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) prepare_write seq 28 0x1b000~1000 crc c99d40fe 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) prepare_write seq 28 0x2000~1000 crc 70cff969 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 deferred_cleanup 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc600 onodes 0x558c53a91900,0x558c54d96000,0x558c55276c80,0x558c57efb680 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55278e18 in 0x558c53a59340) _finish_write discard buffer(0x558c57a593b0 space 0x558c55278e18 0x0~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279588 in 0x558c53a59340) _finish_write discard buffer(0x558c57a59440 space 0x558c55279588 0x1000~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d278 in 0x558c53a59340) _finish_write discard buffer(0x558c57a59320 space 0x558c5794d278 0x0~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc600 done 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc900 deferred_cleanup 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc600 [] 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 deferred_cleanup 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc900 onodes 0x558c53a91900,0x558c55276c80,0x558c57efb900,0x558c57efbb80 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677358 in 0x558c53a59340) _finish_write discard buffer(0x558c57a595f0 space 0x558c54677358 0x0~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538788 in 0x558c53a59340) _finish_write discard buffer(0x558c57a59680 space 0x558c57538788 0x0~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efc1d8 in 0x558c53a59340) _finish_write discard buffer(0x558c57a59710 space 0x558c57efc1d8 0x1000~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc900 done 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde300 deferred_cleanup 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc900 [] 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde300 deferred_cleanup 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde300 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34a00,0x558c57a34c80 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bea7f8 in 0x558c53a59340) _finish_write discard buffer(0x558c580cd560 space 0x558c57bea7f8 0x1000~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e63048 in 0x558c53a59340) _finish_write discard buffer(0x558c580cd5f0 space 0x558c57e63048 0x0~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e63518 in 0x558c53a59340) _finish_write discard buffer(0x558c580cd680 space 0x558c57e63518 0x0~1000 writing nocache) 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde300 done 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd200 deferred_queued 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde300 [] 2019-12-07T19:34:16.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 128 ==== pg_query2(1.2s0 1.2s0 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c574a0280 con 0x558c54c5b400 2019-12-07T19:34:16.616+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s0 epoch_sent: 39 epoch_requested: 39 MQuery 1.2s0 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.616+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.2s0 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 129 ==== pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c574a0280 con 0x558c54c5b400 2019-12-07T19:34:16.616+0000 7f8f24457700 15 osd.7 38 enqueue_peering_evt 1.2s4 epoch_sent: 39 epoch_requested: 39 MQuery 1.2s4 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.616+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.2s4 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7 38 _committed_osd_maps 39..39 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7 38 advance to epoch 39 (<= last 39 <= newest_map 39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 7 osd.7 39 consume_map version 39 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:0.identify_splits_and_merges identify_splits_and_merges slot 1.8s5 has no pg and waiting_for_split 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.0s3 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:1.identify_splits_and_merges identify_splits_and_merges slot 1.1s1 has no pg and waiting_for_split 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.9s4 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:2.identify_splits_and_merges identify_splits_and_merges slot 1.as5 has no pg and waiting_for_split 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.2s0 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.2s4 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.bs3 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs1 has no pg and waiting_for_split 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.3s4 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.cs0 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.4s0 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:5.identify_splits_and_merges identify_splits_and_merges slot 1.5s4 has no pg and waiting_for_split 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.ds2 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:6.identify_splits_and_merges identify_splits_and_merges slot 1.es4 has no pg and waiting_for_split 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.6s2 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:7.identify_splits_and_merges identify_splits_and_merges slot 1.fs5 has no pg and waiting_for_split 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 identify_splits_and_merges 1.7s4 e38 to e39 pg_nums {18=16} 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 39 prune_pg_created 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:0.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 empty, pruning 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:1.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 empty, pruning 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:2.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:16.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 130 ==== pg_query2(1.bs1 1.bs1 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c574a0280 con 0x558c54c5b400 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 empty, pruning 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:3.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 empty, pruning 2019-12-07T19:34:16.616+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.bs1 epoch_sent: 39 epoch_requested: 39 MQuery 1.bs1 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.616+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.bs1 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:4.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 pending_peering first epoch 39 <= 39, requeueing 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.4s0 to_process <> waiting <> waiting_peering {39=0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)>} 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:5.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 empty, pruning 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:6.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 empty, pruning 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7:7.consume_map 39 (was 38) 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 empty, pruning 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] handle_advance_map: 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 38 -> 39 2019-12-07T19:34:16.616+0000 7f8f141a1700 10 osd.7 39 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] handle_advance_map [7,3,1,5,2,4]/[7,2147483647,1,5,2,4] -- 7/7 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.0s3 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] state: verifying no want_acting [] targets didn't go down 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.9s4 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.2s0 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.2s4 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.bs3 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.3s4 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,2147483647,1,5,2,4] 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.cs0 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.ds2 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] exit Started/Primary/WaitActingChange 2019-12-07T19:34:16.617691+0000 1 0.000225 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.6s2 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f141a1700 15 osd.7 39 enqueue_peering_evt 1.7s4 epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] exit Started/Primary 2019-12-07T19:34:16.617701+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f141a1700 7 osd.7 39 activate_map version 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c5500a000 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:16.617729+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=38 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] handle_advance_map: 39 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 38 -> 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 38 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2147483647,7,3] -- 1/1 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] new interval newup [1,4,5,2,7,3] newacting [1,4,5,2147483647,7,3] 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,2147483647,1,5,2,4] 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.616+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:16.617796+0000 3 0.000066 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 39 clear_ready_to_merge 1.4s0 2019-12-07T19:34:16.616+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:16.617809+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,38)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.616+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c55141000 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [7,3,1,5,2,4](7) acting [7,3,1,5,2,4](7)) up_thru 28 up_from 13 last_epoch_clean 19 interval(38-38 up [7,3,1,5,2,4](7) acting [7,3,1,5,2,4](7)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] noting past ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0))) 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] handle_advance_map: 39 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 38 -> 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 38 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,2147483647] -- 2/2 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230,0x558c54677f10,0x558c5523a150,0x558c5523a3f0] 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] new interval newup [1,4,5,2,7,3] newacting [1,4,5,2147483647,7,3] 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 40.053306209s + 0s) 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] new interval newup [2,1,7,3,4,5] newacting [2,1,7,3,4,2147483647] 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:10.918610+0000, must = 0 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.616+0000 7f8f0297e700 1 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] start_peering_interval up [7,3,1,5,2,4] -> [7,3,1,5,2,4], acting [7,3,1,5,2,4] -> [7,2147483647,1,5,2,4], acting_primary 7(0) -> 7, up_primary 7(0) -> 7, role 0 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.616+0000 7f8f08189700 15 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=34'17 unknown NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:16.616+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:16.617892+0000 3 0.000143 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.616+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:16.617903+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [1,4,5,2,7,3](1) acting [1,4,5,2,7,3](1)) up_thru 28 up_from 13 last_epoch_clean 29 interval(38-38 up [1,4,5,2,7,3](1) acting [1,4,5,2,7,3](1)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:16.616+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=38 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.041584949s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] noting past ([28,38] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([28,37] acting 0(3),1(0),3(5),4(1),5(2),7(4))) 2019-12-07T19:34:16.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] publish_stats_to_osd 38:58 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=39 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.041584949s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=39 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.041584949s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=39 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.041584949s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=39 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] new interval newup [2,1,7,3,4,5] newacting [2,1,7,3,4,2147483647] 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] [7,3,1,5,2,4] -> [7,2147483647,1,5,2,4], replicas changed 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.093172886s (mnow 40.053398114s + 14.039774772s) 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=39 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]p2(0) r=2 lpr=39 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:16.616+0000 7f8f0217d700 15 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,38)/1 crt=21'6 unknown NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.616+0000 7f8f08189700 1 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] start_peering_interval up [1,4,5,2,7,3] -> [1,4,5,2,7,3], acting [1,4,5,2,7,3] -> [1,4,5,2147483647,7,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] handle_activate_map: 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [2,1,7,3,4,5](2) acting [2,1,7,3,4,5](2)) up_thru 28 up_from 13 last_epoch_clean 19 interval(38-38 up [2,1,7,3,4,5](2) acting [2,1,7,3,4,5](2)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:16.618010+0000 1 0.000325 2019-12-07T19:34:16.616+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=38 pruub=12.997145394s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] noting past ([18,38] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([18,37] acting 0(5),1(1),2(0),3(3),4(4),7(2))) 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.997145394s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.2s0 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f0297e700 1 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] state: transitioning to Primary 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c54d9b000 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.2s0 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0x558c57e88000 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.997145394s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:16.618042+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.997145394s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a150] 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] enter Started/Primary 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 52.054860248s (mnow 40.053493836s + 12.001366412s) 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] enter Started/Primary/Peering 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] handle_advance_map: 39 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 38 -> 39 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:16.616+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_advance_map: 39 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 38 -> 39 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] handle_advance_map [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5] -- 4/4 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] build_prior all_probe 0(1),1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c54d5c000 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] handle_advance_map [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3] -- 7/4 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] build_prior maybe_rw interval:18, acting: 0(1),1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] build_prior prior osd.0 is down 2019-12-07T19:34:16.616+0000 7f8f0217d700 1 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] start_peering_interval up [2,1,7,3,4,5] -> [2,1,7,3,4,5], acting [2,1,7,3,4,5] -> [2,1,7,3,4,2147483647], acting_primary 2(0) -> 2, up_primary 2(0) -> 2, role 2 -> 2, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: verifying no want_acting [] targets didn't go down 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,2147483647,2,2147483647,1,5] 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] build_prior final: probe 1(2),2(4),3(1),4(5),5(3),7(0) down 0 blocked_by {} 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] up_thru 38 < same_since 39, must notify monitor 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] handle_advance_map: 39 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 38 -> 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: querying info from osd.1(2) 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] new interval newup [7,4,1,5,2,3] newacting [2147483647,4,1,5,2147483647,3] 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] release_backoffs [1:b0000000::::0,1:c0000000::::head) 2019-12-07T19:34:16.616+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:16.618100+0000 3 0.000059 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: querying info from osd.2(4) 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 38 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: querying info from osd.3(1) 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.616+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:16.618111+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: querying info from osd.4(5) 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:16.616+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started/Primary/WaitActingChange 2019-12-07T19:34:16.618113+0000 2 0.000202 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.616+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: querying info from osd.5(3) 2019-12-07T19:34:16.616+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started/Primary 2019-12-07T19:34:16.618122+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=38 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] check_prior_readable_down_osds prior_readable_down_osds osd.0 is dead as of epoch 39 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=39 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=39 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.616+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] exit Started 2019-12-07T19:34:16.618145+0000 0 0.000000 2019-12-07T19:34:16.616+0000 7f8f0297e700 15 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] publish_stats_to_osd 38:59 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=39 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] handle_activate_map: 39 2019-12-07T19:34:16.616+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:16.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=38 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] activate_map 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=39 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,2147483647,2,2147483647,1,5] 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=39 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=39 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:16.616+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=39 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=39 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: Reset advmap 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.616+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:16.616+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]p4(0) r=-1 lpr=39 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=39 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] new interval newup [7,4,1,5,2,3] newacting [2147483647,4,1,5,2147483647,3] 2019-12-07T19:34:16.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=39 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:16.616+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=0 lpr=39 pi=[28,38)/1 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:16.616+0000 7f8f03980700 10 osd.7 39 clear_ready_to_merge 1.2s0 2019-12-07T19:34:16.616+0000 7f8f07187700 15 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,38)/1 crt=21'12 unknown NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:22.460228+0000, must = 0 2019-12-07T19:34:16.616+0000 7f8f03980700 15 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=0 lpr=39 pi=[28,38)/1 crt=0'0 unknown mbc={}] clear_stats 2019-12-07T19:34:16.616+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:16.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:16.616+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [7,4,1,5,2,3](7) acting [7,4,1,5,2,3](7)) up_thru 28 up_from 13 last_epoch_clean 29 interval(38-38 up [7,4,1,5,2,3](7) acting [7,4,1,5,2,3](7)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [4,7,2,3,1,5](4) acting [4,7,2,3,1,5](4)) up_thru 28 up_from 13 last_epoch_clean 29 interval(38-38 up [4,7,2,3,1,5](4) acting [4,7,2,3,1,5](4)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c5500d000 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] noting past ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4))) 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 send_incremental_map 38 -> 39 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] on_new_interval 2019-12-07T19:34:16.617+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ceab972f:::inc_osdmap.39:0# 0x0~0 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~4be size 0x4be (1214) 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] init_hb_stamps now [] 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.039374737s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] noting past ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 40.053669732s + 0s) 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57880 blob([0x5a3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4be) SharedBlob(0x558c575bd500 sbid 0x0)) need 0x0~4be cache has 0x[0~4be] 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] cancel_recovery 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_advance_map: 39 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 38 -> 39 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ceab972f:::inc_osdmap.39:0# 0x0~4be = 1214 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 add_map_inc_bl 39 1214 bytes 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] clear_recovery_state 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.039374737s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 38 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(39..39 src has 1..39) v4 -- 0x558c57325b00 con 0x558c54c64000 2019-12-07T19:34:16.617+0000 7f8f03980700 1 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] start_peering_interval up [7,4,1,5,2,3] -> [7,4,1,5,2,3], acting [7,4,1,5,2,3] -> [2147483647,4,1,5,2147483647,3], acting_primary 7(0) -> 4, up_primary 7(0) -> 7, role 0 -> -1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.039374737s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] clear_primary_state 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001366412s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 38 -> 39 (shared) 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.039374737s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] init_hb_stamps now [] 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.4s2 1.4s2 query(info 0'0 epoch_sent 39) e39/39) v1 -- 0x558c57efaf00 con 0x558c54c64000 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] agent_stop 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.091577138s (mnow 40.053696524s + 14.037880614s) 2019-12-07T19:34:16.617+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 send_incremental_map 38 -> 39 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] on_change 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] cancel_copy_ops 2019-12-07T19:34:16.617+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:16.618260+0000 1 0.000343 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(39..39 src has 1..39) v4 -- 0x558c5730dd40 con 0x558c54c19800 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f0918b700 20 osd.7 39 agent_entry empty queue 2019-12-07T19:34:16.617+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] cancel_flush_ops 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 38 -> 39 (shared) 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.4s4 1.4s4 query(info 0'0 epoch_sent 39) e39/39) v1 -- 0x558c57efb180 con 0x558c54c19800 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] cancel_proxy_ops 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 send_incremental_map 38 -> 39 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f0217d700 1 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:16.617+0000 7f8f07187700 1 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] start_peering_interval up [4,7,2,3,1,5] -> [4,7,2,3,1,5], acting [4,7,2,3,1,5] -> [4,2147483647,2,2147483647,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> -1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] cancel_manifest_ops 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(39..39 src has 1..39) v4 -- 0x558c5742c240 con 0x558c54c5bc00 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] activate_map 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 38 -> 39 (shared) 2019-12-07T19:34:16.617+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:16.618298+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] on_change_cleanup 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.4s1 1.4s1 query(info 0'0 epoch_sent 39) e39/39) v1 -- 0x558c57efb400 con 0x558c54c5bc00 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] on_change 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 send_incremental_map 38 -> 39 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:16.617+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:16.617+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:16.617+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] exit NotTrimming 2019-12-07T19:34:16.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(39..39 src has 1..39) v4 -- 0x558c5742db00 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped mbc={}] enter NotTrimming 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 38 -> 39 (shared) 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] plpg_on_role_change 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.4s5 1.4s5 query(info 0'0 epoch_sent 39) e39/39) v1 -- 0x558c57e8ec80 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 send_incremental_map 38 -> 39 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c54d5f000 2019-12-07T19:34:16.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(39..39 src has 1..39) v4 -- 0x558c5742d680 con 0x558c54c19c00 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] agent_stop 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 38 -> 39 (shared) 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.4s3 1.4s3 query(info 0'0 epoch_sent 39) e39/39) v1 -- 0x558c57e8ef00 con 0x558c54c19c00 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] handle_advance_map: 39 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 38 -> 39 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54ea0f00 seq 20 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map 2019-12-07T19:34:16.617+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 38 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,2147483647,7,2] -- 1/1 2019-12-07T19:34:16.617+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:39 sent:39 1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) 2->0 ([18,38] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([18,37] acting 0(5),1(1),2(0),3(3),4(4),7(2)))) e39/39) v1 -- 0x558c57450700 con 0x558c54c19800 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57ef8300 seq 24 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] exit Reset 2019-12-07T19:34:16.618370+0000 1 0.000245 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 5456 (1 ios * 4000 + 1456 bytes) 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] new interval newup [1,4,3,5,7,2] newacting [1,4,3,2147483647,7,2] 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:16.617+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] enter Started 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] enter Start 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:16.618389+0000 3 0.000044 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 queue_want_up_thru now 39 (was 38), currently 38 2019-12-07T19:34:16.617+0000 7f8f03980700 1 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] state: transitioning to Stray 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8300 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 send_alive up_thru currently 38 want 39 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 send_alive want 39 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8300 onodes shared_blobs 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8300 allocated 0x[] released 0x[] 2019-12-07T19:34:16.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:16.618399+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] exit Start 2019-12-07T19:34:16.618401+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_alive(want up_thru 39 have 39) v1 -- 0x558c53a71a40 con 0x558c5476d000 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 prepare 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8300 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] enter Started/Stray 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:16.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 io_done 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=38 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c57676000 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1300 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_advance_map: 39 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 38 -> 39 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] do_peering_event: epoch_sent: 39 epoch_requested: 39 MQuery 1.2s0 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037880614s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3859873792 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 38 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] sending info 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] new interval newup [1,4,3,5,7,2] newacting [1,4,3,2147483647,7,2] 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998542) bluestore 86 GiB free (0.959938), bluefs_ratio 0.0399469 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:16.618452+0000 1 0.000325 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: Active advmap 2019-12-07T19:34:16.617+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1323 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1323 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f05183700 15 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,38)/1 crt=27'10 unknown NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:16.617+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.2s1 1.2s1 (query:39 sent:39 1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 0->1 ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e39/39) v1 -- 0x558c576de000 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe0f26~52b to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe0f26 0xe0f26~52b to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 131 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c574a0280 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f07187700 1 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb0f26 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf26 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x451 and padding block with 0xbaf 2019-12-07T19:34:16.617+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.bs3 epoch_sent: 39 epoch_requested: 39 MQuery 1.bs3 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.bs3 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.617+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.2s1 1.2s1 (query:39 sent:39 1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 0->1 ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e39/39) v1 -- 0x558c576e1800 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:16.618487+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [1,4,3,5,7,2](1) acting [1,4,3,5,7,2](1)) up_thru 28 up_from 13 last_epoch_clean 29 interval(38-38 up [1,4,3,5,7,2](1) acting [1,4,3,5,7,2](1)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c574ff200 seq 2 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040689079s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] noting past ([28,38] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([28,37] acting 0(3),1(0),2(5),3(2),4(1),7(4))) 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] activate_map 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: Active: handling ActMap 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.040689079s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:16.617+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] Active: kicking snap trim 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff200 cost 5455 (1 ios * 4000 + 1455 bytes) 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.040689079s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff200 onodes shared_blobs 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff200 allocated 0x[] released 0x[] 2019-12-07T19:34:16.617+0000 7f8f0297e700 7 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: handle ActMap primary 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.040689079s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:16.617+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 prepare 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff200 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 io_done 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.095126152s (mnow 40.053975752s + 14.041150400s) 2019-12-07T19:34:16.617+0000 7f8f0297e700 15 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] publish_stats_to_osd 38:120 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.2s4 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:39 sent:39 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) 3->0 ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e39/39) v1 -- 0x558c57453800 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.2s4 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0x558c5731a000 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f05183700 1 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] start_peering_interval up [1,4,3,5,7,2] -> [1,4,3,5,7,2], acting [1,4,3,5,7,2] -> [1,4,3,2147483647,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c548f6000 seq 20 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:16.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f0297e700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 39 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6000 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6000 onodes shared_blobs 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 38 -> 39 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6000 allocated 0x[] released 0x[] 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 prepare 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 38 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3] -- 7/4 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6000 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 io_done 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [2147483647,4,1,5,2147483647,3] 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c5513e000 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:16.618624+0000 3 0.000073 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:16.618633+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] handle_advance_map: 39 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 38 -> 39 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe1451 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=38 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 38 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,2147483647] -- 4/4 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=39 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000008 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:16.617+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=39 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000257838s (0.000000101s flush + 0.000257737s kv commit) 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] new interval newup [4,1,2,5,7,3] newacting [4,1,2,5,7,2147483647] 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=39 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=39 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [2147483647,4,1,5,2147483647,3] 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:16.618678+0000 3 0.000066 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=39 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2579 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.041150400s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]p7(0) r=-1 lpr=39 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:16.618689+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3870 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f03980700 15 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,38)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:16.618697+0000 1 0.000295 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3893 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3893 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:16.617+0000 7f8f05183700 1 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe1451~f35 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:16.618719+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=38 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe1451 0xe1451~f35 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [7,4,1,5,2,3](7) acting [7,4,1,5,2,3](7)) up_thru 28 up_from 13 last_epoch_clean 29 interval(38-38 up [7,4,1,5,2,3](7) acting [7,4,1,5,2,3](7)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.617+0000 7f8f05183700 5 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=39 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb1451 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x451 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x386 and padding block with 0xc7a 2019-12-07T19:34:16.617+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=39 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] noting past ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4))) 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=39 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:16.617+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=39 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] new interval newup [4,1,2,5,7,3] newacting [4,1,2,5,7,2147483647] 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] init_hb_stamps now [] 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=39 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:16.617+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:39 sent:39 1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) 4->0 ([28,38] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([28,37] acting 0(3),1(0),2(5),3(2),4(1),7(4)))) e39/39) v1 -- 0x558c576bb800 con 0x558c54c64000 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]p4(0) r=4 lpr=39 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] set_last_peering_reset 39 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 40.054203335s + 0s) 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:16.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.617+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:16.617+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59200 seq 40 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:16.617+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:16.617+0000 7f8f07187700 15 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,38)/1 crt=37'13 unknown NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:16.617+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:16.617+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:16.617+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:16.617+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:16.617+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:16.617+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:16.617+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:16.617+0000 7f8f03980700 1 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] start_peering_interval up [7,4,1,5,2,3] -> [7,4,1,5,2,3], acting [7,4,1,5,2,3] -> [2147483647,4,1,5,2147483647,3], acting_primary 7(0) -> 4, up_primary 7(0) -> 7, role -1 -> -1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.617+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:16.617+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:16.617+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(38-38 up [4,1,2,5,7,3](4) acting [4,1,2,5,7,3](4)) up_thru 28 up_from 13 last_epoch_clean 29 interval(38-38 up [4,1,2,5,7,3](4) acting [4,1,2,5,7,3](4)) : primary up 13-28 does not include interval 2019-12-07T19:34:16.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe2386 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:16.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=38 pruub=15.040707292s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] noting past ([28,38] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([28,37] acting 0(5),1(1),2(2),4(0),5(3),7(4))) 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000149234s (0.000000105s flush + 0.000149129s kv commit) 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.040707292s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.040707292s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=15.040707292s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe2386~517 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 54.092290280s (mnow 40.054285085s + 14.038005195s) 2019-12-07T19:34:16.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe2386 0xe2386~517 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb2386 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8300,0x558c574ff200,0x558c548f6000> 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 kv_submitted 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8300 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 finishing 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8300 onodes 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8300 done 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8300 [] 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 kv_submitted 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff200 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 finishing 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff200 onodes 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 done 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:16.617+0000 7f8f07187700 1 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] start_peering_interval up [4,1,2,5,7,3] -> [4,1,2,5,7,3], acting [4,1,2,5,7,3] -> [4,1,2,5,7,2147483647], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff200 [] 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 kv_submitted 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6000 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 finishing 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6000 onodes 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 done 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] [7,4,1,5,2,3] -> [2147483647,4,1,5,2147483647,3], acting primary 7(0) -> 4(1) 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6000 [] 2019-12-07T19:34:16.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:16.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:16.618941+0000 1 0.000293 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:16.617+0000 7f8f03980700 1 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:16.618972+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:16.617+0000 7f8f03980700 5 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] handle_activate_map: 39 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MQuery 1.2s4 from 4(1) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038005195s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:16.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:16.619027+0000 1 0.000324 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:16.617+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.2s1 1.2s1 (query:39 sent:39 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->1 ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e39/39) v1 -- 0x558c576dfc00 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:16.617+0000 7f8f07187700 1 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:16.617+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.2s1 1.2s1 (query:39 sent:39 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->1 ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e39/39) v1 -- 0x558c57378e00 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:16.619059+0000 0 0.000000 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c574ff800 seq 27 2019-12-07T19:34:16.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:16.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:16.617+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff800 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff800 onodes shared_blobs 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff800 allocated 0x[] released 0x[] 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 prepare 2019-12-07T19:34:16.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff800 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 io_done 2019-12-07T19:34:16.617+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c57e88000 2019-12-07T19:34:16.617+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:39 sent:39 1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) 4->0 ([28,38] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([28,37] acting 0(5),1(1),2(2),4(0),5(3),7(4)))) e39/39) v1 -- 0x558c576df500 con 0x558c54c5b400 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c548f6c00 seq 40 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 NullEvt) prio 255 cost 10 e39) pg 0x558c5731a000 2019-12-07T19:34:16.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:16.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:16.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes shared_blobs 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:16.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:34:16.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:34:16.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.bs1 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.bs1 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 39 handle_pg_query_nopg 1.bs1 2019-12-07T19:34:16.617+0000 7f8f07187700 10 osd.7 39 pg 1.bs1 dne 2019-12-07T19:34:16.617+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:39 sent:39 1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 39) v7 -- 0x558c54d9fa40 con 0x558c54c5b400 2019-12-07T19:34:16.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.bs3 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.bs3 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0x558c54d9b000 2019-12-07T19:34:16.618+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037690768s) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MQuery 1.bs3 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.618+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY pruub 54.091577138s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 2019-12-07T19:34:16.618+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:16.618+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:16.618+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:39 sent:39 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 3->0 ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e39/39) v1 -- 0x558c576e0300 con 0x558c54c5b400 2019-12-07T19:34:16.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:16.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c54dde000 seq 21 2019-12-07T19:34:16.618+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:16.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:16.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:16.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:16.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde000 onodes shared_blobs 2019-12-07T19:34:16.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde000 allocated 0x[] released 0x[] 2019-12-07T19:34:16.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 prepare 2019-12-07T19:34:16.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde000 2019-12-07T19:34:16.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 io_done 2019-12-07T19:34:16.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.616+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:16.618+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:16.618+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:16.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:16.618+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:16.618+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:16.618+0000 7f8f08189700 20 osd.7 39 advance_pg new pool opts old pool opts 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] handle_activate_map: 39 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.039774772s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:16.618+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:16.619468+0000 1 0.001652 2019-12-07T19:34:16.618+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:16.618+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:16.618+0000 7f8f08189700 1 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:16.618+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:16.619487+0000 0 0.000000 2019-12-07T19:34:16.618+0000 7f8f08189700 5 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:16.618+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 39 2019-12-07T19:34:16.618+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:16.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 NullEvt 2019-12-07T19:34:16.618+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:16.618+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:39 sent:39 1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) 4->0 ([28,38] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([28,37] acting 0(3),1(0),3(5),4(1),5(2),7(4)))) e39/39) v1 -- 0x558c54e04e00 con 0x558c54c64000 2019-12-07T19:34:16.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:16.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740bb00 seq 41 2019-12-07T19:34:16.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:16.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:16.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:16.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:16.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:16.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:16.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:34:16.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:16.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:16.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:16.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:16.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x386 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x89d and padding block with 0x763 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe289d 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001100221s (0.000000075s flush + 0.001100146s kv commit) 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2571 < min_flush_size 524288 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3594 < min_flush_size 524288 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4874 < min_flush_size 524288 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4897 < min_flush_size 524288 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4897 < min_flush_size 524288 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe289d~1321 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe289d 0xe289d~1321 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb289d 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x89d 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbbe and padding block with 0x442 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:16.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:16.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:16.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.618+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe3bbe 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000158412s (0.000000120s flush + 0.000158292s kv commit) 2019-12-07T19:34:16.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff800,0x558c548f6c00,0x558c54dde000,0x558c5740bb00> 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff800 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 finishing 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff800 onodes 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 done 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff800 [] 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 finishing 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:16.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 kv_submitted 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde000 2019-12-07T19:34:16.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde000 finishing 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde000 onodes 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde000 done 2019-12-07T19:34:16.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde000 [] 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 132 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c574a0280 con 0x558c54c5b400 2019-12-07T19:34:16.619+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.3s4 epoch_sent: 39 epoch_requested: 39 MQuery 1.3s4 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.3s4 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.3s4 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.3s4 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0x558c5513e000 2019-12-07T19:34:16.619+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.037832375s) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MQuery 1.3s4 from 4(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 133 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (unknown 0 0 0) 0x558c576e1800 con 0x558c54c5b400 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY pruub 54.092290280s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 2019-12-07T19:34:16.619+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:16.619+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:16.619+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:39 sent:39 1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->0 ([28,38] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([28,37] acting 0(5),1(1),2(2),4(0),5(3),7(4)))) e39/39) v1 -- 0x558c57391500 con 0x558c54c5b400 2019-12-07T19:34:16.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:16.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 190 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57e8ef00 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ddef00 seq 41 2019-12-07T19:34:16.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:16.619+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.es4 epoch_sent: 39 epoch_requested: 39 MQuery 1.es4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.619+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.es4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:16.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:16.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddef00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:16.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddef00 onodes shared_blobs 2019-12-07T19:34:16.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddef00 allocated 0x[] released 0x[] 2019-12-07T19:34:16.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 191 ==== pg_query2(1.as5 1.as5 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efaf00 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 prepare 2019-12-07T19:34:16.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddef00 2019-12-07T19:34:16.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 io_done 2019-12-07T19:34:16.619+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.as5 epoch_sent: 39 epoch_requested: 39 MQuery 1.as5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.619+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.as5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 192 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efaf00 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:16.619+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.7s4 epoch_sent: 39 epoch_requested: 39 MQuery 1.7s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.619+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.7s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe3bbe~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 193 ==== pg_query2(1.8s5 1.8s5 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efaf00 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe3bbe 0xe3bbe~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.8s5 epoch_sent: 39 epoch_requested: 39 MQuery 1.8s5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.619+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.8s5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb3bbe 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbbe 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfd4 and padding block with 0x2c 2019-12-07T19:34:16.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.7s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.7s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0x558c54d5f000 2019-12-07T19:34:16.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.es4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.619+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.040993882s) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MQuery 1.7s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.es4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0 2019-12-07T19:34:16.619+0000 7f8f05984700 10 osd.7 39 handle_pg_query_nopg 1.es4 2019-12-07T19:34:16.619+0000 7f8f05984700 10 osd.7 39 pg 1.es4 dne 2019-12-07T19:34:16.619+0000 7f8f05183700 20 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY pruub 54.095126152s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 2019-12-07T19:34:16.619+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:16.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:39 sent:39 1.es4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 39) v7 -- 0x558c57f112c0 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:16.619+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:39 sent:39 1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->0 ([28,38] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([28,37] acting 0(3),1(0),2(5),3(2),4(1),7(4)))) e39/39) v1 -- 0x558c574e4000 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:16.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59b00 seq 41 2019-12-07T19:34:16.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:16.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:16.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:16.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:16.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:16.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:16.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:16.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:16.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:16.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe3fd4 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000194152s (0.000000090s flush + 0.000194062s kv commit) 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe3fd4~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe3fd4 0xe3fd4~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb3fd4 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfd4 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3ea and padding block with 0xc16 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddef00> 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 kv_submitted 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddef00 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 finishing 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddef00 onodes 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddef00 done 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddef00 [] 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe43ea 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000154550s (0.000000057s flush + 0.000154493s kv commit) 2019-12-07T19:34:16.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00> 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:16.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 194 ==== osd_map(39..39 src has 1..39) v4 ==== 1258+0+0 (crc 0 0 0) 0x558c57325b00 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f1199c700 20 osd.7 39 OSD::ms_dispatch: osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.619+0000 7f8f1199c700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:16.619+0000 7f8f1199c700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:16.619+0000 7f8f1199c700 20 osd.7 39 _dispatch 0x558c57325b00 osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.619+0000 7f8f1199c700 3 osd.7 39 handle_osd_map epochs [39,39], i have 39, src has [1,39] 2019-12-07T19:34:16.619+0000 7f8f1199c700 10 osd.7 39 no new maps here, dropping 2019-12-07T19:34:16.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 4(5) notify: (query:39 sent:39 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got dup osd.4(5) info 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39), identical to ours 2019-12-07T19:34:16.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:16.619+0000 7f8f0297e700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.8s5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.8s5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0 2019-12-07T19:34:16.619+0000 7f8f0898a700 10 osd.7 39 handle_pg_query_nopg 1.8s5 2019-12-07T19:34:16.619+0000 7f8f0898a700 10 osd.7 39 pg 1.8s5 dne 2019-12-07T19:34:16.619+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:39 sent:39 1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 39) v7 -- 0x558c575ccb40 con 0x558c54c64000 2019-12-07T19:34:16.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:16.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 63 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (unknown 0 0 0) 0x558c574e4000 con 0x558c54c19c00 2019-12-07T19:34:16.619+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.620+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 64 ==== osd_map(39..39 src has 1..39) v4 ==== 1258+0+0 (unknown 0 0 0) 0x558c5742d680 con 0x558c54c19c00 2019-12-07T19:34:16.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 65 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (unknown 0 0 0) 0x558c574e4000 con 0x558c54c19c00 2019-12-07T19:34:16.620+0000 7f8f1199c700 20 osd.7 39 OSD::ms_dispatch: osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.620+0000 7f8f1199c700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:16.620+0000 7f8f1199c700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:16.620+0000 7f8f1199c700 20 osd.7 39 _dispatch 0x558c5742d680 osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 2019-12-07T19:34:16.620+0000 7f8f1199c700 3 osd.7 39 handle_osd_map epochs [39,39], i have 39, src has [1,39] 2019-12-07T19:34:16.620+0000 7f8f1199c700 10 osd.7 39 no new maps here, dropping 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:03.092888+0000, must = 0 2019-12-07T19:34:16.620+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:16.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 5(3) notify: (query:39 sent:39 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got dup osd.5(3) info 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39), identical to ours 2019-12-07T19:34:16.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:16.620+0000 7f8f06986700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.as5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.as5 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0 2019-12-07T19:34:16.620+0000 7f8f03980700 10 osd.7 39 handle_pg_query_nopg 1.as5 2019-12-07T19:34:16.620+0000 7f8f03980700 10 osd.7 39 pg 1.as5 dne 2019-12-07T19:34:16.620+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:39 sent:39 1.as5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 39) v7 -- 0x558c54d9f4a0 con 0x558c54c64000 2019-12-07T19:34:16.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:16.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 195 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (crc 0 0 0) 0x558c574e4000 con 0x558c54c64000 2019-12-07T19:34:16.621+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.621+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 196 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (crc 0 0 0) 0x558c574e4000 con 0x558c54c64000 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.621+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.621+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2019-12-07T19:34:16.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 197 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efaf00 con 0x558c54c64000 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:04.837924+0000, must = 0 2019-12-07T19:34:16.621+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.9s4 epoch_sent: 39 epoch_requested: 39 MQuery 1.9s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.621+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.9s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:16.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:16.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.9s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.9s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0x558c5500a000 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.621+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 pruub=14.038272259s) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MQuery 1.9s4 from 1(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 1(2) notify: (query:39 sent:39 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.621+0000 7f8f08189700 20 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY pruub 54.093172886s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 2019-12-07T19:34:16.621+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got dup osd.1(2) info 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39), identical to ours 2019-12-07T19:34:16.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:16.621+0000 7f8f06986700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.621+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:16.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.621+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:39 sent:39 1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->0 ([28,38] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([28,37] acting 0(3),1(0),3(5),4(1),5(2),7(4)))) e39/39) v1 -- 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:16.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:16.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740b500 seq 42 2019-12-07T19:34:16.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:16.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:16.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:16.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:16.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes shared_blobs 2019-12-07T19:34:16.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[] released 0x[] 2019-12-07T19:34:16.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:16.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:16.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:16.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe43ea~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe43ea 0xe43ea~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb43ea 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3ea 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x800 and padding block with 0x800 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe4800 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000080365s (0.000000093s flush + 0.000080272s kv commit) 2019-12-07T19:34:16.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:16.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:16.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:16.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:16.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 2019-12-07T19:34:16.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:16.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:16.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:16.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 116 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57efa780 con 0x558c54c5bc00 2019-12-07T19:34:16.622+0000 7f8f23c56700 15 osd.7 39 enqueue_peering_evt 1.1s1 epoch_sent: 39 epoch_requested: 39 MQuery 1.1s1 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.622+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.1s1 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.1s1 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.1s1 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0 2019-12-07T19:34:16.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 117 ==== pg_query2(1.fs5 1.fs5 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57efb400 con 0x558c54c5bc00 2019-12-07T19:34:16.622+0000 7f8f04181700 10 osd.7 39 handle_pg_query_nopg 1.1s1 2019-12-07T19:34:16.622+0000 7f8f04181700 10 osd.7 39 pg 1.1s1 dne 2019-12-07T19:34:16.622+0000 7f8f23c56700 15 osd.7 39 enqueue_peering_evt 1.fs5 epoch_sent: 39 epoch_requested: 39 MQuery 1.fs5 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.622+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.fs5 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.622+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:39 sent:39 1.1s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 39) v7 -- 0x558c54d0f680 con 0x558c54c5bc00 2019-12-07T19:34:16.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:16.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.fs5 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.fs5 from 3(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0 2019-12-07T19:34:16.622+0000 7f8f0117b700 10 osd.7 39 handle_pg_query_nopg 1.fs5 2019-12-07T19:34:16.622+0000 7f8f0117b700 10 osd.7 39 pg 1.fs5 dne 2019-12-07T19:34:16.622+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:39 sent:39 1.fs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 39) v7 -- 0x558c57ede5a0 con 0x558c54c5bc00 2019-12-07T19:34:16.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:16.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 119 ==== pg_notify((query:39 sent:39 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 39) v7 ==== 985+0+0 (unknown 0 0 0) 0x558c57edeb40 con 0x558c54c5bc00 2019-12-07T19:34:16.622+0000 7f8f23c56700 7 osd.7 39 handle_fast_pg_notify pg_notify((query:39 sent:39 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 39) v7 from osd.3 2019-12-07T19:34:16.622+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 118 ==== osd_map(39..39 src has 1..39) v4 ==== 1258+0+0 (unknown 0 0 0) 0x558c5742c240 con 0x558c54c5bc00 2019-12-07T19:34:16.622+0000 7f8f1199c700 20 osd.7 39 OSD::ms_dispatch: osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.622+0000 7f8f1199c700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:16.622+0000 7f8f1199c700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:16.622+0000 7f8f1199c700 20 osd.7 39 _dispatch 0x558c5742c240 osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.622+0000 7f8f23c56700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 3(1) notify: (query:39 sent:39 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.622+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 3(1) notify: (query:39 sent:39 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.622+0000 7f8f1199c700 3 osd.7 39 handle_osd_map epochs [39,39], i have 39, src has [1,39] 2019-12-07T19:34:16.622+0000 7f8f1199c700 10 osd.7 39 no new maps here, dropping 2019-12-07T19:34:16.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 3(1) notify: (query:39 sent:39 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 3(1) notify: (query:39 sent:39 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.622+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 3(1) notify: (query:39 sent:39 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.622+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got osd.3(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:16.622+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:08.703352+0000, must = 0 2019-12-07T19:34:16.622+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:16.622+0000 7f8f06986700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:16.622+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:16.622+0000 7f8f06986700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 74 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (crc 0 0 0) 0x558c57450700 con 0x558c54c19800 2019-12-07T19:34:16.623+0000 7f8f24c58700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.623+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 75 ==== pg_notify2(1.4s0 1.4s0 (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e39/39) v1 ==== 1226+0+0 (crc 0 0 0) 0x558c57450700 con 0x558c54c19800 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 2019-12-07T19:34:16.623+0000 7f8f24c58700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.623+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:02.404964+0000, must = 0 2019-12-07T19:34:16.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:16.623+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 76 ==== osd_map(39..39 src has 1..39) v4 ==== 1258+0+0 (crc 0 0 0) 0x558c5730dd40 con 0x558c54c19800 2019-12-07T19:34:16.623+0000 7f8f1199c700 20 osd.7 39 OSD::ms_dispatch: osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.623+0000 7f8f1199c700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:16.623+0000 7f8f1199c700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:16.623+0000 7f8f1199c700 20 osd.7 39 _dispatch 0x558c5730dd40 osd_map(39..39 src has 1..39) v4 2019-12-07T19:34:16.623+0000 7f8f1199c700 3 osd.7 39 handle_osd_map epochs [39,39], i have 39, src has [1,39] 2019-12-07T19:34:16.623+0000 7f8f1199c700 10 osd.7 39 no new maps here, dropping 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:16.623+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:16.624803+0000 9 0.000370 2019-12-07T19:34:16.623+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_acting all_info osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_acting all_info osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_acting all_info osd.3(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_acting all_info osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_acting all_info osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_acting all_info osd.7(0) 1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] calc_acting prefer osd.7(0) because it is current primary 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 3(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 4(5) 2019-12-07T19:34:16.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 77 ==== pg_query2(1.5s4 1.5s4 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efb180 con 0x558c54c19800 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_async_recovery_ec candidates by cost are: 4,3(1) 2019-12-07T19:34:16.623+0000 7f8f24c58700 15 osd.7 39 enqueue_peering_evt 1.5s4 epoch_sent: 39 epoch_requested: 39 MQuery 1.5s4 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.623+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.5s4 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_async_recovery_ec result want=[7,2147483647,1,5,2,4] async_recovery=3(1) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] acting_recovery_backfill is 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] choose_acting want=[7,2147483647,1,5,2,4] backfill_targets= async_recovery_targets=3(1) 2019-12-07T19:34:16.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.5s4 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: leaving GetLog 2019-12-07T19:34:16.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.5s4 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0 2019-12-07T19:34:16.623+0000 7f8f0217d700 10 osd.7 39 handle_pg_query_nopg 1.5s4 2019-12-07T19:34:16.623+0000 7f8f0217d700 10 osd.7 39 pg 1.5s4 dne 2019-12-07T19:34:16.623+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:16.624958+0000 0 0.000000 2019-12-07T19:34:16.623+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify((query:39 sent:39 1.5s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 39) v7 -- 0x558c54df6000 con 0x558c54c19800 2019-12-07T19:34:16.623+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetMissing 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: osd.1(2) has no missing, identical log 2019-12-07T19:34:16.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: osd.2(4) has no missing, identical log 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: osd.4(5) has no missing, identical log 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: osd.5(3) has no missing, identical log 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: still need up_thru update before going active 2019-12-07T19:34:16.623+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetMissing 2019-12-07T19:34:16.625011+0000 0 0.000000 2019-12-07T19:34:16.623+0000 7f8f0297e700 5 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c574fef00 seq 21 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fef00 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fef00 onodes shared_blobs 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fef00 allocated 0x[] released 0x[] 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 prepare 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fef00 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 io_done 2019-12-07T19:34:16.623+0000 7f8f0297e700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39)> waiting <> waiting_peering {} 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:16.623+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MNotifyRec 1.4s0 from 2(4) notify: (query:39 sent:39 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe4800~2e to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe4800 0xe4800~2e to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb4800 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x800 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.623+0000 7f8f06986700 7 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: handle_pg_notify from osd.2(4) 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x82e and padding block with 0x7d2 2019-12-07T19:34:16.623+0000 7f8f06986700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] got dup osd.2(4) info 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39), identical to ours 2019-12-07T19:34:16.623+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:16.623+0000 7f8f06986700 10 osd.7 39 queue_want_up_thru want 39 <= queued 39, currently 38 2019-12-07T19:34:16.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe482e 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000072274s (0.000000059s flush + 0.000072215s kv commit) 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fef00> 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 kv_submitted 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fef00 2019-12-07T19:34:16.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 finishing 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fef00 onodes 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fef00 done 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:16.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fef00 [] 2019-12-07T19:34:16.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:16.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 78 ==== pg_query2(1.ds2 1.ds2 query(info 0'0 epoch_sent 39) e39/39) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57efb180 con 0x558c54c19800 2019-12-07T19:34:16.624+0000 7f8f24c58700 15 osd.7 39 enqueue_peering_evt 1.ds2 epoch_sent: 39 epoch_requested: 39 MQuery 1.ds2 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.624+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.ds2 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) 2019-12-07T19:34:16.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.ds2 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) queued 2019-12-07T19:34:16.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 MQuery 1.ds2 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39)) prio 255 cost 10 e39) pg 0x558c55141000 2019-12-07T19:34:16.624+0000 7f8f06185700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.001166040s) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 MQuery 1.ds2 from 2(0) query_epoch 39 query: query(info 0'0 epoch_sent 39) 2019-12-07T19:34:16.624+0000 7f8f06185700 20 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY pruub 52.054860248s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 2019-12-07T19:34:16.624+0000 7f8f06185700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:16.624+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:16.624+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:39 sent:39 1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([18,37] acting 0(5),1(1),2(0),3(3),4(4),7(2)))) e39/39) v1 -- 0x558c574e6a00 con 0x558c54c19800 2019-12-07T19:34:16.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:16.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5730af00 seq 25 2019-12-07T19:34:16.624+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:16.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:16.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:16.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730af00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:16.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730af00 onodes shared_blobs 2019-12-07T19:34:16.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730af00 allocated 0x[] released 0x[] 2019-12-07T19:34:16.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 prepare 2019-12-07T19:34:16.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730af00 2019-12-07T19:34:16.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 io_done 2019-12-07T19:34:16.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe482e~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe482e 0xe482e~416 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb482e 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x82e 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc44 and padding block with 0x3bc 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe4c44 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000095259s (0.000000098s flush + 0.000095161s kv commit) 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730af00> 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 kv_submitted 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730af00 2019-12-07T19:34:16.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 finishing 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730af00 onodes 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730af00 done 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:16.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730af00 [] 2019-12-07T19:34:16.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:16.638+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 120 ==== MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=99, reqid=client.4540.0:323, at_version=39'32, trim_to=0'0, roll_forward_to=38'30)) v2 ==== 122106+0+0 (unknown 0 0 0) 0x558c579e2800 con 0x558c54c5bc00 2019-12-07T19:34:16.638+0000 7f8f23c56700 15 osd.7 39 enqueue_op 0x558c57f3e580 prio 127 cost 0 latency 0.000039 epoch 39 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=99, reqid=client.4540.0:323, at_version=39'32, trim_to=0'0, roll_forward_to=38'30)) v2 2019-12-07T19:34:16.638+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=99, reqid=client.4540.0:323, at_version=39'32, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e39) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=99, reqid=client.4540.0:323, at_version=39'32, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e39) queued 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=99, reqid=client.4540.0:323, at_version=39'32, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e39) pg 0x558c54d5c000 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57f3e580 prio 127 cost 0 latency 0.000073 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=99, reqid=client.4540.0:323, at_version=39'32, trim_to=0'0, roll_forward_to=38'30)) v2 pg pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=99, reqid=client.4540.0:323, at_version=39'32, trim_to=0'0, roll_forward_to=38'30)) v2 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 snap_mapper.add_oid 1:6c205177:::smithi13140796-8:10 c,e,10 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:10 got.empty() 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6c205177:::smithi13140796-8:10 c,e,10 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 38'30 (0'0,38'30] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] append_log log((0'0,38'30], crt=38'30) [39'31 (29'22) clone 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [10,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,39'32 (29'22) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:323 2019-12-07T19:34:16.636518+0000 0 ObjectCleanRegions clean_offsets: [0~311296,786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'31 (0'0,39'31] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=38'30 crt=38'30 lcod 38'30 active mbc={} ps=[a~1,d~1]] add_log_entry 39'31 (29'22) clone 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [10,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'32 (0'0,39'32] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=38'30 crt=38'30 lcod 38'30 active mbc={} ps=[a~1,d~1]] add_log_entry 39'32 (29'22) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:323 2019-12-07T19:34:16.636518+0000 0 ObjectCleanRegions clean_offsets: [0~311296,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'32 (0'0,39'32] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=38'30 crt=38'30 lcod 38'30 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 32 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'32 (0'0,39'32] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=38'30 crt=38'30 lcod 38'30 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:16.638+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 39'31, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730a000 seq 61 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:10# key 0x8280000000000000016c2051'w!smithi13140796-8!='0x0000000000000010ffffffffffffffff'o' 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:10# 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1174 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.onode(0x558c575ba280).flush flush done 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# 0x0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:10# 0x0~13000 -> 0x0~13000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x0~2000: 0x0~2000 Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10390 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bd260 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd260) get_ref 0x0~2000 Blob(0x558c57efd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd260) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57efd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x2000~2000: 0x0~2000 Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10391 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575bd420 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575bd420 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd1f0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd1f0) get_ref 0x0~2000 Blob(0x558c57efd1f0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57efd1f0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10392 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57314230 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57314230 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c54df8070 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8070) get_ref 0x0~2000 Blob(0x558c54df8070 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8070) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c54df8070 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10393 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57314150 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57314150 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c577f3ab0 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3ab0) get_ref 0x0~2000 Blob(0x558c577f3ab0 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c577f3ab0 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10394 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57314070 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57314070 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd2d0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd2d0) get_ref 0x0~2000 Blob(0x558c57efd2d0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57efd2d0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0xa000~2000: 0x0~2000 Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10395 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c573145b0 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c573145b0 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd340 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd340) get_ref 0x0~2000 Blob(0x558c57efd340 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd340) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57efd340 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10396 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57314700 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57314700 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd180) get_ref 0x0~2000 Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd180) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0xe000~2000: 0x0~2000 Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10397 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c573147e0 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c573147e0 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd420 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd420) get_ref 0x0~2000 Blob(0x558c57efd420 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd420) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57efd420 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x10000~2000: 0x0~2000 Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10398 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c573148c0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c573148c0 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd490 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd490) get_ref 0x0~2000 Blob(0x558c57efd490 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd490) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57efd490 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x12000~1000: 0x0~1000 Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10399 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c573149a0 blob([0x281000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c573149a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=1)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd500 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd500) get_ref 0x0~1000 Blob(0x558c57efd500 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd500) get_ref init 0x1000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x12000~1000: 0x0~1000 Blob(0x558c57efd500 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark inline shard dirty 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c574a1c88) dirty_range mark inline shard dirty 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:10# = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# snapset 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# snapset = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# 1 keys 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# 1 keys = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# hinfo_key (42 bytes) 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# hinfo_key (42 bytes) = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x13000~1d000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6c205177:::smithi13140796-8:head# 0x13000~1d000 - have 0x13000 (77824) bytes fadvise_flags 0x0 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x13000~1d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730a000 15 blobs 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1d000/1000,1d000,0 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5a6000~1d000/1000,1d000,0 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5a6000~1d000] 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efd570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efd570 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd570) get_ref 0x0~2000 Blob(0x558c57efd570 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd570) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x13000~2000: 0x0~2000 Blob(0x558c57efd570 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efd5f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efd650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efd650 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd650) get_ref 0x0~2000 Blob(0x558c57efd650 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd650) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x15000~2000: 0x0~2000 Blob(0x558c57efd650 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efd6d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efd730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efd730 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd730) get_ref 0x0~2000 Blob(0x558c57efd730 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd730) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x17000~2000: 0x0~2000 Blob(0x558c57efd730 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efd7b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efd810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efd810 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd810) get_ref 0x0~2000 Blob(0x558c57efd810 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd810) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x19000~2000: 0x0~2000 Blob(0x558c57efd810 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efd898 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efd8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efd8f0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd8f0) get_ref 0x0~2000 Blob(0x558c57efd8f0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558c57efd8f0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efd978 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efd9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efd9d0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd9d0) get_ref 0x0~2000 Blob(0x558c57efd9d0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558c57efd9d0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efda58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efdab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efdab0 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdab0) get_ref 0x0~2000 Blob(0x558c57efdab0 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdab0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558c57efdab0 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efdb38 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efdb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efdb90 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdb90) get_ref 0x0~2000 Blob(0x558c57efdb90 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdb90) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558c57efdb90 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57efdc18 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efdc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efdc70 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdc70) get_ref 0x0~2000 Blob(0x558c57efdc70 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdc70) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c57efdc70 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57e634a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a4930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a4930 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c580a4930) get_ref 0x0~2000 Blob(0x558c580a4930 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c580a4930) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c580a4930 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a4a28 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e57420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e57420 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c54e57420) get_ref 0x0~2000 Blob(0x558c54e57420 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c54e57420) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c54e57420 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54e566a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a070a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a070a0 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57a070a0) get_ref 0x0~2000 Blob(0x558c57a070a0 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c57a070a0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c57a070a0 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54e56da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d810 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d810) get_ref 0x0~2000 Blob(0x558c5792d810 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d810) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c5792d810 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54e56cc8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809b5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809b5e0 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c5809b5e0) get_ref 0x0~2000 Blob(0x558c5809b5e0 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c5809b5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c5809b5e0 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54e56be8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d8f0 blob([!~1000,0x5c2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d8f0) get_ref 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x5c2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~1000: 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x5c2000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 sbid 0x0)) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54e43cf8 in 0x558c53a58fc0) _discard 0x1000~1000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x30000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark inline shard dirty 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x13000~1d000 = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- SNA_1_000000000000000C_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- SNA_1_000000000000000E_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000039.00000000000000000031' <- 0000000039.00000000000000000031 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000039.00000000000000000032' <- 0000000039.00000000000000000032 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a000 cost 187466 (16 ios * 4000 + 123466 bytes) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a000 onodes 0x558c574a1b80,0x558c575ba280 shared_blobs 0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c575bd3b0 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c574a1c88) update 2#1:6c205177:::smithi13140796-8:10# 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c574a1c88) update inline shard 270 bytes from 10 extents 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:10# is 687 (411 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update inline shard 569 bytes from 25 extents 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 1147 (572 bytes onode + 2 bytes spanning blobs + 573 bytes inline extents) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2))) 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a000 allocated 0x[5a6000~1d000] released 0x[] 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 freelist allocate 0x5a6000~1d000 2019-12-07T19:34:16.638+0000 7f8f0197c700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 prepare 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730a000 2019-12-07T19:34:16.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'32 (0'0,39'32] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=38'30 crt=38'30 lcod 38'30 active mbc={} ps=[a~1,d~1]] op_applied version 39'32 2019-12-07T19:34:16.638+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57f3e580 finish 2019-12-07T19:34:16.639+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 aio_wait 2019-12-07T19:34:16.639+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a000 2019-12-07T19:34:16.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 io_done 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5014 < min_flush_size 524288 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1b80 had 1 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5037 < min_flush_size 524288 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5037 < min_flush_size 524288 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe4c44~13ad to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe4c44 0xe4c44~13ad to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb4c44 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc44 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xff1 and padding block with 0xf 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe5ff1 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000172116s (0.000066301s flush + 0.000105815s kv commit) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a000> 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 kv_submitted 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a000 2019-12-07T19:34:16.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 finishing 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a000 onodes 0x558c574a1b80,0x558c575ba280 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43cf8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef680 space 0x558c54e43cf8 0x1000~1000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e566a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef440 space 0x558c54e566a8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56be8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef5f0 space 0x558c54e56be8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56cc8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef560 space 0x558c54e56cc8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56da8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef4d0 space 0x558c54e56da8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e634a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef320 space 0x558c57e634a8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efd5f8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54dbabd0 space 0x558c57efd5f8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efd6d8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548df9e0 space 0x558c57efd6d8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efd7b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c548df950 space 0x558c57efd7b8 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efd898 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef050 space 0x558c57efd898 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efd978 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef0e0 space 0x558c57efd978 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efda58 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef170 space 0x558c57efda58 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 39/28 ECSubWriteReply(tid=99, last_complete=39'32, committed=1, applied=1)) v2 -- 0x558c5763c000 con 0x558c54c5bc00 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efdb38 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef200 space 0x558c57efdb38 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efdc18 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef290 space 0x558c57efdc18 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a4a28 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ef3b0 space 0x558c580a4a28 0x0~2000 writing nocache) 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 done 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:16.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a000 [] 2019-12-07T19:34:16.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.639+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.641+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 121 ==== MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=100, reqid=client.4540.0:324, at_version=39'33, trim_to=0'0, roll_forward_to=38'30)) v2 ==== 158147+0+0 (unknown 0 0 0) 0x558c579e1000 con 0x558c54c5bc00 2019-12-07T19:34:16.641+0000 7f8f23c56700 15 osd.7 39 enqueue_op 0x558c57f3f340 prio 127 cost 0 latency 0.000023 epoch 39 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=100, reqid=client.4540.0:324, at_version=39'33, trim_to=0'0, roll_forward_to=38'30)) v2 2019-12-07T19:34:16.641+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=100, reqid=client.4540.0:324, at_version=39'33, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e39) 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=100, reqid=client.4540.0:324, at_version=39'33, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e39) queued 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=100, reqid=client.4540.0:324, at_version=39'33, trim_to=0'0, roll_forward_to=38'30)) v2) prio 127 cost 0 e39) pg 0x558c54d5c000 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57f3f340 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=100, reqid=client.4540.0:324, at_version=39'33, trim_to=0'0, roll_forward_to=38'30)) v2 pg pg[1.6s2( v 39'32 (0'0,39'32] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'32 (0'0,39'32] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=100, reqid=client.4540.0:324, at_version=39'33, trim_to=0'0, roll_forward_to=38'30)) v2 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'32 (0'0,39'32] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] append_log log((0'0,39'32], crt=38'30) [39'33 (39'32) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:324 2019-12-07T19:34:16.640468+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1409024~18446744073708142591], clean_omap: 1, new_object: 0] 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'33 (0'0,39'33] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'32 crt=38'30 lcod 39'32 active mbc={} ps=[a~1,d~1]] add_log_entry 39'33 (39'32) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:324 2019-12-07T19:34:16.640468+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1409024~18446744073708142591], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'33 (0'0,39'33] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'32 crt=38'30 lcod 39'32 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 33 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'33 (0'0,39'33] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'32 crt=38'30 lcod 39'32 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:16.641+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 39'33, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea1200 seq 62 2019-12-07T19:34:16.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:16.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:16.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x30000~26000 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6c205177:::smithi13140796-8:head# 0x30000~26000 - have 0x30000 (196608) bytes fadvise_flags 0x0 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:16.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x30000~26000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1200 19 blobs 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x26000/1000,26000,0 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5c3000~26000/1000,26000,0 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5c3000~26000] 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efd3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efd3b0 blob([0x5c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd3b0) get_ref 0x0~2000 Blob(0x558c57efd3b0 blob([0x5c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57efd3b0 blob([0x5c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c577f3c88 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3e30 blob([0x5c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3e30) get_ref 0x0~2000 Blob(0x558c577f3e30 blob([0x5c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3e30) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c577f3e30 blob([0x5c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8018 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d730 blob([0x5c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d730) get_ref 0x0~2000 Blob(0x558c5792d730 blob([0x5c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d730) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c5792d730 blob([0x5c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809b518 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792d650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792d650 blob([0x5c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d650) get_ref 0x0~2000 Blob(0x558c5792d650 blob([0x5c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d650) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c5792d650 blob([0x5c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809b4a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8a80 blob([0x5cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8a80) get_ref 0x0~2000 Blob(0x558c54df8a80 blob([0x5cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8a80) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c54df8a80 blob([0x5cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8b08 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8b60 blob([0x5cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8b60) get_ref 0x0~2000 Blob(0x558c54df8b60 blob([0x5cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8b60) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c54df8b60 blob([0x5cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8be8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8c40 blob([0x5cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8c40) get_ref 0x0~2000 Blob(0x558c54df8c40 blob([0x5cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8c40) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c54df8c40 blob([0x5cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8cc8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8d20 blob([0x5d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8d20) get_ref 0x0~2000 Blob(0x558c54df8d20 blob([0x5d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8d20) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c54df8d20 blob([0x5d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8e00 blob([0x5d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8e00) get_ref 0x0~2000 Blob(0x558c54df8e00 blob([0x5d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8e00) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c54df8e00 blob([0x5d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8e88 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8ee0 blob([0x5d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8ee0) get_ref 0x0~2000 Blob(0x558c54df8ee0 blob([0x5d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c54df8ee0 blob([0x5d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8f68 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8fc0 blob([0x5d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8fc0) get_ref 0x0~2000 Blob(0x558c54df8fc0 blob([0x5d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c54df8fc0 blob([0x5d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df9048 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df90a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df90a0 blob([0x5d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df90a0) get_ref 0x0~2000 Blob(0x558c54df90a0 blob([0x5d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df90a0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c54df90a0 blob([0x5d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df9128 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df9180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df9180 blob([0x5db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9180) get_ref 0x0~2000 Blob(0x558c54df9180 blob([0x5db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9180) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c54df9180 blob([0x5db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df9208 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df9260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df9260 blob([0x5dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9260) get_ref 0x0~2000 Blob(0x558c54df9260 blob([0x5dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9260) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c54df9260 blob([0x5dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c577f3a58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3b20 blob([0x5df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3b20) get_ref 0x0~2000 Blob(0x558c577f3b20 blob([0x5df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3b20) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c577f3b20 blob([0x5df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c573970b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809bea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809bea0 blob([0x5e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bea0) get_ref 0x0~2000 Blob(0x558c5809bea0 blob([0x5e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bea0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c5809bea0 blob([0x5e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809bd68 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809bce0 blob([0x5e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bce0) get_ref 0x0~2000 Blob(0x558c5809bce0 blob([0x5e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bce0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c5809bce0 blob([0x5e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809bdd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809be30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809be30 blob([0x5e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809be30) get_ref 0x0~2000 Blob(0x558c5809be30 blob([0x5e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809be30) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c5809be30 blob([0x5e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809bc88 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809bc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809bc00 blob([0x5e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bc00) get_ref 0x0~2000 Blob(0x558c5809bc00 blob([0x5e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bc00) get_ref init 0x2000, 1000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c5809bc00 blob([0x5e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 sbid 0x0)) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809bf98 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x56000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark inline shard dirty 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x30000~26000 = 0 2019-12-07T19:34:16.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:16.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000039.00000000000000000033' <- 0000000039.00000000000000000033 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 237798 (20 ios * 4000 + 157798 bytes) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes 0x558c575ba280 shared_blobs 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update inline shard 931 bytes from 44 extents 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 1509 (572 bytes onode + 2 bytes spanning blobs + 935 bytes inline extents) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[5c3000~26000] released 0x[] 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 freelist allocate 0x5c3000~26000 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea1200 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'33 (0'0,39'33] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'32 crt=38'30 lcod 39'32 active mbc={} ps=[a~1,d~1]] op_applied version 39'33 2019-12-07T19:34:16.642+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57f3f340 finish 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.642+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 aio_wait 2019-12-07T19:34:16.642+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:16.642+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2734 < min_flush_size 524288 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2757 < min_flush_size 524288 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2757 < min_flush_size 524288 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe5ff1~ac5 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe5ff1 0xe5ff1~ac5 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb5ff1 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xff1 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xab6 and padding block with 0x54a 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe6ab6 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000134890s (0.000037291s flush + 0.000097599s kv commit) 2019-12-07T19:34:16.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200> 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:16.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 0x558c575ba280 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8018 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578eefc0 space 0x558c54df8018 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8b08 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65d40 space 0x558c54df8b08 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8be8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65cb0 space 0x558c54df8be8 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8cc8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65c20 space 0x558c54df8cc8 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8da8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65b90 space 0x558c54df8da8 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8e88 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65b00 space 0x558c54df8e88 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8f68 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65a70 space 0x558c54df8f68 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9048 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e659e0 space 0x558c54df9048 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9128 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65950 space 0x558c54df9128 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9208 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e658c0 space 0x558c54df9208 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573970b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e657a0 space 0x558c573970b8 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3a58 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65830 space 0x558c577f3a58 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 39/28 ECSubWriteReply(tid=100, last_complete=39'33, committed=1, applied=1)) v2 -- 0x558c5763c800 con 0x558c54c5bc00 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3c88 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57a09050 space 0x558c577f3c88 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b4a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65dd0 space 0x558c5809b4a8 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b518 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65e60 space 0x558c5809b518 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809bc88 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e655f0 space 0x558c5809bc88 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809bd68 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65710 space 0x558c5809bd68 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809bdd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65680 space 0x558c5809bdd8 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809bf98 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e65560 space 0x558c5809bf98 0x0~2000 writing nocache) 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:16.642+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:16.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.643+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 122 ==== MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=101, reqid=client.4540.0:325, at_version=39'34, trim_to=0'0, roll_forward_to=39'32)) v2 ==== 39363+0+0 (unknown 0 0 0) 0x558c575be000 con 0x558c54c5bc00 2019-12-07T19:34:16.643+0000 7f8f23c56700 15 osd.7 39 enqueue_op 0x558c57f3f4a0 prio 127 cost 0 latency 0.000022 epoch 39 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=101, reqid=client.4540.0:325, at_version=39'34, trim_to=0'0, roll_forward_to=39'32)) v2 2019-12-07T19:34:16.643+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=101, reqid=client.4540.0:325, at_version=39'34, trim_to=0'0, roll_forward_to=39'32)) v2) prio 127 cost 0 e39) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=101, reqid=client.4540.0:325, at_version=39'34, trim_to=0'0, roll_forward_to=39'32)) v2) prio 127 cost 0 e39) queued 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=101, reqid=client.4540.0:325, at_version=39'34, trim_to=0'0, roll_forward_to=39'32)) v2) prio 127 cost 0 e39) pg 0x558c54d5c000 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57f3f4a0 prio 127 cost 0 latency 0.000499 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=101, reqid=client.4540.0:325, at_version=39'34, trim_to=0'0, roll_forward_to=39'32)) v2 pg pg[1.6s2( v 39'33 (0'0,39'33] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'33 (0'0,39'33] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=101, reqid=client.4540.0:325, at_version=39'34, trim_to=0'0, roll_forward_to=39'32)) v2 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'33 (0'0,39'33] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=38'30 active mbc={} ps=[a~1,d~1]] append_log log((0'0,39'33], crt=38'30) [39'34 (39'33) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:325 2019-12-07T19:34:16.641375+0000 0 ObjectCleanRegions clean_offsets: [0~1409024,1556480~18446744073707995135], clean_omap: 1, new_object: 0] 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'33 crt=38'30 lcod 39'33 active mbc={} ps=[a~1,d~1]] add_log_entry 39'34 (39'33) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:325 2019-12-07T19:34:16.641375+0000 0 ObjectCleanRegions clean_offsets: [0~1409024,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'33 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] rollforward: entry=39'31 (29'22) clone 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [10,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'33 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] rollforward: entry=39'32 (29'22) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:323 2019-12-07T19:34:16.636518+0000 0 ObjectCleanRegions clean_offsets: [0~311296,786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'33 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 34 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'33 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:16.644+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 39'34, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0c00 seq 63 2019-12-07T19:34:16.644+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x56000~9000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6c205177:::smithi13140796-8:head# 0x56000~9000 - have 0x56000 (352256) bytes fadvise_flags 0x0 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x56000~9000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0c00 5 blobs 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x9000/1000,9000,0 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5e9000~9000/1000,9000,0 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5e9000~9000] 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df80e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df80e0 blob([0x5e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df80e0) get_ref 0x0~2000 Blob(0x558c54df80e0 blob([0x5e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df80e0) get_ref init 0x2000, 1000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c54df80e0 blob([0x5e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809b588 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df9a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df9a40 blob([0x5eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9a40) get_ref 0x0~2000 Blob(0x558c54df9a40 blob([0x5eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9a40) get_ref init 0x2000, 1000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c54df9a40 blob([0x5eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df9ac8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df9b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df9b90 blob([0x5ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9b90) get_ref 0x0~2000 Blob(0x558c54df9b90 blob([0x5ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9b90) get_ref init 0x2000, 1000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c54df9b90 blob([0x5ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df9c18 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df9c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df9c70 blob([0x5ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9c70) get_ref 0x0~2000 Blob(0x558c54df9c70 blob([0x5ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9c70) get_ref init 0x2000, 1000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~2000: 0x0~2000 Blob(0x558c54df9c70 blob([0x5ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df9cf8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df9d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df9d50 blob([0x5f1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9d50) get_ref 0x0~1000 Blob(0x558c54df9d50 blob([0x5f1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9d50) get_ref init 0x1000, 1000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5e000~1000: 0x0~1000 Blob(0x558c54df9d50 blob([0x5f1000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 sbid 0x0)) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df9dd8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5f000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark inline shard dirty 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x56000~9000 = 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000039.00000000000000000034' <- 0000000039.00000000000000000034 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 63014 (6 ios * 4000 + 39014 bytes) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes 0x558c575ba280 shared_blobs 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update inline shard 1023 bytes from 49 extents 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 1601 (572 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[5e9000~9000] released 0x[] 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 freelist allocate 0x5e9000~9000 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0c00 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'33 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] op_applied version 39'34 2019-12-07T19:34:16.644+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57f3f4a0 finish 2019-12-07T19:34:16.644+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 aio_wait 2019-12-07T19:34:16.644+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:16.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:16.644+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 123 ==== MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=102, reqid=client.4540.0:326, at_version=39'35, trim_to=0'0, roll_forward_to=39'32)) v2 ==== 2464+0+0 (unknown 0 0 0) 0x558c5748d000 con 0x558c54c5bc00 2019-12-07T19:34:16.644+0000 7f8f23c56700 15 osd.7 39 enqueue_op 0x558c578d3b80 prio 127 cost 0 latency 0.000025 epoch 39 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=102, reqid=client.4540.0:326, at_version=39'35, trim_to=0'0, roll_forward_to=39'32)) v2 2019-12-07T19:34:16.644+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=102, reqid=client.4540.0:326, at_version=39'35, trim_to=0'0, roll_forward_to=39'32)) v2) prio 127 cost 0 e39) 2019-12-07T19:34:16.644+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.644+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=102, reqid=client.4540.0:326, at_version=39'35, trim_to=0'0, roll_forward_to=39'32)) v2) prio 127 cost 0 e39) queued 2019-12-07T19:34:16.644+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.644+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=102, reqid=client.4540.0:326, at_version=39'35, trim_to=0'0, roll_forward_to=39'32)) v2) prio 127 cost 0 e39) pg 0x558c54d5c000 2019-12-07T19:34:16.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2826 < min_flush_size 524288 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 39 dequeue_op 0x558c578d3b80 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=102, reqid=client.4540.0:326, at_version=39'35, trim_to=0'0, roll_forward_to=39'32)) v2 pg pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=102, reqid=client.4540.0:326, at_version=39'35, trim_to=0'0, roll_forward_to=39'32)) v2 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:16.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2849 < min_flush_size 524288 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'34 (0'0,39'34] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] append_log log((0'0,39'34], crt=39'32) [39'35 (39'34) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:326 2019-12-07T19:34:16.641390+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:16.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2849 < min_flush_size 524288 2019-12-07T19:34:16.644+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'34 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] add_log_entry 39'35 (39'34) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:326 2019-12-07T19:34:16.641390+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe6ab6~b21 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'34 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 35 2019-12-07T19:34:16.644+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe6ab6 0xe6ab6~b21 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'34 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:16.644+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb6ab6 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xab6 2019-12-07T19:34:16.644+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 39'35, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.644+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5d7 and padding block with 0xa29 2019-12-07T19:34:16.644+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0300 seq 64 2019-12-07T19:34:16.644+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 3 keys 2019-12-07T19:34:16.644+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 3 keys = 0 2019-12-07T19:34:16.644+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:16.644+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:16.644+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000039.00000000000000000035' <- 0000000039.00000000000000000035 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:16.644+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:16.644+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 6115 (1 ios * 4000 + 2115 bytes) 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes 0x558c575ba280 shared_blobs 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 1600 (571 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:34:16.644+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:34:16.644+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:34:16.644+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'34 crt=39'32 lcod 39'33 active mbc={} ps=[a~1,d~1]] op_applied version 39'35 2019-12-07T19:34:16.644+0000 7f8f05984700 10 osd.7 39 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:16.644+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe75d7 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000515523s (0.000043110s flush + 0.000472413s kv commit) 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2601 < min_flush_size 524288 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2796 < min_flush_size 524288 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 0x558c575ba280 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9ac8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c580cd8c0 space 0x558c54df9ac8 0x0~2000 writing nocache) 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9c18 in 0x558c53a58fc0) _finish_write discard buffer(0x558c580cd950 space 0x558c54df9c18 0x0~2000 writing nocache) 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9cf8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57a08e10 space 0x558c54df9cf8 0x0~2000 writing nocache) 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9dd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57a08cf0 space 0x558c54df9dd8 0x0~1000 writing nocache) 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b588 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54e158c0 space 0x558c5809b588 0x0~2000 writing nocache) 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 kv_submitted 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2819 < min_flush_size 524288 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2819 < min_flush_size 524288 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe75d7~b03 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe75d7 0xe75d7~b03 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb75d7 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5d7 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xda and padding block with 0xf26 2019-12-07T19:34:16.645+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 39/28 ECSubWriteReply(tid=101, last_complete=39'34, committed=1, applied=1)) v2 -- 0x558c5763ca00 con 0x558c54c5bc00 2019-12-07T19:34:16.645+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe80da 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127306s (0.000000093s flush + 0.000127213s kv commit) 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0300> 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:34:16.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 0x558c575ba280 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:34:16.645+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.645+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 39/28 ECSubWriteReply(tid=102, last_complete=39'35, committed=1, applied=1)) v2 -- 0x558c5763cc00 con 0x558c54c5bc00 2019-12-07T19:34:16.645+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.647+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 124 ==== MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=103, reqid=client.4540.0:326, at_version=0'0, trim_to=0'0, roll_forward_to=39'35)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558c57e01800 con 0x558c54c5bc00 2019-12-07T19:34:16.647+0000 7f8f23c56700 15 osd.7 39 enqueue_op 0x558c578d2c60 prio 127 cost 0 latency 0.000024 epoch 39 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=103, reqid=client.4540.0:326, at_version=0'0, trim_to=0'0, roll_forward_to=39'35)) v2 2019-12-07T19:34:16.647+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=103, reqid=client.4540.0:326, at_version=0'0, trim_to=0'0, roll_forward_to=39'35)) v2) prio 127 cost 0 e39) 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=103, reqid=client.4540.0:326, at_version=0'0, trim_to=0'0, roll_forward_to=39'35)) v2) prio 127 cost 0 e39) queued 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=103, reqid=client.4540.0:326, at_version=0'0, trim_to=0'0, roll_forward_to=39'35)) v2) prio 127 cost 0 e39) pg 0x558c54d5c000 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c578d2c60 prio 127 cost 0 latency 0.000224 MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=103, reqid=client.4540.0:326, at_version=0'0, trim_to=0'0, roll_forward_to=39'35)) v2 pg pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'32 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'32 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.6s2 39/28 ECSubWrite(tid=103, reqid=client.4540.0:326, at_version=0'0, trim_to=0'0, roll_forward_to=39'35)) v2 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'32 active mbc={} ps=[a~1,d~1]] append_log log((0'0,39'35], crt=39'32) [] 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] rollforward: entry=39'33 (39'32) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:324 2019-12-07T19:34:16.640468+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1409024~18446744073708142591], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] rollforward: entry=39'34 (39'33) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:325 2019-12-07T19:34:16.641375+0000 0 ObjectCleanRegions clean_offsets: [0~1409024,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] rollforward: entry=39'35 (39'34) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:326 2019-12-07T19:34:16.641390+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 35 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:16.647+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea1500 seq 65 2019-12-07T19:34:16.647+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe80da~471 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe80da 0xe80da~471 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb80da 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xda 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x54b and padding block with 0xab5 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xe854b 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:16.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000106802s (0.000000081s flush + 0.000106721s kv commit) 2019-12-07T19:34:16.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:16.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:16.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:16.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:16.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:16.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.647+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 39/28 ECSubWriteReply(tid=103, last_complete=39'35, committed=1, applied=1)) v2 -- 0x558c5763ce00 con 0x558c54c5bc00 2019-12-07T19:34:16.647+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.647+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 125 ==== MOSDECSubOpRead(1.6s2 39/28 ECSubRead(tid=104, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (unknown 0 0 0) 0x558c5793bb80 con 0x558c54c5bc00 2019-12-07T19:34:16.647+0000 7f8f23c56700 15 osd.7 39 enqueue_op 0x558c57eff1e0 prio 127 cost 0 latency 0.000016 epoch 39 MOSDECSubOpRead(1.6s2 39/28 ECSubRead(tid=104, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:16.647+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 39/28 ECSubRead(tid=104, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e39) 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 39/28 ECSubRead(tid=104, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e39) queued 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 39/28 ECSubRead(tid=104, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e39) pg 0x558c54d5c000 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57eff1e0 prio 127 cost 0 latency 0.000054 MOSDECSubOpRead(1.6s2 39/28 ECSubRead(tid=104, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpRead(1.6s2 39/28 ECSubRead(tid=104, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:16.647+0000 7f8f0197c700 25 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:16.647+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~1000 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x5f000 (389120) 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0~1000 = 4096 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] get_hash_info: Getting attr on 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] get_hash_info: not in cache 1:6c205177:::smithi13140796-8:head 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] get_hash_info: found on disk, size 389120 2019-12-07T19:34:16.647+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key = 0 2019-12-07T19:34:16.647+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 39/28 ECSubReadReply(tid=104, attrs_read=0)) v2 -- 0x558c57efa780 con 0x558c54c5bc00 2019-12-07T19:34:16.647+0000 7f8f0197c700 10 osd.7 39 dequeue_op 0x558c57eff1e0 finish 2019-12-07T19:34:16.647+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:16.652+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258984 2019-12-07T19:34:16.652+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 10 osd.7 39 tick_without_osd_lock 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163fa0e000/0x40000000/0x1680000000, data 0x6230b1/0x5f0000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 25 osd.7 39 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674802+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 25 osd.7 39 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674781+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 25 osd.7 39 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674735+0000 last_rx_front 2019-12-07T19:34:15.674747+0000 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 25 osd.7 39 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674761+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 25 osd.7 39 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674643+0000 last_rx_front 2019-12-07T19:34:15.674689+0000 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 20 osd.7 39 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 20 osd.7 39 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 20 osd.7 39 scrub_load_below_threshold loadavg per cpu 0.04875 < max 5 = yes 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 20 osd.7 39 sched_scrub load_is_low=1 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 10 osd.7 39 sched_scrub 1.4s0 scheduled at 2019-12-07T19:35:02.404964+0000 > 2019-12-07T19:34:16.695707+0000 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 20 osd.7 39 sched_scrub done 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 10 osd.7 39 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 20 osd.7 39 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:16.694+0000 7f8f1fbbe700 10 osd.7 39 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:16.702+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.746+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:16.751+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.802+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.843+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 11 ==== osd_ping(ping e39 up_from 13 ping_stamp 2019-12-07T19:34:16.844267+0000/40.315985843s send_stamp 40.315985843s delta_ub 0.036599101s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54dfac00 con 0x558c552d1c00 2019-12-07T19:34:16.843+0000 7f8f24457700 20 osd.7 39 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036166548s,0.036599101s]) 2019-12-07T19:34:16.843+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e39 up_from 13 ping_stamp 2019-12-07T19:34:16.844267+0000/40.315985843s send_stamp 40.279819295s delta_ub 18446744073.673385068s) v5 -- 0x558c57612400 con 0x558c552d1c00 2019-12-07T19:34:16.843+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 11 ==== osd_ping(ping e39 up_from 13 ping_stamp 2019-12-07T19:34:16.844267+0000/40.315985843s send_stamp 40.315985843s delta_ub 0.036599101s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c552d1800 2019-12-07T19:34:16.843+0000 7f8f24c58700 20 osd.7 39 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.035982138s,0.036599101s]) 2019-12-07T19:34:16.843+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e39 up_from 13 ping_stamp 2019-12-07T19:34:16.844267+0000/40.315985843s send_stamp 40.280003705s delta_ub 18446744073.673569478s) v5 -- 0x558c57612200 con 0x558c552d1800 2019-12-07T19:34:16.852+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.902+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:16.952+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.003+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.053+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.102+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.153+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.203+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.253+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.303+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.353+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.403+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.422+0000 7f8f21451700 10 osd.7 39 tick 2019-12-07T19:34:17.422+0000 7f8f21451700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:17.422+0000 7f8f21451700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:17.422+0000 7f8f21451700 20 osd.7 39 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:17.453+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.503+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:17.503+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.504+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54d42400 0x558c574b9700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:17.504+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54d42400 0x558c574b9700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:17.504+0000 7f8f23c56700 10 osd.7 39 ms_handle_authentication new session 0x558c54c0b1e0 con 0x558c54d42400 entity client.admin addr 2019-12-07T19:34:17.505+0000 7f8f23c56700 10 osd.7 39 ms_handle_authentication session 0x558c54c0b1e0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:17.505+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2685087073 conn(0x558c54d42400 0x558c574b9700 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).ready entity=client.4469 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:17.505+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4469 172.21.15.131:0/2685087073 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c57edfe00 con 0x558c54d42400 2019-12-07T19:34:17.505+0000 7f8f164bb700 20 osd.7 39 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:17.505+0000 7f8f164bb700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:17.505+0000 7f8f164bb700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:17.505+0000 7f8f164bb700 20 osd.7 39 _dispatch 0x558c57edfe00 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:17.505+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2685087073 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce0540 con 0x558c54d42400 2019-12-07T19:34:17.528+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4469 172.21.15.131:0/2685087073 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c57edfc20 con 0x558c54d42400 2019-12-07T19:34:17.528+0000 7f8f164bb700 20 osd.7 39 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:17.528+0000 7f8f164bb700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:17.528+0000 7f8f164bb700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:17.528+0000 7f8f164bb700 20 osd.7 39 _dispatch 0x558c57edfc20 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:17.529+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2685087073 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce1180 con 0x558c54d42400 2019-12-07T19:34:17.532+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2685087073 conn(0x558c54d42400 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 62 2019-12-07T19:34:17.532+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2685087073 conn(0x558c54d42400 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:17.532+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2685087073 conn(0x558c54d42400 0x558c574b9700 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:17.532+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2685087073 conn(0x558c54d42400 0x558c574b9700 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:17.532+0000 7f8f164bb700 2 osd.7 39 ms_handle_reset con 0x558c54d42400 session 0x558c54c0b1e0 2019-12-07T19:34:17.554+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.604+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 610376 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 134 ==== pg_notify2(1.4s0 1.4s0 (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e40/40) v1 ==== 1226+0+0 (unknown 0 0 0) 0x558c54e05500 con 0x558c54c5b400 2019-12-07T19:34:17.616+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)> waiting <> waiting_peering {} 2019-12-07T19:34:17.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.616+0000 7f8f0297e700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 40, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.616+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 24 ==== osd_map(40..40 src has 1..40) v4 ==== 673+0+0 (crc 0 0 0) 0x558c579e4900 con 0x558c5476d000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 osd.7 39 OSD::ms_dispatch: osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 osd.7 39 _dispatch 0x558c579e4900 osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 135 ==== PGlog(1.2s0 log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) e40/39) v6 ==== 6203+0+0 (unknown 0 0 0) 0x558c57a45000 con 0x558c54c5b400 2019-12-07T19:34:17.616+0000 7f8f164bb700 3 osd.7 39 handle_osd_map epochs [40,40], i have 39, src has [1,40] 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 osd.7 39 handle_osd_map got inc map for epoch 40 2019-12-07T19:34:17.616+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.2s0 epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info 2019-12-07T19:34:17.616+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.616+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3248e826:::osdmap.39:0# 0x0~0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1681 size 0x1681 (5761) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3110 blob([0x5a4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314620 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:17.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efc310 blob([!~1000,0x5a5000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,681]) SharedBlob(0x558c57efc380 sbid 0x0)) need 0x1000~681 cache has 0x[1000~681] 2019-12-07T19:34:17.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.616+0000 7f8f07988700 20 osd.7 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 40, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3248e826:::osdmap.39:0# 0x0~1681 = 5761 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 osd.7 39 add_map_bl 39 5761 bytes 2019-12-07T19:34:17.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.616+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 136 ==== osd_map(40..40 src has 1..40) v4 ==== 673+0+0 (unknown 0 0 0) 0x558c573526c0 con 0x558c54c5b400 2019-12-07T19:34:17.616+0000 7f8f1199c700 20 osd.7 39 OSD::ms_dispatch: osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 osd.7 39 got_full_map 40, nothing requested 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 osd.7 39 handle_osd_map pg_num_history pg_num_history(e40 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {40={}} 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 osd.7 39 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e40 [1,40] lci=[13,40]) 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54d58300 seq 31 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cbd7972f:::inc_osdmap.40:0# key 0x7f7fffffffffffffffcbd797'/!inc_osdmap.40!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:17.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 137 ==== PGlog(1.bs1 log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) e40/39) v6 ==== 7462+0+0 (unknown 0 0 0) 0x558c54d86000 con 0x558c54c5b400 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cbd7972f:::inc_osdmap.40:0# 0x0~275 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1175 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cbd7972f:::inc_osdmap.40:0# 0x0~275 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~275 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd8b on front/back, now 0x0~1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58300 1 blobs 2019-12-07T19:34:17.616+0000 7f8f24457700 15 osd.7 39 enqueue_peering_evt 1.bs1 epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:17.616+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5f2000~1000/1000,1000,0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5f2000~1000] 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fdb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bdb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fdb20 blob([0x5f2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bdb20 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c574fdb20) get_ref 0x0~275 Blob(0x558c574fdb20 blob([0x5f2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bdb20 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c574fdb20) get_ref init 0x1000, 1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~275: 0x0~275 Blob(0x558c574fdb20 blob([0x5f2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x275) SharedBlob(0x558c575bdb20 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c575bdb38 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:17.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x275 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efa388) dirty_range mark inline shard dirty 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cbd7972f:::inc_osdmap.40:0# 0x0~275 = 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3508e826:::osdmap.40:0# key 0x7f7fffffffffffffff3508e8'&!osdmap.40!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3508e826:::osdmap.40:0# 0x0~1681 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1176 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3508e826:::osdmap.40:0# 0x0~1681 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~681 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0x97f on front/back, now 0x0~1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58300 2 blobs 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5f3000~2000/1000,2000,0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5f3000~2000] 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5523a930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f3dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5523a930 blob([0x5f3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3dc0 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a930) get_ref 0x0~1000 Blob(0x558c5523a930 blob([0x5f3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3dc0 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c5523a930) get_ref init 0x1000, 1000 2019-12-07T19:34:17.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 map 40 > 39 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5523a930 blob([0x5f3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c577f3dc0 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c577f3dd8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:17.616+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 40, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:17.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efc4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efc540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efc4d0 blob([!~1000,0x5f4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc540 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc4d0) get_ref 0x1000~681 Blob(0x558c57efc4d0 blob([!~1000,0x5f4000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57efc540 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~681: 0x1000~681 Blob(0x558c57efc4d0 blob([!~1000,0x5f4000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,681]) SharedBlob(0x558c57efc540 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efc558 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1681 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efad88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3508e826:::osdmap.40:0# 0x0~1681 = 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58300 0 blobs 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58300 0 blobs 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 23871 (4 ios * 4000 + 7871 bytes) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes 0x558c53a91900,0x558c55276c80,0x558c57efa280,0x558c57efac80 shared_blobs 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efa388) update #-1:cbd7972f:::inc_osdmap.40:0# 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efa388) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cbd7972f:::inc_osdmap.40:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efad88) update #-1:3508e826:::osdmap.40:0# 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efad88) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3508e826:::osdmap.40:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[5f2000~3000] released 0x[] 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 freelist allocate 0x5f2000~3000 2019-12-07T19:34:17.616+0000 7f8f164bb700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:34:17.616+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58300 2019-12-07T19:34:17.616+0000 7f8f1199c700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:17.616+0000 7f8f1199c700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:17.616+0000 7f8f1199c700 20 osd.7 39 _dispatch 0x558c573526c0 osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.616+0000 7f8f1199c700 3 osd.7 39 handle_osd_map epochs [40,40], i have 40, src has [1,40] 2019-12-07T19:34:17.616+0000 7f8f1199c700 10 osd.7 39 no new maps here, dropping 2019-12-07T19:34:17.616+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 aio_wait 2019-12-07T19:34:17.616+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:34:17.616+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:34:17.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:17.616+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 79 ==== pg_notify2(1.4s0 1.4s0 (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e40/40) v1 ==== 1226+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c19800 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efa280 had 1 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efac80 had 1 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:17.617+0000 7f8f24c58700 15 osd.7 39 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xe854b~226c to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xe854b 0xe854b~226c to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb854b 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x54b 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7b7 and padding block with 0x849 2019-12-07T19:34:17.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {40=0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)>} 2019-12-07T19:34:17.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)> waiting <> waiting_peering {40=0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)>} 2019-12-07T19:34:17.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.617+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 40, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.617+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 80 ==== osd_map(40..40 src has 1..40) v4 ==== 673+0+0 (crc 0 0 0) 0x558c57945200 con 0x558c54c19800 2019-12-07T19:34:17.617+0000 7f8f1199c700 20 osd.7 39 OSD::ms_dispatch: osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.617+0000 7f8f1199c700 10 osd.7 39 do_waiters -- start 2019-12-07T19:34:17.617+0000 7f8f1199c700 10 osd.7 39 do_waiters -- finish 2019-12-07T19:34:17.617+0000 7f8f1199c700 20 osd.7 39 _dispatch 0x558c57945200 osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.617+0000 7f8f1199c700 3 osd.7 39 handle_osd_map epochs [40,40], i have 40, src has [1,40] 2019-12-07T19:34:17.617+0000 7f8f1199c700 10 osd.7 39 no new maps here, dropping 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xea7b7 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000173469s (0.000048431s flush + 0.000125038s kv commit) 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300> 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54d58300 osr 0x558c54ea4b40 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) _discard drop 28 0x1b000~1000 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7 39 _committed_osd_maps 40..40 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7 39 advance to epoch 40 (<= last 40 <= newest_map 40) 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) prepare_write seq 29 0x1b000~1000 crc 5b7e61e5 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) _discard drop 28 0x2000~1000 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) prepare_write seq 29 0x2000~1000 crc 94df0214 2019-12-07T19:34:17.617+0000 7f8f141a1700 7 osd.7 40 consume_map version 40 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:0.identify_splits_and_merges identify_splits_and_merges slot 1.8s5 has no pg and waiting_for_split 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.0s3 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:1.identify_splits_and_merges identify_splits_and_merges slot 1.1s1 has no pg and waiting_for_split 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.9s4 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:2.identify_splits_and_merges identify_splits_and_merges slot 1.as5 has no pg and waiting_for_split 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.2s0 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.2s4 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.bs3 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs1 has no pg and waiting_for_split 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.3s4 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.cs0 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.4s0 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:5.identify_splits_and_merges identify_splits_and_merges slot 1.5s4 has no pg and waiting_for_split 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.ds2 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:6.identify_splits_and_merges identify_splits_and_merges slot 1.es4 has no pg and waiting_for_split 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.6s2 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:7.identify_splits_and_merges identify_splits_and_merges slot 1.fs5 has no pg and waiting_for_split 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 identify_splits_and_merges 1.7s4 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7 40 prune_pg_created 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:0.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 empty, pruning 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:1.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 empty, pruning 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:2.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 empty, pruning 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 pending_peering first epoch 40 <= 40, requeueing 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {40=} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:3.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 pending_peering first epoch 40 <= 40, requeueing 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {40=} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:4.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 pending_peering first epoch 40 <= 40, requeueing 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.4s0 to_process <> waiting <> waiting_peering {40=0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40),OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)>} 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:5.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 empty, pruning 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:6.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 empty, pruning 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.617+0000 7f8f141a1700 10 osd.7:7.consume_map 40 (was 39) 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 empty, pruning 2019-12-07T19:34:17.617+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 no pg, should create on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 39 -> 40 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 39 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] handle_advance_map [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3] -- 7/4 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] state: Started advmap 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 40 _make_pg 1.bs1 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)> waiting <> waiting_peering {} 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] hit_set_clear 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.b(unlocked)] enter Initial 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] agent_stop 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] handle_advance_map: 40 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 39 -> 40 2019-12-07T19:34:17.617+0000 7f8f07187700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] handle_advance_map [7,3,1,5,2,4]/[7,2147483647,1,5,2,4] -- 7/7 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.617+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 4(1) log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 57.066233270s int 16s) +create_info 2019-12-07T19:34:17.617+0000 7f8f07187700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Peering advmap 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 remapped NOTIFY mbc={}] state: got info+log from osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,29'15], crt=29'15) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] adjust_need_up_thru now 39, need_up_thru now false 2019-12-07T19:34:17.617+0000 7f8f03980700 10 merge_log log((0'0,29'15], crt=29'15) from osd.4(1) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.617+0000 7f8f03980700 10 merge_log extending head to 29'15 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.617+0000 7f8f07187700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.b(unlocked)] enter NotTrimming 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.bs1_head fresh osr 0x558c57c6f540 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [4,7,2,3,1,5] acting [4,2147483647,2,2147483647,1,5] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 past_intervals ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] handle_activate_map: 40 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:17.617+0000 7f8f03980700 20 ? 21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.617+0000 7f8f0297e700 7 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 41.053996292s + 0s) 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] publish_stats_to_osd 39:60 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:17.617+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:17.618569+0000 3 0.000346 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] state: Leaving Peering 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped+peering mbc={} ps=[a~1]] exit Started/Primary/Peering 2019-12-07T19:34:17.618588+0000 0 0.000000 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] enter Started/Primary/Active 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:17.618606+0000 0 0.000000 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] state: In Active, about to call activate 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate - no missing, moving last_complete 37'4 -> 37'4 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate - purged_snaps [a~1] removed_snaps [a~1,d~1] 2019-12-07T19:34:17.617+0000 7f8f03980700 20 update missing, append 29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:17.617+0000 7f8f03980700 10 merge_log result log((0'0,29'15], crt=29'15) missing(5 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 39 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:17.617+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 0'0 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY m=5 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.618647+0000 5 0.000090 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:17.618656+0000 1 0.000039 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:17.617+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 0'0 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY m=5 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 57.054076579s) 2019-12-07T19:34:17.617+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 0'0 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY m=5 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:17.617+0000 7f8f07187700 1 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 0'0 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY m=5 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:17.618684+0000 0 0.000000 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.1(2) is up to date, queueing in pending_activators 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 0'0 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped m=5 mbc={} ps=[a~1]] activate - not complete, missing(5 may_include_deletes = 1) 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.995763418s) uptodate 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 39 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 lcod 0'0 remapped m=5 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.617+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57ef8900 seq 1 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) 2019-12-07T19:34:17.617+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.bs1_head bits 4 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.2(4) is up to date, queueing in pending_activators 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.bs1_head bits 4 = 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.993318125s) uptodate 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.3(1) 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:d0000000::::head# key 0x818000000000000001d000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 copy_after v 0'0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 copy_after copy log version 37'4 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 copy_after copy log version 37'3 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 copy_after copy log version 21'2 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 copy_after copy log version 21'1 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 copy_up_to/copy_after earliest_dup_version 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1177 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.3(1) sending log((0'0,37'4], crt=37'4) 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._infover' <- _infover 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- PGlog(1.4s1 log log((0'0,37'4], crt=37'4) pi ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0))) pg_lease(ru 0s ub 57.054076579s int 16s) e40/39) v6 -- 0x558c57e7d800 con 0x558c54c5bc00 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8900 cost 5945 (1 ios * 4000 + 1945 bytes) 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8900 onodes 0x558c57572500 shared_blobs 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore.extentmap(0x558c57572608) update 1#1:d0000000::::head# 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore.extentmap(0x558c57572608) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.3(1) 1.4s1( DNE v 37'4 lc 0'0 (0'0,37'4] local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) missing missing(2 may_include_deletes = 1) 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8900 allocated 0x[] released 0x[] 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 prepare 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8900 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 io_done 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.4(5) is up to date, queueing in pending_activators 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 40 handle_pg_create_info new pg pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7:3._attach_pg 1.bs1 0x558c575ef000 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1539 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=12.000740286s) uptodate 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 40 identify_splits_and_merges 1.bs1 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7:3._prime_splits 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.5(3) is up to date, queueing in pending_activators 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 40 send_incremental_map 39 -> 40 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate peer osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39 pruub=11.997137915s) uptodate 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3859873792 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate setting up missing_loc from shard 1(2) 2019-12-07T19:34:17.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cbd7972f:::inc_osdmap.40:0# 0x0~0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~275 size 0x275 (629) 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate setting up missing_loc from shard 2(4) 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 39 -> 40 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998542) bluestore 86 GiB free (0.959935), bluefs_ratio 0.039947 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate setting up missing_loc from shard 3(1) 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fdb20 blob([0x5f2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x275) SharedBlob(0x558c575bdb20 sbid 0x0)) need 0x0~275 cache has 0x[0~275] 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 39 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5] -- 4/4 2019-12-07T19:34:17.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cbd7972f:::inc_osdmap.40:0# 0x0~275 = 629 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 40 add_map_inc_bl 40 629 bytes 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1562 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate setting up missing_loc from shard 4(5) 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1562 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate setting up missing_loc from shard 5(3) 2019-12-07T19:34:17.617+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(40..40 src has 1..40) v4 -- 0x558c57352480 con 0x558c54c5b400 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] activate setting up missing_loc from shard 7(0) 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xea7b7~61a to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xea7b7 0xea7b7~61a to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] needs_recovery osd.3(1) has 2 missing 2019-12-07T19:34:17.617+0000 7f8f03980700 10 osd.7 40 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 39 -> 40 (shared) 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xba7b7 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7b7 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdd1 and padding block with 0x22f 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={} ps=[a~1]] add_batch_sources_info: adding sources in batch 5 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.617+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.2s1 1.2s1 (query:40 sent:40 1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 0->1 ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e40/40) v1 -- 0x558c54e98000 con 0x558c54c5b400 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57ef8300 seq 3 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] build_might_have_unfound 2019-12-07T19:34:17.617+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.617+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore.onode(0x558c57e8e280).flush flush done 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) pg_lease(ru 0s ub 57.066590630s int 16s) +create_info 2019-12-07T19:34:17.617+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] build_might_have_unfound: built 0(1),1(2),2(4),3(1),4(5),5(3) 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,21'12], crt=21'12) 2019-12-07T19:34:17.617+0000 7f8f07187700 10 merge_log log((0'0,21'12], crt=21'12) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.617+0000 7f8f07187700 10 merge_log extending head to 21'12 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] discover_all_missing 0 missing, 0 unfound 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] discover_all_missing skipping down osd.0(1) 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] discover_all_missing: osd.1(2): we already have pg_missing_t 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] discover_all_missing: osd.2(4): we already have pg_missing_t 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] discover_all_missing: osd.4(5): we already have pg_missing_t 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} ps=[a~1]] discover_all_missing: osd.5(3): we already have pg_missing_t 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 ? 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x0000000000000493'.0000000000.00000000000000000000' end: 0x0000000000000493'.4294967295.18446744073709551615' 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000023.00000000000000000010' <- 0000000023.00000000000000000010 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000027.00000000000000000011' <- 0000000027.00000000000000000011 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000027.00000000000000000012' <- 0000000027.00000000000000000012 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000029.00000000000000000013' <- 0000000029.00000000000000000013 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000029.00000000000000000014' <- 0000000029.00000000000000000014 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000029.00000000000000000015' <- 0000000029.00000000000000000015 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.missing/0000000000000001.23972223.head.smithi13140796-48..' <- missing/0000000000000001.23972223.head.smithi13140796-48.. 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.missing/0000000000000001.28BF48E5.4.smithi13140796-39..' <- missing/0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.missing/0000000000000001.28BF48E5.head.smithi13140796-39..' <- missing/0000000000000001.28BF48E5.head.smithi13140796-39.. 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.missing/0000000000000001.28F71588.head.smithi13140796-45..' <- missing/0000000000000001.28F71588.head.smithi13140796-45.. 2019-12-07T19:34:17.617+0000 7f8f07187700 20 update missing, append 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.missing/0000000000000001.2916DE48.head.smithi13140796-43..' <- missing/0000000000000001.2916DE48.head.smithi13140796-43.. 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.617+0000 7f8f07187700 10 merge_log result log((0'0,21'12], crt=21'12) missing(4 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8300 cost 12229 (1 ios * 4000 + 8229 bytes) 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xeadd1 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8300 onodes shared_blobs 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 unknown m=4 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.619048+0000 3 0.000055 2019-12-07T19:34:17.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 prepare 2019-12-07T19:34:17.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8300 2019-12-07T19:34:17.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 io_done 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000246091s (0.000000102s flush + 0.000245989s kv commit) 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 unknown m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.617+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 unknown m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.617+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 39:61 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8078 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 unknown m=4 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 unknown m=4 mbc={} ps=[a~1]] activate - not complete, missing(4 may_include_deletes = 1) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.617+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8900> 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 kv_submitted 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8900 2019-12-07T19:34:17.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 finishing 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8900 onodes 0x558c57572500 2019-12-07T19:34:17.617+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] enter Started/Primary/Active/Activating 2019-12-07T19:34:17.617+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8900 done 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:17.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8900 [] 2019-12-07T19:34:17.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8101 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8101 < min_flush_size 524288 2019-12-07T19:34:17.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xeadd1~1fa5 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xeadd1 0xeadd1~1fa5 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xbadd1 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdd1 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 4(5) notify: (query:40 sent:40 1.4s5( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 5->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd76 and padding block with 0x28a 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: Active: got notify from 4(5), already have info from that osd, ignoring 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 40 send_incremental_map 39 -> 40 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(40..40 src has 1..40) v4 -- 0x558c5742c240 con 0x558c54c64000 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 40 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 39 -> 40 (shared) 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) e40/40) v1 -- 0x558c576b8e00 con 0x558c54c64000 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 40 send_incremental_map 39 -> 40 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(40..40 src has 1..40) v4 -- 0x558c57325b00 con 0x558c54c19800 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 40 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 39 -> 40 (shared) 2019-12-07T19:34:17.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) e40/40) v1 -- 0x558c57379500 con 0x558c54c19800 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c548f6300 seq 2 2019-12-07T19:34:17.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.617+0000 7f8f07187700 20 bluestore.onode(0x558c57572500).flush flush done 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) e40/40) v1 -- 0x558c5737a300 con 0x558c54c5b400 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 40 send_incremental_map 39 -> 40 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(40..40 src has 1..40) v4 -- 0x558c576b3200 con 0x558c54c19c00 2019-12-07T19:34:17.617+0000 7f8f0297e700 10 osd.7 40 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 39 -> 40 (shared) 2019-12-07T19:34:17.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) e40/40) v1 -- 0x558c57379c00 con 0x558c54c19c00 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x0000000000000499'.0000000000.00000000000000000000' end: 0x0000000000000499'.4294967295.18446744073709551615' 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54ddf200 seq 22 2019-12-07T19:34:17.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:17.618+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xecd76 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000165761s (0.000000091s flush + 0.000165670s kv commit) 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8300> 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 kv_submitted 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8300 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.missing/0000000000000001.B0E469FE.head.smithi13140796-28..' <- missing/0000000000000001.B0E469FE.head.smithi13140796-28.. 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 finishing 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8300 onodes 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.missing/0000000000000001.B376C33C.head.smithi13140796-50..' <- missing/0000000000000001.B376C33C.head.smithi13140796-50.. 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8300 done 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8300 [] 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.missing/0000000000000001.BC8CBBB8.head.smithi13140796-49..' <- missing/0000000000000001.BC8CBBB8.head.smithi13140796-49.. 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.missing/0000000000000001.BD1A381A.head.smithi13140796-18..' <- missing/0000000000000001.BD1A381A.head.smithi13140796-18.. 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 13019 (1 ios * 4000 + 9019 bytes) 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes shared_blobs 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:17.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:17.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8841 < min_flush_size 524288 2019-12-07T19:34:17.618+0000 7f8f141a1700 10 osd.7 40 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.0s3 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f07988700 15 osd.7 40 enqueue_peering_evt 1.2s0 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8864 < min_flush_size 524288 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8864 < min_flush_size 524288 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.9s4 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xecd76~22a0 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.2s0 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 81 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) e40/40) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c19800 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xecd76 0xecd76~22a0 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.2s4 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xbcd76 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd76 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.bs3 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x16 and padding block with 0xfea 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.ds2 epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) pg_lease(ru 0s ub 57.090455690s int 16s) 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 lcod 0'0 remapped m=5 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.618+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) pg_lease(ru 0s ub 57.090455690s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.bs1 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c5500d000 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 lcod 0'0 remapped m=5 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c54d9b000 2019-12-07T19:34:17.618+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c57ed2700 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) pg_lease(ru 0s ub 57.090455690s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) pg_lease(ru 0s ub 57.090455690s int 16s)) prio 255 cost 10 e40) pg 0x558c55141000 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 39 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 138 ==== pg_query2(1.2s4 1.2s4 query(fulllog 0'0 epoch_sent 40) e40/40) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57efaf00 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 39 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5] -- 4/4 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.2s4 epoch_sent: 40 epoch_requested: 40 MQuery 1.2s4 from 4(1) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40) 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MQuery 1.2s4 from 4(1) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40)) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c5731a000 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,2147483647] -- 2/2 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:17.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 139 ==== pg_query2(1.bs3 1.bs3 query(fulllog 0'0 epoch_sent 40) e40/40) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57efaf00 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_activate_map: 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:17.618+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.bs3 epoch_sent: 40 epoch_requested: 40 MQuery 1.bs3 from 4(0) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:17.618+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MQuery 1.bs3 from 4(0) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40)) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] activate_map 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s4 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 40 2019-12-07T19:34:17.618+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3] -- 7/4 2019-12-07T19:34:17.618+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:17.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] handle_activate_map: 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] handle_activate_map: 40 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c5500a000 2019-12-07T19:34:17.618+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) pg_lease(ru 0s ub 57.090455690s int 16s) 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] state: got info from osd.2(0) 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 57.090455690s int 16s) 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 39 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2147483647,7,3] -- 1/1 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] handle_activate_map: 40 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.035982138s -> ruub 57.054473552s 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:17.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MQuery 1.bs3 from 4(0) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 57.054473552s 2019-12-07T19:34:17.618+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:40 sent:40 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 3->0 ([28,38] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e40/40) v1 -- 0x558c54e99c00 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.619526+0000 5 0.000102 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] handle_activate_map: 40 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MQuery 1.bs3 from 4(0) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40)) prio 255 cost 10 e40) pg 0x558c54d9b000 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped mbc={} ps=[a~1]] activate - no missing, moving last_complete 21'6 -> 21'6 2019-12-07T19:34:17.618+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.2s1 1.2s1 (query:40 sent:40 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->1 ([28,38] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e40/40) v1 -- 0x558c57ed2e00 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MQuery 1.bs3 from 4(0) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.618+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] log request from 4(0) 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xef016 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MQuery 1.2s4 from 4(1) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] sending info+missing+full log 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000004 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MQuery 1.2s4 from 4(1) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40)) prio 255 cost 10 e40) pg 0x558c5731a000 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.618+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MQuery 1.2s4 from 4(1) query_epoch 40 query: query(fulllog 0'0 epoch_sent 40) 2019-12-07T19:34:17.618+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000309225s (0.000000081s flush + 0.000309144s kv commit) 2019-12-07T19:34:17.618+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.618+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:40 sent:40 1.ds2( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([18,37] acting 0(5),1(1),2(0),3(3),4(4),7(2)))) e40/40) v1 -- 0x558c57450e00 con 0x558c54c19800 2019-12-07T19:34:17.618+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] sending log((0'0,21'12], crt=21'12) missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] log request from 4(1) 2019-12-07T19:34:17.618+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- PGlog(1.bs0 log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) e40/40) v6 -- 0x558c54d86000 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:17.618+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:40 sent:40 1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->0 ([28,38] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([28,37] acting 0(3),1(0),3(5),4(1),5(2),7(4)))) e40/40) v1 -- 0x558c54e07100 con 0x558c54c64000 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300> 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57ef8600 seq 26 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] sending info+missing+full log 2019-12-07T19:34:17.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.618+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:17.618+0000 7f8f07187700 15 osd.7 40 enqueue_peering_evt 1.bs1 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:17.618+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] sending log((0'0,29'15], crt=29'15) missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8600 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8600 onodes shared_blobs 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8600 allocated 0x[] released 0x[] 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- PGlog(1.2s1 log log((0'0,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) e40/40) v6 -- 0x558c5748e000 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 prepare 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c575ef000 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8600 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 io_done 2019-12-07T19:34:17.618+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c548a1500 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:17.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 82 ==== PGlog(1.5s4 log log((0'0,27'8], crt=27'8) pi ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) pg_lease(ru 0s ub 57.091205159s int 16s) e40/39) v6 ==== 4180+0+0 (crc 0 0 0) 0x558c54d87000 con 0x558c54c19800 2019-12-07T19:34:17.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:17.618+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.5s4 epoch_sent: 40 epoch_requested: 39 MLogRec from 2(0) log log((0'0,27'8], crt=27'8) pi ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) pg_lease(ru 0s ub 57.091205159s int 16s) +create_info 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:17.618+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 2(0) log log((0'0,27'8], crt=27'8) pi ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) pg_lease(ru 0s ub 57.091205159s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xef016~517 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xef016 0xef016~517 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 2(0) log log((0'0,27'8], crt=27'8) pi ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) pg_lease(ru 0s ub 57.091205159s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xbf016 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x16 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 2(0) log log((0'0,27'8], crt=27'8) pi ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) pg_lease(ru 0s ub 57.091205159s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x52d and padding block with 0xad3 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 no pg, should create on OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 2(0) log log((0'0,27'8], crt=27'8) pi ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) pg_lease(ru 0s ub 57.091205159s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 40 _make_pg 1.5s4 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.3s4 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.cs0 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5(unlocked)] enter Initial 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.ds2 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.5(unlocked)] enter NotTrimming 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.5s4_head fresh osr 0x558c57c6e280 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [2,1,3,4,7,5] acting [2,1,3,4,2147483647,5] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 past_intervals ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddf200 cost 5574 (1 ios * 4000 + 1574 bytes) 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddf200 onodes shared_blobs 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 41.055319303s + 0s) 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddf200 allocated 0x[] released 0x[] 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 prepare 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddf200 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 io_done 2019-12-07T19:34:17.618+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c57676000 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 2(4) notify: (query:40 sent:40 1.4s4( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 4->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c575ef000 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:17.619936+0000 0 0.000000 2019-12-07T19:34:17.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 126 ==== PGlog(1.1s1 log log((0'0,29'9], crt=29'9) pi ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) pg_lease(ru 0s ub 57.076160187s int 16s) e40/39) v6 ==== 6736+0+0 (unknown 0 0 0) 0x558c57e01000 con 0x558c54c5bc00 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: Active: got notify from 2(4), already have info from that osd, ignoring 2019-12-07T19:34:17.618+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c5513e000 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:17.618+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.1s1 epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'9], crt=29'9) pi ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) pg_lease(ru 0s ub 57.076160187s int 16s) +create_info 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 39 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.618+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'9], crt=29'9) pi ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) pg_lease(ru 0s ub 57.076160187s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: Active advmap 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 39 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 39 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,2147483647] -- 4/4 2019-12-07T19:34:17.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:17.619983+0000 1 0.000036 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'9], crt=29'9) pi ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) pg_lease(ru 0s ub 57.076160187s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'9], crt=29'9) pi ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) pg_lease(ru 0s ub 57.076160187s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 no pg, should create on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'9], crt=29'9) pi ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) pg_lease(ru 0s ub 57.076160187s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] handle_activate_map: 40 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:17.618+0000 7f8f0217d700 1 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:17.620013+0000 0 0.000000 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_activate_map: 40 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 5 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 40 _make_pg 1.1s1 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] activate_map 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 39 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: Active: handling ActMap 2019-12-07T19:34:17.618+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:40 sent:40 1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->0 ([28,38] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([28,37] acting 0(5),1(1),2(2),4(0),5(3),7(4)))) e40/40) v1 -- 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c577fc000 seq 1 2019-12-07T19:34:17.618+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1(unlocked)] enter Initial 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.5s4_head bits 4 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] Active: kicking snap trim 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.5s4_head bits 4 = 0 2019-12-07T19:34:17.618+0000 7f8f0297e700 7 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: handle ActMap primary 2019-12-07T19:34:17.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xef52d 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) get_onode oid 4#1:a0000000::::head# key 0x848000000000000001a000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.618+0000 7f8f04181700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:17.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] publish_stats_to_osd 38:121 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) r -2 v.len 0 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000367319s (0.000000100s flush + 0.000367219s kv commit) 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8600> 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 40 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 kv_submitted 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8600 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1178 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1418 < min_flush_size 524288 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f04181700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 finishing 2019-12-07T19:34:17.618+0000 7f8f04181700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8600 onodes 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8600 done 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._infover' <- _infover 2019-12-07T19:34:17.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8600 [] 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.618+0000 7f8f04181700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:17.618+0000 7f8f04181700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc000 cost 5934 (1 ios * 4000 + 1934 bytes) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc000 onodes 0x558c58092a00 shared_blobs 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore.extentmap(0x558c58092b08) update 4#1:a0000000::::head# 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore.extentmap(0x558c58092b08) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 pg_epoch: 39 pg[1.1(unlocked)] enter NotTrimming 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:a0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc000 allocated 0x[] released 0x[] 2019-12-07T19:34:17.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.1s1_head fresh osr 0x558c54e10140 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 prepare 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc000 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 io_done 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 40 handle_pg_create_info new pg pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7:5._attach_pg 1.5s4 0x558c57522000 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [3,7,2,4,5,1] acting [3,2147483647,2,4,5,1] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 past_intervals ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) 2019-12-07T19:34:17.618+0000 7f8f0217d700 20 osd.7 40 identify_splits_and_merges 1.5s4 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7:5._prime_splits 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.6s2 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c55141000 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:17.618+0000 7f8f141a1700 15 osd.7 40 enqueue_peering_evt 1.7s4 epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 41.055589652s + 0s) 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:17.618+0000 7f8f141a1700 7 osd.7 40 activate_map version 40 2019-12-07T19:34:17.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:17.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c54d5c000 2019-12-07T19:34:17.618+0000 7f8f0918b700 20 osd.7 40 agent_entry empty queue 2019-12-07T19:34:17.618+0000 7f8f06185700 15 osd.7 40 enqueue_peering_evt 1.ds2 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.618+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c55141000 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:17.618+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:17.620179+0000 0 0.000000 2019-12-07T19:34:17.618+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.618+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:17.618+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_advance_map: 40 2019-12-07T19:34:17.618+0000 7f8f04181700 20 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:17.618+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 crt=21'6 remapped mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.618+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 39 -> 40 2019-12-07T19:34:17.618+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:17.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:17.618+0000 7f8f05984700 10 osd.7 pg_epoch: 39 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:17.619+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.ds0 1.ds0 1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 -- 0x558c576baa00 con 0x558c54c19800 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 39 2019-12-07T19:34:17.619+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:17.620217+0000 1 0.000030 2019-12-07T19:34:17.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1441 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:17.619+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1441 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 1 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.619+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:17.620231+0000 0 0.000000 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xef52d~5a1 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:17.619+0000 7f8f05984700 20 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 39 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xef52d 0xef52d~5a1 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f05984700 20 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xbf52d 2019-12-07T19:34:17.619+0000 7f8f05984700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x52d 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xace and padding block with 0x532 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 39 -> 40 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c54836f00 seq 1 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.1s1_head bits 4 2019-12-07T19:34:17.619+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_activate_map: 40 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 39 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,2147483647,5] -- 2/2 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.1s1_head bits 4 = 0 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:17.619+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] activate_map 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) get_onode oid 1#1:80000000::::head# key 0x8180000000000000018000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.619+0000 7f8f05984700 20 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.619+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.619+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:17.619+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.619+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) r -2 v.len 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1179 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._infover' <- _infover 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 NullEvt) prio 255 cost 10 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 5934 (1 ios * 4000 + 1934 bytes) 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes 0x558c57976a00 shared_blobs 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map: 40 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore.extentmap(0x558c57976b08) update 1#1:80000000::::head# 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 2(0) log log((0'0,27'8], crt=27'8) pi ([28,38] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([28,37] acting 0(4),1(1),2(0),3(2),4(3),5(5))) pg_lease(ru 0s ub 57.091205159s int 16s) +create_info 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 39 -> 40 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore.extentmap(0x558c57976b08) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:80000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xeface 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.2(0) 1.5s0( v 27'8 (0'0,27'8] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,27'8], crt=27'8) 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 39 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,2147483647,7,2] -- 1/1 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 merge_log log((0'0,27'8], crt=27'8) from osd.2(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 merge_log extending head to 27'8 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000263608s (0.000000092s flush + 0.000263516s kv commit) 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 27'8 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:203 0.000000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 23'7 (21'6) delete 1:a7686634:::smithi13140796-35:head by client.4540.0:185 2019-12-07T19:34:00.666376+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1330 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 21'6 (21'5) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:104 2019-12-07T19:33:58.715053+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 40 handle_pg_create_info new pg pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 21'5 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:103 2019-12-07T19:33:58.715018+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1539 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7:1._attach_pg 1.1s1 0x558c5736a000 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 21'4 (21'3) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:71 2019-12-07T19:33:58.678863+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 40 identify_splits_and_merges 1.1s1 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7:1._prime_splits 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 21'3 (0'0) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:70 2019-12-07T19:33:58.678831+0000 0 ObjectCleanRegions clean_offsets: [393216~18446744073709158399], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 21'2 (21'1) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:35 2019-12-07T19:33:58.636666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 ? 21'1 (0'0) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:34 2019-12-07T19:33:58.636620+0000 0 ObjectCleanRegions clean_offsets: [196608~18446744073709355007], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3067 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 21'1 (0'0) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:34 2019-12-07T19:33:58.636620+0000 0 ObjectCleanRegions clean_offsets: [196608~18446744073709355007], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 39 -> 40 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] handle_activate_map: 40 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3090 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3090 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 21'2 (21'1) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:35 2019-12-07T19:33:58.636666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddf200> 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 39 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map [3,7,2,4,5,1]/[3,2147483647,2,4,5,1] -- 3/3 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 kv_submitted 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddf200 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 21'3 (0'0) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:70 2019-12-07T19:33:58.678831+0000 0 ObjectCleanRegions clean_offsets: [393216~18446744073709158399], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xeface~c12 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 finishing 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddf200 onodes 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddf200 done 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 21'4 (21'3) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:71 2019-12-07T19:33:58.678863+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xeface 0xeface~c12 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:16.616102+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _allocate len 0x6e0 from 0 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddf200 [] 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _allocate len 0x6e0 from 1 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.619+0000 7f8f06986700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd220000 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 21'5 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:103 2019-12-07T19:33:58.715018+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd220000~10000/10000,10000,acd220000 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:17.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x100000 to include allocated 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 37 (was clean) 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 21'6 (21'5) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:104 2019-12-07T19:33:58.715053+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xbface 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xace 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.619+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 NullEvt 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6e0 and padding block with 0x920 2019-12-07T19:34:17.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x100000 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 23'7 (21'6) delete 1:a7686634:::smithi13140796-35:head by client.4540.0:185 2019-12-07T19:34:00.666376+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 update missing, append 27'8 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:203 0.000000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 merge_log result log((0'0,27'8], crt=27'8) missing(3 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.619+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:40 sent:40 1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 4->0 ([28,38] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([28,37] acting 0(3),1(0),2(5),3(2),4(1),7(4)))) e40/40) v1 -- 0x558c574e5500 con 0x558c54c64000 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.619+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 0'0 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 unknown m=3 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.620424+0000 3 0.000058 2019-12-07T19:34:17.619+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 0'0 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 unknown m=3 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'9], crt=29'9) pi ([28,38] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([28,37] acting 0(1),1(5),2(2),3(0),4(3),5(4))) pg_lease(ru 0s ub 57.076160187s int 16s) +create_info 2019-12-07T19:34:17.619+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: _activate_committed 40 peer_activated now 7(0) last_interval_started 18 last_epoch_started 19 same_interval_since 39 2019-12-07T19:34:17.619+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 0'0 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 unknown m=3 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.1s0( v 29'9 (0'0,29'9] local-lis/les=39/40 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,29'9], crt=29'9) 2019-12-07T19:34:17.619+0000 7f8f04181700 10 merge_log log((0'0,29'9], crt=29'9) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.619+0000 7f8f04181700 10 merge_log extending head to 29'9 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 0'0 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 unknown m=3 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 29'9 (27'7) modify 1:891f0d13:::smithi13140796-14:head by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 29'8 (27'4) delete 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 0'0 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 unknown m=3 mbc={} ps=[a~1]] activate - not complete, missing(3 may_include_deletes = 1) 2019-12-07T19:34:17.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 27'7 (27'6) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:216 2019-12-07T19:34:01.670030+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 27'6 (27'5) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:215 2019-12-07T19:34:01.670014+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 lcod 0'0 unknown m=3 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 27'5 (21'3) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:214 2019-12-07T19:34:01.667796+0000 0 ObjectCleanRegions clean_offsets: [0~229376,884736~18446744073708666879], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 27'4 (21'3) clone 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:33:58.819566+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 ? 21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 27'4 (21'3) clone 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:33:58.819566+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 27'5 (21'3) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:214 2019-12-07T19:34:01.667796+0000 0 ObjectCleanRegions clean_offsets: [0~229376,884736~18446744073708666879], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 27'6 (27'5) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:215 2019-12-07T19:34:01.670014+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 27'7 (27'6) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:216 2019-12-07T19:34:01.670030+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf06e0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 29'8 (27'4) delete 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 update missing, append 29'9 (27'7) modify 1:891f0d13:::smithi13140796-14:head by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c577fd800 seq 2 2019-12-07T19:34:17.619+0000 7f8f04181700 10 merge_log result log((0'0,29'9], crt=29'9) missing(2 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.619+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore.onode(0x558c58092a00).flush flush done 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 0'0 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 unknown m=2 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.620552+0000 3 0.000036 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 0'0 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 unknown m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.619+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 0'0 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 unknown m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (37) on file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]), flushing log 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 0'0 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 unknown m=2 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000049a'.0000000000.00000000000000000000' end: 0x000000000000049a'.4294967295.18446744073709551615' 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 37 len 0x49 crc 0x581a2578) 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x24000~1000 to file(ino 1 size 0x24000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x24000 0x24000~1000 to file(ino 1 size 0x24000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x25000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x24000 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 0'0 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 unknown m=2 mbc={} ps=[a~1]] activate - not complete, missing(2 may_include_deletes = 1) 2019-12-07T19:34:17.619+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.619+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 lcod 0'0 unknown m=2 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:17.619+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000023.00000000000000000007' <- 0000000023.00000000000000000007 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.0000000027.00000000000000000008' <- 0000000027.00000000000000000008 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x25000 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.missing/0000000000000001.51B7DFC8.head.smithi13140796-24..' <- missing/0000000000000001.51B7DFC8.head.smithi13140796-24.. 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.missing/0000000000000001.59B75E6C.head.smithi13140796-12..' <- missing/0000000000000001.59B75E6C.head.smithi13140796-12.. 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.missing/0000000000000001.5E6166C2.head.smithi13140796-35..' <- missing/0000000000000001.5E6166C2.head.smithi13140796-35.. 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd800 cost 9390 (1 ios * 4000 + 5390 bytes) 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd800 onodes shared_blobs 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd800 allocated 0x[] released 0x[] 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 prepare 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd800 2019-12-07T19:34:17.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 io_done 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000024 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c57ef8c00 seq 2 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore.onode(0x558c57976a00).flush flush done 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 37 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000345037s (0.000000083s flush + 0.000344954s kv commit) 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc000,0x558c54836f00> 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000049b'.0000000000.00000000000000000000' end: 0x000000000000049b'.4294967295.18446744073709551615' 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5178 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 kv_submitted 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc000 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 finishing 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc000 onodes 0x558c58092a00 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000027.00000000000000000004' <- 0000000027.00000000000000000004 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc000 done 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd800 kv_submitted 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc000 [] 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000027.00000000000000000005' <- 0000000027.00000000000000000005 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5201 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5201 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000027.00000000000000000006' <- 0000000027.00000000000000000006 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 finishing 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000027.00000000000000000007' <- 0000000027.00000000000000000007 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 0x558c57976a00 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8c00 prepare 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf06e0~1451 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000029.00000000000000000008' <- 0000000029.00000000000000000008 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf06e0 0xf06e0~1451 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.0000000029.00000000000000000009' <- 0000000029.00000000000000000009 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc06e0 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6e0 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb31 and padding block with 0x4cf 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.missing/0000000000000001.198F0B8C.4.smithi13140796-14..' <- missing/0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.missing/0000000000000001.198F0B8C.head.smithi13140796-14..' <- missing/0000000000000001.198F0B8C.head.smithi13140796-14.. 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8c00 cost 11831 (1 ios * 4000 + 7831 bytes) 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8c00 onodes shared_blobs 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8c00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 prepare 2019-12-07T19:34:17.619+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8c00 2019-12-07T19:34:17.619+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 io_done 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf1b31 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000097000s (0.000000090s flush + 0.000096910s kv commit) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7620 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7643 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7643 < min_flush_size 524288 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf1b31~1ddb to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf1b31 0xf1b31~1ddb to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc1b31 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb31 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x90c and padding block with 0x6f4 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf390c 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000117847s (0.000000076s flush + 0.000117771s kv commit) 2019-12-07T19:34:17.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd800,0x558c57ef8c00> 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 kv_submitted 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd800 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 finishing 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd800 onodes 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd800 done 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd800 [] 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 kv_submitted 2019-12-07T19:34:17.619+0000 7f8f06185700 15 osd.7 40 enqueue_peering_evt 1.5s4 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8c00 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 finishing 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8c00 onodes 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8c00 done 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8c00 [] 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c57522000 2019-12-07T19:34:17.619+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 lcod 0'0 unknown m=3 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f08189700 15 osd.7 40 enqueue_peering_evt 1.1s1 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.619+0000 7f8f08189700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.619+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 crt=27'8 lcod 0'0 unknown m=3 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c5736a000 2019-12-07T19:34:17.619+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.5s0 1.5s0 1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c576b8700 con 0x558c54c19800 2019-12-07T19:34:17.619+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 lcod 0'0 unknown m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'9 lcod 0'0 unknown m=2 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f08189700 10 osd.7 40 send_incremental_map 39 -> 40 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:17.619+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(40..40 src has 1..40) v4 -- 0x558c5738b8c0 con 0x558c54c5bc00 2019-12-07T19:34:17.619+0000 7f8f08189700 10 osd.7 40 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 39 -> 40 (shared) 2019-12-07T19:34:17.619+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.1s0 1.1s0 1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c54e06a00 con 0x558c54c5bc00 2019-12-07T19:34:17.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 140 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) e40/40) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c574e5500 con 0x558c54c5b400 2019-12-07T19:34:17.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.619+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.3s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.068106367s int 16s) 2019-12-07T19:34:17.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.068106367s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.068106367s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.068106367s int 16s)) prio 255 cost 10 e40) pg 0x558c5513e000 2019-12-07T19:34:17.619+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.068106367s int 16s) 2019-12-07T19:34:17.619+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] state: got info from osd.4(0) 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.619+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 57.068106367s int 16s) 2019-12-07T19:34:17.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 141 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) e40/40) v1 ==== 1030+0+0 (unknown 0 0 0) 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:17.619+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.013603746s -> ruub 57.054502621s 2019-12-07T19:34:17.619+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 57.054502621s 2019-12-07T19:34:17.619+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.621124+0000 6 0.000095 2019-12-07T19:34:17.619+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.bs3 epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39)) prio 255 cost 10 e40) 2019-12-07T19:34:17.619+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.619+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39)) prio 255 cost 10 e40) pg 0x558c54d9b000 2019-12-07T19:34:17.619+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped mbc={} ps=[a~1]] activate - no missing, moving last_complete 37'13 -> 37'13 2019-12-07T19:34:17.619+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 142 ==== pg_info2(1.4s0 1.4s0 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:17.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] state: got info from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.619+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:17.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c548f6900 seq 42 2019-12-07T19:34:17.619+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.621160+0000 7 0.000162 2019-12-07T19:34:17.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:17.619+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:17.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:17.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6900 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:17.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6900 onodes shared_blobs 2019-12-07T19:34:17.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6900 allocated 0x[] released 0x[] 2019-12-07T19:34:17.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped mbc={} ps=[a~1]] activate - no missing, moving last_complete 21'12 -> 21'12 2019-12-07T19:34:17.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 prepare 2019-12-07T19:34:17.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6900 2019-12-07T19:34:17.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 io_done 2019-12-07T19:34:17.619+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.620+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c577fcf00 seq 22 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 57.054076579s 2019-12-07T19:34:17.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcf00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf390c~517 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 57.054076579s) 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcf00 onodes shared_blobs 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf390c 0xf390c~517 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcf00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc390c 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x90c 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe23 and padding block with 0x1dd 2019-12-07T19:34:17.620+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: peer osd.4(5) activated and committed 2019-12-07T19:34:17.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 prepare 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcf00 2019-12-07T19:34:17.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 io_done 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf3e23 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000110260s (0.000000123s flush + 0.000110137s kv commit) 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf3e23~522 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf3e23 0xf3e23~522 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc3e23 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe23 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x345 and padding block with 0xcbb 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.620+0000 7f8f06986700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:62 2019-12-07T19:34:17.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6900> 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 kv_submitted 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6900 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 finishing 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6900 onodes 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 done 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6900 [] 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.620+0000 7f8f07187700 15 osd.7 40 enqueue_peering_evt 1.3s4 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c5513e000 2019-12-07T19:34:17.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 127 ==== PGlog(1.fs5 log log((0'0,29'12], crt=29'12) pi ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) pg_lease(ru 0s ub 57.078349916s int 16s) e40/39) v6 ==== 6871+0+0 (unknown 0 0 0) 0x558c57e7d800 con 0x558c54c5bc00 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.620+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.fs5 epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'12], crt=29'12) pi ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) pg_lease(ru 0s ub 57.078349916s int 16s) +create_info 2019-12-07T19:34:17.620+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'12], crt=29'12) pi ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) pg_lease(ru 0s ub 57.078349916s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.620+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 crt=37'13 remapped mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'12], crt=29'12) pi ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) pg_lease(ru 0s ub 57.078349916s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.620+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c57393800 con 0x558c54c5b400 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 128 ==== pg_info2(1.4s0 1.4s0 1.4s1( v 37'4 lc 21'2 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c54e06a00 con 0x558c54c5bc00 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'12], crt=29'12) pi ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) pg_lease(ru 0s ub 57.078349916s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.620+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(1) info: 1.4s1( v 37'4 lc 21'2 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 0s) 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf4345 2019-12-07T19:34:17.620+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(1) info: 1.4s1( v 37'4 lc 21'2 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 0s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 no pg, should create on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'12], crt=29'12) pi ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) pg_lease(ru 0s ub 57.078349916s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.620+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 129 ==== osd_map(40..40 src has 1..40) v4 ==== 673+0+0 (unknown 0 0 0) 0x558c5738b8c0 con 0x558c54c5bc00 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132624s (0.000000078s flush + 0.000132546s kv commit) 2019-12-07T19:34:17.620+0000 7f8f1199c700 20 osd.7 40 OSD::ms_dispatch: osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.620+0000 7f8f1199c700 10 osd.7 40 do_waiters -- start 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.620+0000 7f8f1199c700 10 osd.7 40 do_waiters -- finish 2019-12-07T19:34:17.620+0000 7f8f1199c700 20 osd.7 40 _dispatch 0x558c5738b8c0 osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcf00> 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 kv_submitted 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcf00 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 finishing 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcf00 onodes 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcf00 done 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcf00 [] 2019-12-07T19:34:17.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 40 _make_pg 1.fs5 2019-12-07T19:34:17.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f07187700 15 osd.7 40 enqueue_peering_evt 1.bs3 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.f(unlocked)] enter Initial 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40) prio 255 cost 10 e40) pg 0x558c54d9b000 2019-12-07T19:34:17.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40 2019-12-07T19:34:17.620+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'12 remapped mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:17.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.620+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=29/29/0 sis=39) e40/40) v1 -- 0x558c57392a00 con 0x558c54c5b400 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 39 pg[1.f(unlocked)] enter NotTrimming 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.fs5_head fresh osr 0x558c576aa640 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [3,2,5,4,1,7] acting [3,2147483647,5,4,1,2147483647] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 past_intervals ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 41.057022471s + 0s) 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:17.620+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:17.621635+0000 0 0.000000 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 39 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:17.621664+0000 1 0.000023 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:17.620+0000 7f8f0117b700 1 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:17.621677+0000 0 0.000000 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 39 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c574fe000 seq 1 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.fs5_head bits 4 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.fs5_head bits 4 = 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:f0000000::::head# key 0x858000000000000001f000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1180 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._infover' <- _infover 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe000 cost 5945 (1 ios * 4000 + 1945 bytes) 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe000 onodes 0x558c5756f180 shared_blobs 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5756f288) update 5#1:f0000000::::head# 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5756f288) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:f0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe000 allocated 0x[] released 0x[] 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 prepare 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe000 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 io_done 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 40 handle_pg_create_info new pg pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7:7._attach_pg 1.fs5 0x558c57383000 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 40 identify_splits_and_merges 1.fs5 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7:7._prime_splits 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 39 -> 40 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 39 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647] -- 3/3 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 3(0) log log((0'0,29'12], crt=29'12) pi ([28,38] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) pg_lease(ru 0s ub 57.078349916s int 16s) +create_info 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.fs0( v 29'12 (0'0,29'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,29'12], crt=29'12) 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 merge_log log((0'0,29'12], crt=29'12) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 merge_log extending head to 29'12 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 29'12 (27'10) modify 1:f7919c75:::smithi13140796-20:head by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 29'11 (27'7) delete 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 27'10 (27'9) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:207 2019-12-07T19:34:01.662612+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 27'9 (27'8) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:206 2019-12-07T19:34:01.662593+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 27'8 (21'2) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:205 2019-12-07T19:34:01.660134+0000 0 ObjectCleanRegions clean_offsets: [0~327680,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 27'7 (21'2) clone 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:33:58.665983+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f1199c700 3 osd.7 40 handle_osd_map epochs [40,40], i have 40, src has [1,40] 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f1199c700 10 osd.7 40 no new maps here, dropping 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 ? 21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 27'7 (21'2) clone 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:33:58.665983+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 27'8 (21'2) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:205 2019-12-07T19:34:01.660134+0000 0 ObjectCleanRegions clean_offsets: [0~327680,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 27'9 (27'8) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:206 2019-12-07T19:34:01.662593+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 27'10 (27'9) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:207 2019-12-07T19:34:01.662612+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 29'11 (27'7) delete 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 update missing, append 29'12 (27'10) modify 1:f7919c75:::smithi13140796-20:head by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 merge_log result log((0'0,29'12], crt=29'12) missing(4 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 0'0 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 unknown m=4 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.622023+0000 3 0.000032 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 0'0 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 unknown m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.620+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 0'0 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 unknown m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 0'0 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 unknown m=4 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 0'0 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 unknown m=4 mbc={} ps=[a~1]] activate - not complete, missing(4 may_include_deletes = 1) 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 lcod 0'0 unknown m=4 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.620+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1539 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1562 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1562 < min_flush_size 524288 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf4345~61a to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf4345 0xf4345~61a to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc4345 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x345 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x95f and padding block with 0x6a1 2019-12-07T19:34:17.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c574fe300 seq 2 2019-12-07T19:34:17.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.620+0000 7f8f0117b700 20 bluestore.onode(0x558c5756f180).flush flush done 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000049c'.0000000000.00000000000000000000' end: 0x000000000000049c'.4294967295.18446744073709551615' 2019-12-07T19:34:17.621+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000027.00000000000000000007' <- 0000000027.00000000000000000007 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000027.00000000000000000008' <- 0000000027.00000000000000000008 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000027.00000000000000000009' <- 0000000027.00000000000000000009 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000027.00000000000000000010' <- 0000000027.00000000000000000010 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000029.00000000000000000011' <- 0000000029.00000000000000000011 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000029.00000000000000000012' <- 0000000029.00000000000000000012 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.missing/0000000000000001.F3C51654.head.smithi13140796-33..' <- missing/0000000000000001.F3C51654.head.smithi13140796-33.. 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.missing/0000000000000001.F9DF9D5A.head.smithi13140796-26..' <- missing/0000000000000001.F9DF9D5A.head.smithi13140796-26.. 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.missing/0000000000000001.FE9893EA.4.smithi13140796-20..' <- missing/0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.missing/0000000000000001.FE9893EA.head.smithi13140796-20..' <- missing/0000000000000001.FE9893EA.head.smithi13140796-20.. 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.621+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe300 cost 12425 (1 ios * 4000 + 8425 bytes) 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe300 onodes shared_blobs 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.621+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 prepare 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe300 2019-12-07T19:34:17.621+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 io_done 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 143 ==== pg_info2(1.2s4 1.2s4 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) e40/40) v1 ==== 1030+0+0 (unknown 0 0 0) 0x558c576e0a00 con 0x558c54c5b400 2019-12-07T19:34:17.621+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.2s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.621+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39)) prio 255 cost 10 e40) 2019-12-07T19:34:17.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 198 ==== pg_notify2(1.4s0 1.4s0 (query:40 sent:40 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e40/40) v1 ==== 1226+0+0 (crc 0 0 0) 0x558c576e0a00 con 0x558c54c64000 2019-12-07T19:34:17.621+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 1(2) notify: (query:40 sent:40 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.621+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 1(2) notify: (query:40 sent:40 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 199 ==== PGlog(1.as5 log log((0'0,21'2], crt=21'2) pi ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) pg_lease(ru 0s ub 57.077952194s int 16s) e40/39) v6 ==== 2124+0+0 (crc 0 0 0) 0x558c5748e000 con 0x558c54c64000 2019-12-07T19:34:17.621+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.as5 epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'2], crt=21'2) pi ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) pg_lease(ru 0s ub 57.077952194s int 16s) +create_info 2019-12-07T19:34:17.621+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'2], crt=21'2) pi ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) pg_lease(ru 0s ub 57.077952194s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 200 ==== PGlog(1.es4 log log((0'0,21'4], crt=21'4) pi ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) pg_lease(ru 0s ub 57.078380077s int 16s) e40/39) v6 ==== 2901+0+0 (crc 0 0 0) 0x558c54d86000 con 0x558c54c64000 2019-12-07T19:34:17.621+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.es4 epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'4], crt=21'4) pi ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) pg_lease(ru 0s ub 57.078380077s int 16s) +create_info 2019-12-07T19:34:17.621+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'4], crt=21'4) pi ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) pg_lease(ru 0s ub 57.078380077s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'4], crt=21'4) pi ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) pg_lease(ru 0s ub 57.078380077s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'4], crt=21'4) pi ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) pg_lease(ru 0s ub 57.078380077s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 no pg, should create on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'4], crt=21'4) pi ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) pg_lease(ru 0s ub 57.078380077s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 40 _make_pg 1.es4 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.e(unlocked)] enter Initial 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39)) prio 255 cost 10 e40) pg 0x558c5731a000 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.e(unlocked)] enter NotTrimming 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.es4_head fresh osr 0x558c57895b80 2019-12-07T19:34:17.621+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 201 ==== osd_map(40..40 src has 1..40) v4 ==== 673+0+0 (crc 0 0 0) 0x558c57641d40 con 0x558c54c64000 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.621+0000 7f8f1199c700 20 osd.7 40 OSD::ms_dispatch: osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.621+0000 7f8f1199c700 10 osd.7 40 do_waiters -- start 2019-12-07T19:34:17.621+0000 7f8f1199c700 10 osd.7 40 do_waiters -- finish 2019-12-07T19:34:17.621+0000 7f8f1199c700 20 osd.7 40 _dispatch 0x558c57641d40 osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [1,3,4,5,7,2] acting [1,3,4,5,2147483647,2] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 past_intervals ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] state: got info from osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf495f 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.622717+0000 7 0.000155 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 41.058172084s + 0s) 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000609314s (0.000000101s flush + 0.000609213s kv commit) 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8247 < min_flush_size 524288 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped mbc={} ps=[a~1]] activate - no missing, moving last_complete 29'15 -> 29'15 2019-12-07T19:34:17.621+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8270 < min_flush_size 524288 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8270 < min_flush_size 524288 2019-12-07T19:34:17.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:17.622775+0000 0 0.000000 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf495f~204e to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c548f7b00 seq 28 2019-12-07T19:34:17.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf495f 0xf495f~204e to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc495f 2019-12-07T19:34:17.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x95f 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9ad and padding block with 0x653 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes shared_blobs 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 39 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:17.622819+0000 1 0.000033 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'2], crt=21'2) pi ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) pg_lease(ru 0s ub 57.077952194s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'2], crt=21'2) pi ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) pg_lease(ru 0s ub 57.077952194s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 1 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:17.622843+0000 0 0.000000 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 no pg, should create on OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'2], crt=21'2) pi ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) pg_lease(ru 0s ub 57.077952194s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 39 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c577fdb00 seq 1 2019-12-07T19:34:17.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.es4_head bits 4 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.es4_head bits 4 = 0 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 40 _make_pg 1.as5 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) get_onode oid 4#1:70000000::::head# key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.a(unlocked)] enter Initial 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe000> 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) r -2 v.len 0 2019-12-07T19:34:17.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 kv_submitted 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe000 2019-12-07T19:34:17.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1181 2019-12-07T19:34:17.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 finishing 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe000 onodes 0x558c5756f180 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 done 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe300 kv_submitted 2019-12-07T19:34:17.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe000 [] 2019-12-07T19:34:17.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.621+0000 7f8f07988700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._infover' <- _infover 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.621+0000 7f8f07988700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f07988700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:17.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.621+0000 7f8f07988700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.621+0000 7f8f07988700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fdb00 cost 5934 (1 ios * 4000 + 1934 bytes) 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fdb00 onodes 0x558c54e12a00 shared_blobs 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12b08) update 4#1:70000000::::head# 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12b08) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 pg_epoch: 39 pg[1.a(unlocked)] enter NotTrimming 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:70000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.as5_head fresh osr 0x558c54e10f00 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fdb00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 prepare 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fdb00 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 io_done 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [1,4,2,5,3,7] acting [1,4,2,2147483647,3,2147483647] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 past_intervals ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 40 handle_pg_create_info new pg pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7:6._attach_pg 1.es4 0x558c54d98000 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 40 identify_splits_and_merges 1.es4 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7:6._prime_splits 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 41.058421958s + 0s) 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 39 -> 40 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 39 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,2147483647,2] -- 1/1 2019-12-07T19:34:17.621+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:17.623025+0000 0 0.000000 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 39 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:17.621+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'4], crt=21'4) pi ([28,38] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([28,37] acting 0(4),1(0),2(5),3(1),4(2),5(3))) pg_lease(ru 0s ub 57.078380077s int 16s) +create_info 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:17.623071+0000 1 0.000035 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.es0( v 21'4 (0'0,21'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,21'4], crt=21'4) 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 merge_log log((0'0,21'4], crt=21'4) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 merge_log extending head to 21'4 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 ? 21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f07988700 1 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 ? 21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 ? 21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:17.623097+0000 0 0.000000 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 ? 21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.621+0000 7f8f07988700 5 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 update missing, append 21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f07988700 20 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 39 2019-12-07T19:34:17.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54d21b00 seq 1 2019-12-07T19:34:17.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.as5_head bits 4 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.as5_head bits 4 = 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 update missing, append 21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 update missing, append 21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) get_onode oid 5#1:50000000::::head# key 0x8580000000000000015000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.621+0000 7f8f0197c700 20 update missing, append 21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 merge_log result log((0'0,21'4], crt=21'4) missing(2 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) r -2 v.len 0 2019-12-07T19:34:17.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1182 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 0'0 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 unknown m=2 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.623148+0000 3 0.000059 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._infover' <- _infover 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 0'0 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 unknown m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:17.621+0000 7f8f0197c700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 0'0 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 unknown m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 0'0 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 unknown m=2 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5956 (1 ios * 4000 + 1956 bytes) 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes 0x558c57e8e500 shared_blobs 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore.extentmap(0x558c57e8e608) update 5#1:50000000::::head# 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore.extentmap(0x558c57e8e608) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 0'0 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 unknown m=2 mbc={} ps=[a~1]] activate - not complete, missing(2 may_include_deletes = 1) 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:50000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 lcod 0'0 unknown m=2 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.622+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:17.622+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:17.622+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:17.622+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 40 handle_pg_create_info new pg pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7:2._attach_pg 1.as5 0x558c57e8b000 2019-12-07T19:34:17.622+0000 7f8f07988700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f07988700 20 osd.7 40 identify_splits_and_merges 1.as5 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7:2._prime_splits 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.622+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 39 -> 40 2019-12-07T19:34:17.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c577fcc00 seq 2 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 39 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647] -- 1/1 2019-12-07T19:34:17.622+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore.onode(0x558c54e12a00).flush flush cnt:1 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.622+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.622+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:34:17.622+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:17.622+0000 7f8f07988700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(1) info: 1.4s1( v 37'4 lc 21'2 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 0s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(1) info: 1.4s1( v 37'4 lc 21'2 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 0s)) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.622+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.622+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,21'2], crt=21'2) pi ([28,38] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) pg_lease(ru 0s ub 57.077952194s int 16s) +create_info 2019-12-07T19:34:17.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(1) info: 1.4s1( v 37'4 lc 21'2 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 0s) 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.as0( v 21'2 (0'0,21'2] local-lis/les=39/40 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,21'2], crt=21'2) 2019-12-07T19:34:17.622+0000 7f8f07988700 10 merge_log log((0'0,21'2], crt=21'2) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.622+0000 7f8f07988700 10 merge_log extending head to 21'2 2019-12-07T19:34:17.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: peer osd.3(1) activated and committed 2019-12-07T19:34:17.622+0000 7f8f07988700 20 ? 21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.622+0000 7f8f07988700 20 ? 21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.622+0000 7f8f07988700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.622+0000 7f8f07988700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.622+0000 7f8f07988700 20 update missing, append 21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.622+0000 7f8f07988700 20 update missing, append 21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.622+0000 7f8f07988700 10 merge_log result log((0'0,21'2], crt=21'2) missing(1 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.622+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 0'0 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 unknown m=1 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.623357+0000 3 0.000054 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.622+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 0'0 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 unknown m=1 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.622+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 0'0 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 unknown m=1 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 0'0 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 unknown m=1 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 0'0 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 unknown m=1 mbc={} ps=[a~1]] activate - not complete, missing(1 may_include_deletes = 1) 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.622+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 lcod 0'0 unknown m=1 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.622+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf69ad 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.622+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000696460s (0.000000084s flush + 0.000696376s kv commit) 2019-12-07T19:34:17.622+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54ea1200 seq 2 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:17.622+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.622+0000 7f8f07988700 20 bluestore.onode(0x558c57e8e500).flush flush cnt:1 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2819 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4369 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4392 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4392 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf69ad~1128 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf69ad 0xf69ad~1128 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc69ad 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9ad 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xad5 and padding block with 0x52b 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.622+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:63 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe300> 2019-12-07T19:34:17.622+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 kv_submitted 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe300 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 finishing 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe300 onodes 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe300 done 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 1(2) notify: (query:40 sent:40 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe300 [] 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 1(2) notify: (query:40 sent:40 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 1(2) notify: (query:40 sent:40 1.4s2( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 2->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.622+0000 7f8f05183700 15 osd.7 40 enqueue_peering_evt 1.fs5 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: Active: got notify from 1(2), already have info from that osd, ignoring 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c57383000 2019-12-07T19:34:17.622+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore.onode(0x558c54e12a00).flush flush done 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 lcod 0'0 unknown m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 crt=29'12 lcod 0'0 unknown m=4 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.622+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000049d'.0000000000.00000000000000000000' end: 0x000000000000049d'.4294967295.18446744073709551615' 2019-12-07T19:34:17.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.622+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.fs0 1.fs0 1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c548a1c00 con 0x558c54c5bc00 2019-12-07T19:34:17.622+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:17.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf7ad5 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.missing/0000000000000001.E6B036CF.head.smithi13140796-38..' <- missing/0000000000000001.E6B036CF.head.smithi13140796-38.. 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.missing/0000000000000001.ECCBE2E2.head.smithi13140796-5..' <- missing/0000000000000001.ECCBE2E2.head.smithi13140796-5.. 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000209884s (0.000000104s flush + 0.000209780s kv commit) 2019-12-07T19:34:17.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcc00 cost 7762 (1 ios * 4000 + 3762 bytes) 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcc00 onodes shared_blobs 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcc00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7b00,0x558c577fdb00,0x558c54d21b00> 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:17.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 prepare 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcc00 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:17.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 io_done 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 kv_submitted 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fdb00 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 finishing 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fdb00 onodes 0x558c54e12a00 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fdb00 done 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcc00 kv_queued 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fdb00 [] 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 0x558c57e8e500 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 prepare 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1323 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3527 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3550 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3550 < min_flush_size 524288 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf7ad5~dde to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf7ad5 0xf7ad5~dde to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc7ad5 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xad5 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8b3 and padding block with 0x74d 2019-12-07T19:34:17.622+0000 7f8f1199c700 3 osd.7 40 handle_osd_map epochs [40,40], i have 40, src has [1,40] 2019-12-07T19:34:17.622+0000 7f8f1199c700 10 osd.7 40 no new maps here, dropping 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf88b3 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000115099s (0.000000052s flush + 0.000115047s kv commit) 2019-12-07T19:34:17.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcc00> 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 kv_submitted 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcc00 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 finishing 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcc00 onodes 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcc00 done 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcc00 [] 2019-12-07T19:34:17.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f05984700 15 osd.7 40 enqueue_peering_evt 1.es4 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c54d98000 2019-12-07T19:34:17.622+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 lcod 0'0 unknown m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 66 ==== pg_notify2(1.4s0 1.4s0 (query:40 sent:40 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) e40/40) v1 ==== 1226+0+0 (unknown 0 0 0) 0x558c57392a00 con 0x558c54c19c00 2019-12-07T19:34:17.622+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'4 lcod 0'0 unknown m=2 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.622+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:17.622+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.es0 1.es0 1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c574e7100 con 0x558c54c64000 2019-12-07T19:34:17.622+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 5(3) notify: (query:40 sent:40 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.622+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 5(3) notify: (query:40 sent:40 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 202 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) e40/40) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57392a00 con 0x558c54c64000 2019-12-07T19:34:17.622+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.7s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.078428600s int 16s) 2019-12-07T19:34:17.622+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.078428600s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 203 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) e40/40) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57392a00 con 0x558c54c64000 2019-12-07T19:34:17.622+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.9s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.079056700s int 16s) 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.079056700s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.078428600s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.078428600s int 16s)) prio 255 cost 10 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.078428600s int 16s) 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] state: got info from osd.1(0) 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 57.078428600s int 16s) 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.021415354s -> ruub 57.057013246s 2019-12-07T19:34:17.622+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 57.057013246s 2019-12-07T19:34:17.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.079056700s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.622+0000 7f8f05183700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.624090+0000 6 0.000091 2019-12-07T19:34:17.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.079056700s int 16s)) prio 255 cost 10 e40) pg 0x558c5500a000 2019-12-07T19:34:17.622+0000 7f8f05183700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.622+0000 7f8f05183700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.622+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) pg_lease(ru 0s ub 57.079056700s int 16s) 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.622+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] state: got info from osd.1(0) 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped mbc={} ps=[a~1]] activate - no missing, moving last_complete 27'10 -> 27'10 2019-12-07T19:34:17.622+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 57.079056700s int 16s) 2019-12-07T19:34:17.622+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.622+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.021415354s -> ruub 57.057641346s 2019-12-07T19:34:17.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 204 ==== PGlog(1.8s5 log log((0'0,36'33], crt=36'33) pi ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) pg_lease(ru 0s ub 57.080492985s int 16s) e40/39) v6 ==== 13747+0+0 (crc 0 0 0) 0x558c57e7b000 con 0x558c54c64000 2019-12-07T19:34:17.622+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.622+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 57.057641346s 2019-12-07T19:34:17.622+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.8s5 epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,36'33], crt=36'33) pi ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) pg_lease(ru 0s ub 57.080492985s int 16s) +create_info 2019-12-07T19:34:17.622+0000 7f8f08189700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.624152+0000 6 0.000086 2019-12-07T19:34:17.622+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,36'33], crt=36'33) pi ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) pg_lease(ru 0s ub 57.080492985s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:17.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5730a600 seq 42 2019-12-07T19:34:17.622+0000 7f8f08189700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:17.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:17.622+0000 7f8f08189700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:17.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:17.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:17.622+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a600 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:17.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a600 onodes shared_blobs 2019-12-07T19:34:17.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a600 allocated 0x[] released 0x[] 2019-12-07T19:34:17.622+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped mbc={} ps=[a~1]] activate - no missing, moving last_complete 34'17 -> 34'17 2019-12-07T19:34:17.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 prepare 2019-12-07T19:34:17.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a600 2019-12-07T19:34:17.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 io_done 2019-12-07T19:34:17.622+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,36'33], crt=36'33) pi ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) pg_lease(ru 0s ub 57.080492985s int 16s) +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,36'33], crt=36'33) pi ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) pg_lease(ru 0s ub 57.080492985s int 16s) +create_info) prio 255 cost 10 e40) pg 0 2019-12-07T19:34:17.623+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:17.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740a600 seq 43 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 no pg, should create on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,36'33], crt=36'33) pi ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) pg_lease(ru 0s ub 57.080492985s int 16s) +create_info) prio 255 cost 10 e40) 2019-12-07T19:34:17.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:17.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:17.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:17.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:17.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 40 _make_pg 1.8s5 2019-12-07T19:34:17.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes shared_blobs 2019-12-07T19:34:17.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:17.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:17.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:17.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8(unlocked)] enter Initial 2019-12-07T19:34:17.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2560 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2583 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2583 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 39 pg[1.8(unlocked)] enter NotTrimming 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf88b3~a17 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf88b3 0xf88b3~a17 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_attach 1.8s5_head fresh osr 0x558c57f06c80 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc88b3 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8b3 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2ca and padding block with 0xd36 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [1,4,5,2,3,7] acting [1,4,5,2,3,2147483647] history ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39 past_intervals ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 41.059795054s + 0s) 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:17.623+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:17.624391+0000 0 0.000000 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[28,39)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 39 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf92ca 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 39 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000151154s (0.000000089s flush + 0.000151065s kv commit) 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:17.624426+0000 1 0.000026 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a600,0x558c5740a600> 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 kv_submitted 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a600 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 finishing 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a600 onodes 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a600 done 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a600 [] 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.623+0000 7f8f0898a700 1 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:17.624450+0000 0 0.000000 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:17.623+0000 7f8f05183700 15 osd.7 40 enqueue_peering_evt 1.7s4 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.623+0000 7f8f05183700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 39 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c54ddef00 seq 1 2019-12-07T19:34:17.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.8s5_head bits 4 2019-12-07T19:34:17.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _create_collection 1.8s5_head bits 4 = 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:17.623+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10000000::::head# key 0x8580000000000000011000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.623+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=27'10 remapped mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.623+0000 7f8f08189700 15 osd.7 40 enqueue_peering_evt 1.9s4 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.623+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f08189700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1183 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c548a3800 con 0x558c54c64000 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c5500a000 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._infover' <- _infover 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.623+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.623+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 crt=34'17 remapped mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:17.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddef00 cost 5934 (1 ios * 4000 + 1934 bytes) 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddef00 onodes 0x558c5738db80 shared_blobs 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5738dc88) update 5#1:10000000::::head# 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5738dc88) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.623+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c54e06300 con 0x558c54c64000 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddef00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 prepare 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddef00 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 io_done 2019-12-07T19:34:17.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1528 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 40 handle_pg_create_info new pg pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7:0._attach_pg 1.8s5 0x558c57380000 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 40 identify_splits_and_merges 1.8s5 e39 to e40 pg_nums {18=16} 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7:0._prime_splits 2019-12-07T19:34:17.623+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 67 ==== osd_map(40..40 src has 1..40) v4 ==== 673+0+0 (unknown 0 0 0) 0x558c54d2f680 con 0x558c54c19c00 2019-12-07T19:34:17.623+0000 7f8f1199c700 20 osd.7 40 OSD::ms_dispatch: osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.623+0000 7f8f1199c700 10 osd.7 40 do_waiters -- start 2019-12-07T19:34:17.623+0000 7f8f1199c700 10 osd.7 40 do_waiters -- finish 2019-12-07T19:34:17.623+0000 7f8f1199c700 20 osd.7 40 _dispatch 0x558c54d2f680 osd_map(40..40 src has 1..40) v4 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1551 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f1199c700 3 osd.7 40 handle_osd_map epochs [40,40], i have 40, src has [1,40] 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1551 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f1199c700 10 osd.7 40 no new maps here, dropping 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map: 40 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf92ca~60f to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 39 -> 40 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf92ca 0xf92ca~60f to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc92ca 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 39 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,2147483647] -- 1/1 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2ca 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8d9 and padding block with 0x727 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 40 advance_pg new pool opts old pool opts 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] handle_activate_map: 40 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xf98d9 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 39 while current is 40 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000090777s (0.000000088s flush + 0.000090689s kv commit) 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 40 epoch_requested: 39 MLogRec from 1(0) log log((0'0,36'33], crt=36'33) pi ([28,38] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([28,37] acting 0(5),1(0),2(3),3(4),4(1),5(2))) pg_lease(ru 0s ub 57.080492985s int 16s) +create_info 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddef00> 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 kv_submitted 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddef00 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 finishing 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.8s0( v 36'33 (0'0,36'33] local-lis/les=39/40 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) log((0'0,36'33], crt=36'33) 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddef00 onodes 0x558c5738db80 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddef00 done 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 merge_log log((0'0,36'33], crt=36'33) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddef00 [] 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 merge_log extending head to 36'33 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 36'33 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:284 2019-12-07T19:34:13.610267+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 31'32 (31'31) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:263 2019-12-07T19:34:08.615270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 31'31 (31'30) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:262 2019-12-07T19:34:08.615254+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1163264~18446744073708388351], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 31'30 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:261 2019-12-07T19:34:08.612951+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 29'29 (27'21) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 29'28 (27'20) delete 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 29'27 (27'23) delete 1:15219530:::smithi13140796-21:head by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 29'26 (27'22) delete 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 29'25 (27'18) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 29'24 (27'15) delete 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'23 (21'6) modify 1:15219530:::smithi13140796-21:head by client.4540.0:236 2019-12-07T19:34:04.740015+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'22 (21'6) clone 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:33:58.668552+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'21 (23'14) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:229 0.000000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'20 (23'14) clone 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'18 (27'17) modify 1:10779340:::smithi13140796-32:head by client.4540.0:191 2019-12-07T19:34:01.653811+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'17 (27'16) modify 1:10779340:::smithi13140796-32:head by client.4540.0:190 2019-12-07T19:34:01.653781+0000 0 ObjectCleanRegions clean_offsets: [0~1146880,1523712~18446744073708027903], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'16 (21'8) modify 1:10779340:::smithi13140796-32:head by client.4540.0:189 2019-12-07T19:34:01.651210+0000 0 ObjectCleanRegions clean_offsets: [0~524288,1146880~18446744073708404735], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 27'15 (21'8) clone 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:33:58.703940+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 23'14 (21'3) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:181 0.000000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'13 (21'11) delete 1:14fc029d:::smithi13140796-41:head by client.4540.0:174 2019-12-07T19:33:58.823507+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'12 (21'10) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:125 2019-12-07T19:33:58.745945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'11 (21'9) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:122 2019-12-07T19:33:58.741144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'10 (0'0) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:124 2019-12-07T19:33:58.745795+0000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'9 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:121 2019-12-07T19:33:58.741112+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'8 (21'7) modify 1:10779340:::smithi13140796-32:head by client.4540.0:95 2019-12-07T19:33:58.703940+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'7 (0'0) modify 1:10779340:::smithi13140796-32:head by client.4540.0:94 2019-12-07T19:33:58.703907+0000 0 ObjectCleanRegions clean_offsets: [524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'6 (21'5) modify 1:15219530:::smithi13140796-21:head by client.4540.0:62 2019-12-07T19:33:58.668552+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'5 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:61 2019-12-07T19:33:58.668524+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'4 (21'2) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:32 2019-12-07T19:33:58.635147+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'3 (21'1) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:20 2019-12-07T19:33:58.628837+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'2 (0'0) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:31 2019-12-07T19:33:58.635123+0000 0 ObjectCleanRegions clean_offsets: [180224~18446744073709371391], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 ? 21'1 (0'0) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:19 2019-12-07T19:33:58.628809+0000 0 ObjectCleanRegions clean_offsets: [114688~18446744073709436927], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 merge_log original_crt = 0'0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'1 (0'0) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:19 2019-12-07T19:33:58.628809+0000 0 ObjectCleanRegions clean_offsets: [114688~18446744073709436927], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'2 (0'0) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:31 2019-12-07T19:33:58.635123+0000 0 ObjectCleanRegions clean_offsets: [180224~18446744073709371391], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'3 (21'1) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:20 2019-12-07T19:33:58.628837+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'4 (21'2) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:32 2019-12-07T19:33:58.635147+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'5 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:61 2019-12-07T19:33:58.668524+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'6 (21'5) modify 1:15219530:::smithi13140796-21:head by client.4540.0:62 2019-12-07T19:33:58.668552+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'7 (0'0) modify 1:10779340:::smithi13140796-32:head by client.4540.0:94 2019-12-07T19:33:58.703907+0000 0 ObjectCleanRegions clean_offsets: [524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'8 (21'7) modify 1:10779340:::smithi13140796-32:head by client.4540.0:95 2019-12-07T19:33:58.703940+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'9 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:121 2019-12-07T19:33:58.741112+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'10 (0'0) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:124 2019-12-07T19:33:58.745795+0000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'11 (21'9) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:122 2019-12-07T19:33:58.741144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'12 (21'10) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:125 2019-12-07T19:33:58.745945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 21'13 (21'11) delete 1:14fc029d:::smithi13140796-41:head by client.4540.0:174 2019-12-07T19:33:58.823507+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 23'14 (21'3) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:181 0.000000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'15 (21'8) clone 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:33:58.703940+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'16 (21'8) modify 1:10779340:::smithi13140796-32:head by client.4540.0:189 2019-12-07T19:34:01.651210+0000 0 ObjectCleanRegions clean_offsets: [0~524288,1146880~18446744073708404735], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'17 (27'16) modify 1:10779340:::smithi13140796-32:head by client.4540.0:190 2019-12-07T19:34:01.653781+0000 0 ObjectCleanRegions clean_offsets: [0~1146880,1523712~18446744073708027903], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'18 (27'17) modify 1:10779340:::smithi13140796-32:head by client.4540.0:191 2019-12-07T19:34:01.653811+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'20 (23'14) clone 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 5(3) notify: (query:40 sent:40 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40)> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'21 (23'14) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:229 0.000000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 5(3) notify: (query:40 sent:40 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'22 (21'6) clone 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:33:58.668552+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 27'23 (21'6) modify 1:15219530:::smithi13140796-21:head by client.4540.0:236 2019-12-07T19:34:04.740015+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 0, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 29'24 (27'15) delete 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MNotifyRec 1.4s0 from 5(3) notify: (query:40 sent:40 1.4s3( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) 3->0 ([18,38] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([18,37] acting 0(1),1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 29'25 (27'18) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 29'26 (27'22) delete 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 29'27 (27'23) delete 1:15219530:::smithi13140796-21:head by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: Active: got notify from 5(3), already have info from that osd, ignoring 2019-12-07T19:34:17.623+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 29'28 (27'20) delete 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 29'29 (27'21) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore.onode(0x558c57e8e500).flush flush done 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 31'30 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:261 2019-12-07T19:34:08.612951+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 31'31 (31'30) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:262 2019-12-07T19:34:08.615254+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1163264~18446744073708388351], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 31'32 (31'31) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:263 2019-12-07T19:34:08.615270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 update missing, append 36'33 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:284 2019-12-07T19:34:13.610267+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 merge_log result log((0'0,36'33], crt=36'33) missing(9 may_include_deletes = 1) changed=1 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000049e'.0000000000.00000000000000000000' end: 0x000000000000049e'.4294967295.18446744073709551615' 2019-12-07T19:34:17.623+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 0'0 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 unknown m=9 mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:17.624874+0000 3 0.000050 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 0'0 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 unknown m=9 mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:17.623+0000 7f8f0898a700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 0'0 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 unknown m=9 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.missing/0000000000000001.A833F94F.head.smithi13140796-15..' <- missing/0000000000000001.A833F94F.head.smithi13140796-15.. 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 0'0 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 unknown m=9 mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.623+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 6734 (1 ios * 4000 + 2734 bytes) 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 0'0 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 unknown m=9 mbc={} ps=[a~1]] activate - not complete, missing(9 may_include_deletes = 1) 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 lcod 0'0 unknown m=9 mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:17.623+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:17.623+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:17.623+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:17.623+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.623+0000 7f8f07988700 15 osd.7 40 enqueue_peering_evt 1.2s4 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40) prio 255 cost 10 e40) pg 0x558c5731a000 2019-12-07T19:34:17.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:17.623+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 29 processed at 40 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:34:17.623+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 crt=29'15 remapped mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xf98d9~9bb to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xf98d9 0xf98d9~9bb to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=29/29/0 sis=39) e40/40) v1 -- 0x558c576de000 con 0x558c54c5b400 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xc98d9 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8d9 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x294 and padding block with 0xd6c 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xfa294 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131337s (0.000000085s flush + 0.000131252s kv commit) 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200> 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:17.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:17.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.623+0000 7f8f07988700 15 osd.7 40 enqueue_peering_evt 1.as5 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c57e8b000 2019-12-07T19:34:17.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 lcod 0'0 unknown m=1 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.623+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=21'2 lcod 0'0 unknown m=1 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.623+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.623+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.as0 1.as0 1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c576e1800 con 0x558c54c64000 2019-12-07T19:34:17.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c54ea0c00 seq 2 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore.onode(0x558c5738db80).flush flush done 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x000000000000049f'.0000000000.00000000000000000000' end: 0x000000000000049f'.4294967295.18446744073709551615' 2019-12-07T19:34:17.623+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.623+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:17.623+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000021.00000000000000000013' <- 0000000021.00000000000000000013 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000023.00000000000000000014' <- 0000000023.00000000000000000014 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000015' <- 0000000027.00000000000000000015 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000016' <- 0000000027.00000000000000000016 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000017' <- 0000000027.00000000000000000017 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000018' <- 0000000027.00000000000000000018 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000019' <- 0000000027.00000000000000000019 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000020' <- 0000000027.00000000000000000020 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000021' <- 0000000027.00000000000000000021 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000022' <- 0000000027.00000000000000000022 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000027.00000000000000000023' <- 0000000027.00000000000000000023 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000029.00000000000000000024' <- 0000000029.00000000000000000024 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000029.00000000000000000025' <- 0000000029.00000000000000000025 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000029.00000000000000000026' <- 0000000029.00000000000000000026 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000029.00000000000000000027' <- 0000000029.00000000000000000027 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000029.00000000000000000028' <- 0000000029.00000000000000000028 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000029.00000000000000000029' <- 0000000029.00000000000000000029 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000031.00000000000000000030' <- 0000000031.00000000000000000030 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000031.00000000000000000031' <- 0000000031.00000000000000000031 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000031.00000000000000000032' <- 0000000031.00000000000000000032 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000036.00000000000000000033' <- 0000000036.00000000000000000033 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.80EE9C20.4.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.80EE9C20.head.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.head.smithi13140796-32.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.80FE695C.head.smithi13140796-42..' <- missing/0000000000000001.80FE695C.head.smithi13140796-42.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.82F3049B.head.smithi13140796-41..' <- missing/0000000000000001.82F3049B.head.smithi13140796-41.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.8A489AC0.4.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.8A489AC0.head.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.head.smithi13140796-21.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.8C824871.4.smithi13140796-7..' <- missing/0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.8C824871.head.smithi13140796-7..' <- missing/0000000000000001.8C824871.head.smithi13140796-7.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.missing/0000000000000001.8CAC8A97.head.smithi13140796-11..' <- missing/0000000000000001.8CAC8A97.head.smithi13140796-11.. 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 21073 (1 ios * 4000 + 17073 bytes) 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes shared_blobs 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:17.624+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 17072 < min_flush_size 524288 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 17095 < min_flush_size 524288 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 17095 < min_flush_size 524288 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xfa294~42c7 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xfa294 0xfa294~42c7 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xca294 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x294 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x55b and padding block with 0xaa5 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xfe55b 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000173985s (0.000000120s flush + 0.000173865s kv commit) 2019-12-07T19:34:17.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:17.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:17.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.624+0000 7f8f0898a700 15 osd.7 40 enqueue_peering_evt 1.8s5 epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 68 ==== pg_info2(1.4s0 1.4s0 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c57392a00 con 0x558c54c19c00 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.624+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40) prio 255 cost 10 e40) pg 0x558c57380000 2019-12-07T19:34:17.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.624+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 lcod 0'0 unknown m=9 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 ActivateCommitted from 40 processed at 40 2019-12-07T19:34:17.624+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 crt=36'33 lcod 0'0 unknown m=9 mbc={} ps=[a~1]] state: react 40 telling primary 2019-12-07T19:34:17.624+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:17.624+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.8s0 1.8s0 1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 -- 0x558c576e1100 con 0x558c54c64000 2019-12-07T19:34:17.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 57.054076579s 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 57.054076579s 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 57.054076579s) 2019-12-07T19:34:17.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: peer osd.5(3) activated and committed 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.624+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:64 2019-12-07T19:34:17.624+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.625+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 83 ==== pg_info2(1.4s0 1.4s0 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c19800 2019-12-07T19:34:17.625+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.625+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.625+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 57.054076579s 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 57.054076579s 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 57.054076579s 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 57.054076579s) 2019-12-07T19:34:17.625+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: peer osd.2(4) activated and committed 2019-12-07T19:34:17.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 144 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c576de000 con 0x558c54c5b400 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.625+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.bs1 epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066590630s ub 57.066590630s int 16s) 2019-12-07T19:34:17.625+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066590630s ub 57.066590630s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066590630s ub 57.066590630s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 145 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c576de000 con 0x558c54c5b400 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066590630s ub 57.066590630s int 16s)) prio 255 cost 10 e40) pg 0x558c575ef000 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.625+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.3s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.068106367s ub 57.068106367s int 16s) 2019-12-07T19:34:17.625+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.068106367s ub 57.068106367s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066590630s ub 57.066590630s int 16s) 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.625+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c577fc300 seq 3 2019-12-07T19:34:17.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.068106367s ub 57.068106367s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.625+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.068106367s ub 57.068106367s int 16s)) prio 255 cost 10 e40) pg 0x558c5513e000 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.625+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.625+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc300 onodes shared_blobs 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.625+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.068106367s ub 57.068106367s int 16s) 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.625+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 prepare 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc300 2019-12-07T19:34:17.625+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 io_done 2019-12-07T19:34:17.625+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.625+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:17.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c548f7800 seq 43 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:17.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:17.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes shared_blobs 2019-12-07T19:34:17.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[] released 0x[] 2019-12-07T19:34:17.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:34:17.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:34:17.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:17.625+0000 7f8f06986700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:65 2019-12-07T19:34:17.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2046 < min_flush_size 524288 2019-12-07T19:34:17.625+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:17.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:34:17.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:34:17.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xfe55b~815 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xfe55b 0xfe55b~815 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xce55b 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x55b 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd70 and padding block with 0x290 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xfed70 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000095459s (0.000000118s flush + 0.000095341s kv commit) 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc300,0x558c548f7800> 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 kv_submitted 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc300 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 finishing 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc300 onodes 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc300 done 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc300 [] 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 finishing 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 84 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c19800 2019-12-07T19:34:17.626+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.5s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.091205159s ub 57.091205159s int 16s) 2019-12-07T19:34:17.626+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.091205159s ub 57.091205159s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.091205159s ub 57.091205159s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.091205159s ub 57.091205159s int 16s)) prio 255 cost 10 e40) pg 0x558c57522000 2019-12-07T19:34:17.626+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.091205159s ub 57.091205159s int 16s) 2019-12-07T19:34:17.626+0000 7f8f06185700 20 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.626+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:17.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:17.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5730b800 seq 3 2019-12-07T19:34:17.626+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:17.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b800 onodes shared_blobs 2019-12-07T19:34:17.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b800 allocated 0x[] released 0x[] 2019-12-07T19:34:17.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 prepare 2019-12-07T19:34:17.626+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b800 2019-12-07T19:34:17.626+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 io_done 2019-12-07T19:34:17.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xfed70~416 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xfed70 0xfed70~416 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xced70 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd70 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x186 and padding block with 0xe7a 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xff186 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000083442s (0.000000094s flush + 0.000083348s kv commit) 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b800> 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 kv_submitted 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b800 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 finishing 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b800 onodes 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 done 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b800 [] 2019-12-07T19:34:17.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 205 ==== pg_info2(1.4s0 1.4s0 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c576de000 con 0x558c54c64000 2019-12-07T19:34:17.628+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.628+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s)) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease_ack(ruub 57.054076579s) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 57.054076579s 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 57.054076579s 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 57.054076579s 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 57.054076579s 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 57.054076579s (sent 57.054076579s) 2019-12-07T19:34:17.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: peer osd.1(2) activated and committed 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 206 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c576e1100 con 0x558c54c64000 2019-12-07T19:34:17.628+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.es4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078380077s ub 57.078380077s int 16s) 2019-12-07T19:34:17.628+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078380077s ub 57.078380077s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078380077s ub 57.078380077s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078380077s ub 57.078380077s int 16s)) prio 255 cost 10 e40) pg 0x558c54d98000 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078380077s ub 57.078380077s int 16s) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.628+0000 7f8f05984700 20 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:17.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c548f6c00 seq 3 2019-12-07T19:34:17.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes shared_blobs 2019-12-07T19:34:17.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:34:17.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:34:17.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xff186~416 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xff186 0xff186~416 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xcf186 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x186 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x59c and padding block with 0xa64 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xff59c 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.628+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:66 2019-12-07T19:34:17.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 146 ==== MRecoveryReserve(1.3s4 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54d9f4a0 con 0x558c54c5b400 2019-12-07T19:34:17.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: all_activated_and_committed 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000089371s (0.000000080s flush + 0.000089291s kv commit) 2019-12-07T19:34:17.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6c00> 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:34:17.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 finishing 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:17.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:34:17.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.3s4 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.628+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c5513e000 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.628+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.629918+0000 3 0.000071 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.628+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.628+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.628+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.3s4 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.628+0000 7f8f0b990700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.628+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.628+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.628+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.628+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c5513e000 2019-12-07T19:34:17.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] state: mnow 41.065413486s >= prior_readable_until_ub 0s 2019-12-07T19:34:17.628+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.628+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.3s0 GRANT e40/40) v3 -- 0x558c54d9e1e0 con 0x558c54c5b400 2019-12-07T19:34:17.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] share_pg_info 2019-12-07T19:34:17.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 -- 0x558c574e6300 con 0x558c54c64000 2019-12-07T19:34:17.628+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.630007+0000 1 0.000032 2019-12-07T19:34:17.628+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.628+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.628+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 -- 0x558c54e04700 con 0x558c54c19800 2019-12-07T19:34:17.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 -- 0x558c54e06a00 con 0x558c54c5bc00 2019-12-07T19:34:17.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 -- 0x558c54e04e00 con 0x558c54c5b400 2019-12-07T19:34:17.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 -- 0x558c54e05500 con 0x558c54c19c00 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.629+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:67 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] check_local 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] needs_recovery osd.3(1) has 2 missing 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] activate not all replicas are up-to-date, queueing recovery 2019-12-07T19:34:17.629+0000 7f8f0297e700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 DoRecovery 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 DoRecovery) prio 255 cost 10 e40) 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 DoRecovery) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 207 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c54e05500 con 0x558c54c64000 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.629+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.7s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078428600s ub 57.078428600s int 16s) 2019-12-07T19:34:17.629+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078428600s ub 57.078428600s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078428600s ub 57.078428600s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078428600s ub 57.078428600s int 16s)) prio 255 cost 10 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.629+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078428600s ub 57.078428600s int 16s) 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.629+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.629+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:17.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5730b200 seq 43 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.629+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:17.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:17.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes shared_blobs 2019-12-07T19:34:17.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[] released 0x[] 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:17.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:17.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:68 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xff59c~416 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xff59c 0xff59c~416 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] hit_set_clear 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xcf59c 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x59c 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] agent_stop 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9b2 and padding block with 0x64e 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54ddfb00 seq 23 2019-12-07T19:34:17.629+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddfb00 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddfb00 onodes shared_blobs 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddfb00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 prepare 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddfb00 2019-12-07T19:34:17.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 io_done 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xff9b2 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 DoRecovery) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000086728s (0.000000114s flush + 0.000086614s kv commit) 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b200> 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:17.629+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 DoRecovery 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [] 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f06986700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] exit Started/Primary/Active/Activating 2019-12-07T19:34:17.630664+0000 12 0.002705 2019-12-07T19:34:17.629+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xff9b2~48c to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xff9b2 0xff9b2~48c to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~d0000 x_off 0xcf9b2 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9b2 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe3e and padding block with 0x1c2 2019-12-07T19:34:17.629+0000 7f8f06986700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] enter Started/Primary/Active/WaitLocalRecoveryReserved 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0xffe3e 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122421s (0.000000094s flush + 0.000122327s kv commit) 2019-12-07T19:34:17.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddfb00> 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 kv_submitted 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddfb00 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 finishing 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddfb00 onodes 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddfb00 done 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddfb00 [] 2019-12-07T19:34:17.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 2 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.629+0000 7f8f06986700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:69 2019-12-07T19:34:17.629+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 147 ==== MRecoveryReserve(1.bs1 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54d9e1e0 con 0x558c54c5b400 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.bs1 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.629+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c575ef000 2019-12-07T19:34:17.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 148 ==== pg_info2(1.2s0 1.2s0 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c574e6300 con 0x558c54c5b400 2019-12-07T19:34:17.629+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.629+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.2s0 epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066233270s ub 57.066233270s int 16s) 2019-12-07T19:34:17.629+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066233270s ub 57.066233270s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.629+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.631006+0000 4 0.000196 2019-12-07T19:34:17.629+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.629+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 LocalRecoveryReserved 2019-12-07T19:34:17.629+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.629+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 LocalRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.bs1 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.629+0000 7f8f0b990700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 LocalRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 LocalRecoveryReserved) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.629+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 LocalRecoveryReserved 2019-12-07T19:34:17.629+0000 7f8f06986700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] exit Started/Primary/Active/WaitLocalRecoveryReserved 2019-12-07T19:34:17.631072+0000 1 0.000283 2019-12-07T19:34:17.629+0000 7f8f06986700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] enter Started/Primary/Active/WaitRemoteRecoveryReserved 2019-12-07T19:34:17.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:17.629+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.4s2 REQUEST prio: 190 e40/40) v3 -- 0x558c576dcd20 con 0x558c54c64000 2019-12-07T19:34:17.629+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c575ef000 2019-12-07T19:34:17.629+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.629+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.bs0 GRANT e40/40) v3 -- 0x558c54d9e780 con 0x558c54c5b400 2019-12-07T19:34:17.629+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.631158+0000 1 0.000037 2019-12-07T19:34:17.629+0000 7f8f07187700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.629+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:17.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066233270s ub 57.066233270s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066233270s ub 57.066233270s int 16s)) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.630+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.066233270s ub 57.066233270s int 16s) 2019-12-07T19:34:17.630+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.630+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:17.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57ef9200 seq 4 2019-12-07T19:34:17.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:17.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9200 onodes shared_blobs 2019-12-07T19:34:17.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9200 allocated 0x[] released 0x[] 2019-12-07T19:34:17.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 prepare 2019-12-07T19:34:17.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9200 2019-12-07T19:34:17.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 io_done 2019-12-07T19:34:17.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 85 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) e40/40) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c19800 2019-12-07T19:34:17.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.630+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.ds2 epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease(ru 57.090455690s ub 57.090455690s int 16s) 2019-12-07T19:34:17.630+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease(ru 57.090455690s ub 57.090455690s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 450 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease(ru 57.090455690s ub 57.090455690s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease(ru 57.090455690s ub 57.090455690s int 16s)) prio 255 cost 10 e40) pg 0x558c55141000 2019-12-07T19:34:17.630+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) pg_lease(ru 57.090455690s ub 57.090455690s int 16s) 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39 2019-12-07T19:34:17.630+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1034 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c574ff800 seq 27 2019-12-07T19:34:17.630+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1057 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1057 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:17.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff800 onodes shared_blobs 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff800 allocated 0x[] released 0x[] 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0xffe3e~421 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0xffe3e 0xffe3e~421 to file(ino 16 size 0x100000 mtime 2019-12-07T19:34:17.620405+0000 allocated 100000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~d0000]) 2019-12-07T19:34:17.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 prepare 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _allocate len 0x25f from 0 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff800 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _allocate len 0x25f from 1 2019-12-07T19:34:17.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 io_done 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd230000 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd230000~10000/10000,10000,acd230000 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x110000 to include allocated 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 38 (was clean) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xcfe3e 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe3e 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x25f and padding block with 0xda1 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x110000 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x10025f 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (38) on file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]), flushing log 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 38 len 0x49 crc 0x77fa429f) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x25000~1000 to file(ino 1 size 0x25000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x25000 0x25000~1000 to file(ino 1 size 0x25000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x26000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x25000 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x26000 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000024 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 38 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000461544s (0.000000076s flush + 0.000461468s kv commit) 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9200> 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 kv_submitted 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9200 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 finishing 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9200 onodes 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9200 done 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9200 [] 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x10025f~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x10025f 0x10025f~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd025f 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x25f 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x675 and padding block with 0x98b 2019-12-07T19:34:17.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 130 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c57379500 con 0x558c54c5bc00 2019-12-07T19:34:17.630+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.1s1 epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.076160187s ub 57.076160187s int 16s) 2019-12-07T19:34:17.630+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.076160187s ub 57.076160187s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x100675 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000067889s (0.000000088s flush + 0.000067801s kv commit) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 131 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558c54e06a00 con 0x558c54c5bc00 2019-12-07T19:34:17.630+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.fs5 epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078349916s ub 57.078349916s int 16s) 2019-12-07T19:34:17.630+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078349916s ub 57.078349916s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078349916s ub 57.078349916s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078349916s ub 57.078349916s int 16s)) prio 255 cost 10 e40) pg 0x558c57383000 2019-12-07T19:34:17.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.078349916s ub 57.078349916s int 16s) 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.630+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff800> 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff800 2019-12-07T19:34:17.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 finishing 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff800 onodes 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c5740a000 seq 3 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 done 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff800 [] 2019-12-07T19:34:17.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:17.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes shared_blobs 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[] released 0x[] 2019-12-07T19:34:17.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:34:17.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x100675~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x100675 0x100675~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd0675 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x675 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa8b and padding block with 0x575 2019-12-07T19:34:17.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.076160187s ub 57.076160187s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.076160187s ub 57.076160187s int 16s)) prio 255 cost 10 e40) pg 0x558c5736a000 2019-12-07T19:34:17.631+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.076160187s ub 57.076160187s int 16s) 2019-12-07T19:34:17.631+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.631+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.631+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:17.631+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c574fef00 seq 3 2019-12-07T19:34:17.631+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.631+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:17.631+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.631+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fef00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.631+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fef00 onodes shared_blobs 2019-12-07T19:34:17.631+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fef00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.631+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 prepare 2019-12-07T19:34:17.631+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fef00 2019-12-07T19:34:17.631+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 io_done 2019-12-07T19:34:17.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.631+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 149 ==== MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=62, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57e8ef00 con 0x558c54c5b400 2019-12-07T19:34:17.631+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c54e79600 prio 3 cost 0 latency 0.000022 epoch 40 MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=62, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.631+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=62, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=62, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=62, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c54d9b000 2019-12-07T19:34:17.631+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c54e79600 prio 3 cost 0 latency 0.000065 MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=62, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.631+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=62, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.631+0000 7f8f0317f700 25 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.631+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~40000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x12000 (73728) 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1960 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ad730 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ad880 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad8f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ad960 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ada40 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576adb20 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576adc00 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576adce0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576add50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576addc0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ade30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1960 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ad730 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ad880 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad8f0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ad960 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ada40 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adab0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576adb20 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576adc00 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576adce0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576add50 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576addc0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ade30 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.631+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x100a8b 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001138478s (0.000000074s flush + 0.001138404s kv commit) 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x100a8b~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.631+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x100a8b 0x100a8b~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd0a8b 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa8b 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.631+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xea1 and padding block with 0x15f 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000> 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x100ea1 2019-12-07T19:34:17.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000096439s (0.000000072s flush + 0.000096367s kv commit) 2019-12-07T19:34:17.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fef00> 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 kv_submitted 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fef00 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 finishing 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fef00 onodes 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fef00 done 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fef00 [] 2019-12-07T19:34:17.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.632+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 150 ==== MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=63, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558c57e8ec80 con 0x558c54c5b400 2019-12-07T19:34:17.632+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c54e786e0 prio 3 cost 0 latency 0.000025 epoch 40 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=63, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.632+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=63, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=63, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=63, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5513e000 2019-12-07T19:34:17.632+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c54e786e0 prio 3 cost 0 latency 0.000081 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=63, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.632+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=63, to_read={1:cd22b62d:::smithi13140796-27:head=0,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.632+0000 7f8f07187700 25 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a80 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0cb0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75f80 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e427e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 208 ==== MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=170, reqid=client.4540.0:315, at_version=40'12, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 118136+0+0 (crc 0 0 0) 0x558c57e7c800 con 0x558c54c64000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000032 epoch 40 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=170, reqid=client.4540.0:315, at_version=40'12, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a00~200 2019-12-07T19:34:17.633+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=170, reqid=client.4540.0:315, at_version=40'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42770 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e428c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=170, reqid=client.4540.0:315, at_version=40'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=170, reqid=client.4540.0:315, at_version=40'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6600~200 2019-12-07T19:34:17.633+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000084 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=170, reqid=client.4540.0:315, at_version=40'12, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=170, reqid=client.4540.0:315, at_version=40'12, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 snap_mapper.add_oid 1:ec1285ab:::smithi13140796-19:10 c,e,10 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42930 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e429a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ec1285ab:::smithi13140796-19:10 got.empty() 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ec1285ab:::smithi13140796-19:10 c,e,10 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.633+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.633+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x8e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 27'10 (0'0,27'10] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] append_log log((0'0,27'10], crt=27'10) [40'11 (21'4) clone 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [10,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,40'12 (21'4) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:315 2019-12-07T19:34:16.627511+0000 0 ObjectCleanRegions clean_offsets: [0~311296,770048~18446744073708781567], clean_omap: 1, new_object: 0] 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42a10 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1500 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x9e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'11 (0'0,40'11] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active+remapped mbc={} ps=[a~1]] add_log_entry 40'11 (21'4) clone 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [10,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'12 (0'0,40'12] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active+remapped mbc={} ps=[a~1]] add_log_entry 40'12 (21'4) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:315 2019-12-07T19:34:16.627511+0000 0 ObjectCleanRegions clean_offsets: [0~311296,770048~18446744073708781567], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xa600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xaa00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'12 (0'0,40'12] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active+remapped mbc={} ps=[a~1]] append_log approx pg log length = 12 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xaa00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xae00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'12 (0'0,40'12] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active+remapped mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:17.633+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xae00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 40'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xb600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xba00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xba00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xbe00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1490 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1420 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xbe00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57ff0000 seq 44 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ec1285ab:::smithi13140796-19:10# key 0x848000000000000001ec1285ab21736d'ithi13140796-19!='0x0000000000000010ffffffffffffffff'o' 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xc600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xca00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xca00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xce00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:10# 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1184 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.onode(0x558c54e44000).flush flush done 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xce00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x0 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:10# 0x0~13000 -> 0x0~13000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10400 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xd600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d3d180 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xda00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5792df80) get_ref 0x0~2000 Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5792df80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x0~2000: 0x0~2000 Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10401 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xda00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d3d030 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xde00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d3d030 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e13b0 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1340 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c574d23f0) get_ref 0x0~2000 Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c574d23f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xde00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10402 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d3ccb0 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d3ccb0 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c574d3f80) get_ref 0x0~2000 Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c574d3f80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xe600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x6000~2000: 0x0~2000 Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xea00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10403 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c55265a40 blob([0x8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xea00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c55265a40 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xee00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xee00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5792df10) get_ref 0x0~2000 Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5792df10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x8000~2000: 0x0~2000 Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10404 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54e43030 blob([0x8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54e43030 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5792d0a0) get_ref 0x0~2000 Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf600~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5792d0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0xa000~2000: 0x0~2000 Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10405 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xf600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54e43110 blob([0x8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfa00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54e43110 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfa00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57566fc0) get_ref 0x0~2000 Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfe00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57566fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e12d0 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1260 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0xc000~2000: 0x0~2000 Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10406 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0xfe00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54e431f0 blob([0x90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54e431f0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10600~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c575677a0) get_ref 0x0~2000 Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c575677a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0xe000~2000: 0x0~2000 Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10407 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54e432d0 blob([0x92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54e432d0 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57567730) get_ref 0x0~2000 Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57567730) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x10e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x10000~2000: 0x0~2000 Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10408 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54e433b0 blob([0x94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54e433b0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c575676c0) get_ref 0x0~2000 Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c575676c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11a00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x12000~1000: 0x0~1000 Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10409 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54e43490 blob([0x96000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54e43490 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=1)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11e00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c57567650 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57567650) get_ref 0x0~1000 Blob(0x558c57567650 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57567650) get_ref init 0x1000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e11f0 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1180 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x12000~1000: 0x0~1000 Blob(0x558c57567650 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x11e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0608) dirty_range mark inline shard dirty 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:10# = 0 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# snapset 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# snapset = 0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 1 keys 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12a00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 1 keys = 0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key (42 bytes) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key (42 bytes) = 0 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x12e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x13000~1c000 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ec1285ab:::smithi13140796-19:head# 0x13000~1c000 - have 0x13000 (77824) bytes fadvise_flags 0x20 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x13000~1c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ff0000 14 blobs 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1110 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e10a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x13e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14a00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14e00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x14e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1030 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0ee0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x15e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1c000/1000,1c000,0 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x5f5000~1c000/1000,1c000,0 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x5f5000~1c000] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16600~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57566d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567ce0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57566d90) get_ref 0x0~2000 Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567ce0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16a00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57566d90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x13000~2000: 0x0~2000 Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57567cf8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57567b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x16e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b730 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57567b90) get_ref 0x0~2000 Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b730 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57567b90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x15000~2000: 0x0~2000 Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5742b748 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b8f0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5742b810) get_ref 0x0~2000 Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b8f0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5742b810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x17000~2000: 0x0~2000 Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5742b908 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0fc0 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0f50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x17e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b7a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5742b960) get_ref 0x0~2000 Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b7a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c5742b960) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x19000~2000: 0x0~2000 Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5742b7b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578977a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18a00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578977a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c574fa930) get_ref 0x0~2000 Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578977a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c574fa930) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18e00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c578977b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578976c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x18e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578976c0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19200~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57897730) get_ref 0x0~2000 Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578976c0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57897730) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19200~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c578976d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19600~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578975e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578975e0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19600~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57897650) get_ref 0x0~2000 Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578975e0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19a00~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57897650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c578975f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19a00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19e00~200 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897500 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e08c0 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0a80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57897570) get_ref 0x0~2000 Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897500 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.blob(0x558c57897570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x19e00~200 = 512 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) 2019-12-07T19:34:17.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a200~200 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57897518 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x54000 (344064) 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a59d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a59d0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a600~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c57897490) get_ref 0x0~2000 Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a59d0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c57897490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580a59e8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1a600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a56c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1aa00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a56c0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5570) get_ref 0x0~2000 Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a56c0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1aa00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ae00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580a56d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a55e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0850 blob([0xe0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e0a10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a55e0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ae00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b200~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5650) get_ref 0x0~2000 Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a55e0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580a55f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b600~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a57a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a57a0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5490) get_ref 0x0~2000 Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a57a0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1b600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ba00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580a57b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ba00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c70 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1be00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5ce0) get_ref 0x0~2000 Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c70 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1be00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580a5c88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677960 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5c00) get_ref 0x0~2000 Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677960 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.blob(0x558c580a5c00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1c600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ca00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c54677978 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ca00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2f000 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ce00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b880 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c9a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x13000~1c000 = 0 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ce00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d200~200 2019-12-07T19:34:17.634+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d600~200 2019-12-07T19:34:17.634+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1d600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1da00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- SNA_1_000000000000000C_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1da00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1de00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- SNA_1_000000000000000E_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1de00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e200~200 2019-12-07T19:34:17.634+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000040.00000000000000000011' <- 0000000040.00000000000000000011 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000040.00000000000000000012' <- 0000000040.00000000000000000012 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1e600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ea00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ea00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ee00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fe30 blob([0x78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f19d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1ee00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0000 cost 179485 (15 ios * 4000 + 119485 bytes) 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0000 onodes 0x558c54e44000,0x558c574a0500 shared_blobs 0x558c54cf9650,0x558c54d3d110,0x558c54e42fc0,0x558c54e430a0,0x558c54e43180,0x558c54e43260,0x558c54e43340,0x558c54e43420,0x558c54e43500,0x558c55265960 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update inline shard 537 bytes from 24 extents 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1f600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fa00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1116 (573 bytes onode + 2 bytes spanning blobs + 541 bytes inline extents) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0608) update 4#1:ec1285ab:::smithi13140796-19:10# 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0608) update inline shard 270 bytes from 10 extents 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fa00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fe00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:10# is 688 (412 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a0 is 11 SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x1fe00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a1 is 11 SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a3 is 11 SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a4 is 11 SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a5 is 11 SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a6 is 11 SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20600~200 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a7 is 11 SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a8 is 11 SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a9 is 11 SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2))) 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a2 is 11 SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2))) 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0000 allocated 0x[5f5000~1c000] released 0x[] 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20a00~200 2019-12-07T19:34:17.634+0000 7f8f05183700 10 freelist allocate 0x5f5000~1c000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 20 freelist _xor first_key 0x580000 last_key 0x600000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56000 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x20e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 prepare 2019-12-07T19:34:17.634+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ff0000 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x21e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1c70 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1d50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x22e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x23e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1dc0 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1e30 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x24e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x25e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1ea0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1f10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x26e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x27e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e1f80 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x28e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'12 (0'0,40'12] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active+remapped mbc={} ps=[a~1]] op_applied version 40'12 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c54e78840 finish 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x29e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2a600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2aa00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 209 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c574e6300 con 0x558c54c64000 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2aa00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ae00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fce0 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f650 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ae00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.9s4 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.079056700s ub 57.079056700s int 16s) 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b600~200 2019-12-07T19:34:17.634+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.079056700s ub 57.079056700s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2b600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ba00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ba00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2be00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2be00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 210 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c574e6300 con 0x558c54c64000 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2c600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ca00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.as5 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.077952194s ub 57.077952194s int 16s) 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ca00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.077952194s ub 57.077952194s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ce00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ecb0 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fd50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ce00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2d600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2da00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2da00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2de00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2de00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2e600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ea00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ea00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ee00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f6c0 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ed90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2ee00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2f600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fa00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fa00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fe00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x2fe00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ed20 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fdc0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x30e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x31e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f730 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ea80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x32e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x33e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757ea10 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fb90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x34e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35200~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35600~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35600~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35a00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35a00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35e00~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x35e00~200 = 512 2019-12-07T19:34:17.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36200~200 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x54000 (344064) 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36a00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36a00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36e00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f500 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757eb60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x36e00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37a00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37a00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37e00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x37e00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38a00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38a00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38e00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757eaf0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc00 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x38e00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39a00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39a00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39e00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x39e00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3a600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3aa00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3aa00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ae00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f570 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757fc70 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ae00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3b600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ba00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ba00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3be00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3be00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3c600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ca00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ca00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ce00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757f5e0 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757ebd0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ce00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3d600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3da00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3da00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3de00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3de00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3e600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ea00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ea00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ee00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e70 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1260 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3ee00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f200~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f200~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f600~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3f600~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fa00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fa00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fe00~200 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x3fe00~200 = 512 2019-12-07T19:34:17.635+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=512 len=131072 2019-12-07T19:34:17.635+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:17.635+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:17.635+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 344064 2019-12-07T19:34:17.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key 2019-12-07T19:34:17.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key = 0 2019-12-07T19:34:17.635+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.3s0 40/39 ECSubReadReply(tid=63, attrs_read=0)) v2 -- 0x558c57f34a00 con 0x558c54c5b400 2019-12-07T19:34:17.635+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:34:17.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 aio_wait 2019-12-07T19:34:17.635+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0000 2019-12-07T19:34:17.635+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 io_done 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.635+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.635+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.079056700s ub 57.079056700s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.635+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.635+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.079056700s ub 57.079056700s int 16s)) prio 255 cost 10 e40) pg 0x558c5500a000 2019-12-07T19:34:17.635+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.079056700s ub 57.079056700s int 16s) 2019-12-07T19:34:17.635+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.635+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:17.635+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:17.635+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57ef8000 seq 44 2019-12-07T19:34:17.635+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5120 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:17.635+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:17.635+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.635+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8000 onodes shared_blobs 2019-12-07T19:34:17.635+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8000 allocated 0x[] released 0x[] 2019-12-07T19:34:17.635+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 prepare 2019-12-07T19:34:17.635+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8000 2019-12-07T19:34:17.635+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 io_done 2019-12-07T19:34:17.635+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1960 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0500 had 1 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57a08b40 space 0x558c574f1208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5143 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5143 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ad730 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983d40 space 0x558c576ad828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ad880 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad8f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x100ea1~1417 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad908 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c551a8c60 space 0x558c576ad908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x100ea1 0x100ea1~1417 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ad960 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad9e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd0ea1 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xea1 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c573d8510 space 0x558c576ad9e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2b8 and padding block with 0xd48 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ada40 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983dd0 space 0x558c576adac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576adb20 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983e60 space 0x558c576adba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576adc00 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adc88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57983ef0 space 0x558c576adc88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576adce0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576add50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576add68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f774d0 space 0x558c576add68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576addc0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ade30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ade48 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77560 space 0x558c576ade48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~40000 = 73728 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=73728 len=73728 2019-12-07T19:34:17.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:17.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:17.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1022b8 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 73728 2019-12-07T19:34:17.635+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key 2019-12-07T19:34:17.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key = 0 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000201724s (0.000041105s flush + 0.000160619s kv commit) 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 40/39 ECSubReadReply(tid=62, attrs_read=0)) v2 -- 0x558c57572f00 con 0x558c54c5b400 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1022b8~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1022b8 0x1022b8~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd22b8 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2b8 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6ce and padding block with 0x932 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0000> 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 kv_submitted 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0000 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 finishing 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0000 onodes 0x558c54e44000,0x558c574a0500 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677978 in 0x558c53a59880) _finish_write discard buffer(0x558c57f76fc0 space 0x558c54677978 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b748 in 0x558c53a59880) _finish_write discard buffer(0x558c54dba120 space 0x558c5742b748 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b7b8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f765a0 space 0x558c5742b7b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b908 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9bb90 space 0x558c5742b908 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567cf8 in 0x558c53a59880) _finish_write discard buffer(0x558c54c9aab0 space 0x558c57567cf8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897518 in 0x558c53a59880) _finish_write discard buffer(0x558c57f76ea0 space 0x558c57897518 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578975f8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f76e10 space 0x558c578975f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578976d8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f76d80 space 0x558c578976d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578977b8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f76cf0 space 0x558c578977b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a55f8 in 0x558c53a59880) _finish_write discard buffer(0x558c551a9dd0 space 0x558c580a55f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.077952194s ub 57.077952194s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a56d8 in 0x558c53a59880) _finish_write discard buffer(0x558c580cdb90 space 0x558c580a56d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a57b8 in 0x558c53a59880) _finish_write discard buffer(0x558c551a8900 space 0x558c580a57b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a59e8 in 0x558c53a59880) _finish_write discard buffer(0x558c57a08ea0 space 0x558c580a59e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5c88 in 0x558c53a59880) _finish_write discard buffer(0x558c57f76f30 space 0x558c580a5c88 0x0~2000 writing nocache) 2019-12-07T19:34:17.635+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.077952194s ub 57.077952194s int 16s)) prio 255 cost 10 e40) pg 0x558c57e8b000 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0000 done 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:17.635+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0000 [] 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.635+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 40/39 ECSubWriteReply(tid=170, last_complete=40'12, committed=1, applied=1)) v2 -- 0x558c575b9a00 con 0x558c54c64000 2019-12-07T19:34:17.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=39/40 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.077952194s ub 57.077952194s int 16s) 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1026ce 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.635+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107586s (0.000000148s flush + 0.000107438s kv commit) 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8000> 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 kv_submitted 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8000 2019-12-07T19:34:17.635+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 finishing 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8000 onodes 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8000 done 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8000 [] 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.635+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.635+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:17.635+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c5730af00 seq 3 2019-12-07T19:34:17.635+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.635+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:17.635+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.635+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730af00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.635+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730af00 onodes shared_blobs 2019-12-07T19:34:17.635+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730af00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.635+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 prepare 2019-12-07T19:34:17.635+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730af00 2019-12-07T19:34:17.635+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 io_done 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.635+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1026ce~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1026ce 0x1026ce~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd26ce 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6ce 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xae4 and padding block with 0x51c 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x102ae4 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000164917s (0.000000068s flush + 0.000164849s kv commit) 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730af00> 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 kv_submitted 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730af00 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 finishing 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730af00 onodes 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730af00 done 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730af00 [] 2019-12-07T19:34:17.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.635+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.635+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.638+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 211 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 36'33 (0'0,36'33] local-lis/les=39/40 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) e40/40) v1 ==== 1178+0+0 (crc 0 0 0) 0x558c574e6300 con 0x558c54c64000 2019-12-07T19:34:17.638+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.8s5 epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=39/40 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.080492985s ub 57.080492985s int 16s) 2019-12-07T19:34:17.638+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=39/40 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.080492985s ub 57.080492985s int 16s)) prio 255 cost 10 e40) 2019-12-07T19:34:17.638+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 212 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=171, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57572f00 con 0x558c54c64000 2019-12-07T19:34:17.638+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000015 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=171, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.638+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=171, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.638+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.638+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=39/40 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.080492985s ub 57.080492985s int 16s)) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.638+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.638+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=39/40 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.080492985s ub 57.080492985s int 16s)) prio 255 cost 10 e40) pg 0x558c57380000 2019-12-07T19:34:17.638+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=39/40 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) pg_lease(ru 57.080492985s ub 57.080492985s int 16s) 2019-12-07T19:34:17.638+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39 2019-12-07T19:34:17.638+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:17.638+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.638+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57ff1500 seq 3 2019-12-07T19:34:17.638+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.638+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.638+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.638+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1500 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:17.638+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1500 onodes shared_blobs 2019-12-07T19:34:17.638+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1500 allocated 0x[] released 0x[] 2019-12-07T19:34:17.638+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 prepare 2019-12-07T19:34:17.638+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1500 2019-12-07T19:34:17.638+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 io_done 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.638+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x102ae4~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x102ae4 0x102ae4~416 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd2ae4 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xae4 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xefa and padding block with 0x106 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x102efa 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000098808s (0.000000122s flush + 0.000098686s kv commit) 2019-12-07T19:34:17.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff1500> 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 kv_submitted 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1500 2019-12-07T19:34:17.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 finishing 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1500 onodes 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1500 done 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1500 [] 2019-12-07T19:34:17.638+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.638+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.638+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.638+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 151 ==== MOSDPGPush(1.bs1 40/39 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 74790+0+0 (unknown 0 0 0) 0x558c54dfbe00 con 0x558c54c5b400 2019-12-07T19:34:17.638+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740d080 prio 127 cost 74728 latency 0.000024 epoch 40 MOSDPGPush(1.bs1 40/39 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.638+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e40) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e40) queued 2019-12-07T19:34:17.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e40) pg 0x558c575ef000 2019-12-07T19:34:17.638+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c5740d080 prio 127 cost 74728 latency 0.000066 MOSDPGPush(1.bs1 40/39 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] 2019-12-07T19:34:17.638+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.bs1 40/39 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.638+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] on_local_recover: 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:17.638+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] got missing 1:db85c185:::smithi13140796-18:head v 21'2 2019-12-07T19:34:17.638+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[a~1]] last_complete now 21'8 log.complete_to 21'9 2019-12-07T19:34:17.638+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c548f6000 seq 4 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:db85c185:::smithi13140796-18:head# key 0x818000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:db85c185:::smithi13140796-18:head# key 0x818000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.638+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1185 2019-12-07T19:34:17.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# = 0 2019-12-07T19:34:17.638+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~12000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f6000 9 blobs 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.638+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x12000/1000,12000,0 2019-12-07T19:34:17.638+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x611000~12000/1000,12000,0 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x611000~12000] 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552796c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a07e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552796c0 blob([0x611000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07e30 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c552796c0) get_ref 0x0~2000 Blob(0x558c552796c0 blob([0x611000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07e30 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c552796c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c552796c0 blob([0x611000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57a07e48 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f49b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a075e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f49b90 blob([0x613000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a075e0 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f49b90) get_ref 0x0~2000 Blob(0x558c57f49b90 blob([0x613000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a075e0 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f49b90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57f49b90 blob([0x613000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57a075f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a06850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a06850 blob([0x615000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49b20 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57a06850) get_ref 0x0~2000 Blob(0x558c57a06850 blob([0x615000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49b20 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57a06850) get_ref init 0x2000, 1000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57a06850 blob([0x615000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f49b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f49ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f49ab0 blob([0x617000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49a40 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f49ab0) get_ref 0x0~2000 Blob(0x558c57f49ab0 blob([0x617000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49a40 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f49ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57f49ab0 blob([0x617000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f49a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f499d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f499d0 blob([0x619000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49960 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f499d0) get_ref 0x0~2000 Blob(0x558c57f499d0 blob([0x619000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49960 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f499d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57f499d0 blob([0x619000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f49978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f498f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f49880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f498f0 blob([0x61b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49880 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f498f0) get_ref 0x0~2000 Blob(0x558c57f498f0 blob([0x61b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49880 sbid 0x0)) 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore.blob(0x558c57f498f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57f498f0 blob([0x61b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f49898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578812d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578812d0 blob([0x61d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a10 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.blob(0x558c578812d0) get_ref 0x0~2000 Blob(0x558c578812d0 blob([0x61d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a10 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.blob(0x558c578812d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c578812d0 blob([0x61d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57c38a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=171, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3bab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3bab0 blob([0x61f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ea0 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bab0) get_ref 0x0~2000 Blob(0x558c57f3bab0 blob([0x61f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ea0 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.blob(0x558c57f3bab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57f3bab0 blob([0x61f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971eb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=171, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c540 blob([0x621000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b2d0 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.blob(0x558c5792c540) get_ref 0x0~2000 Blob(0x558c5792c540 blob([0x621000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b2d0 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.blob(0x558c5792c540) get_ref init 0x2000, 1000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5792c540 blob([0x621000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 sbid 0x0)) 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5809b2e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x12000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.extentmap(0x558c57e8ed88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 = 0 2019-12-07T19:34:17.639+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000888 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=171, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 4 keys 2019-12-07T19:34:17.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 4 keys = 0 2019-12-07T19:34:17.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.639+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=171, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000499'.missing/0000000000000001.BD1A381A.head.smithi13140796-18..' <- missing/0000000000000001.BD1A381A.head.smithi13140796-18.. 2019-12-07T19:34:17.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.639+0000 7f8f08189700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:17.639+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~52000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~52000 size 0x52000 (335872) 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6000 cost 115965 (10 ios * 4000 + 75965 bytes) 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6000 onodes 0x558c57e8ec80 shared_blobs 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore.extentmap(0x558c57e8ed88) update inline shard 174 bytes from 9 extents 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 665 (485 bytes onode + 2 bytes spanning blobs + 178 bytes inline extents) 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6000 allocated 0x[611000~12000] released 0x[] 2019-12-07T19:34:17.639+0000 7f8f07187700 10 freelist allocate 0x611000~12000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 freelist _xor first_key 0x600000 last_key 0x600000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 prepare 2019-12-07T19:34:17.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f6000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) need 0x1000~1000 cache has 0x[] 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need {<0x0, 0x2000> : [0x11000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.639+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need {<0x0, 0x2000> : [0xf000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need {<0x0, 0x2000> : [0xd000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need {<0x0, 0x2000> : [0x3b000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3b000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need {<0x0, 0x2000> : [0x3d000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3d000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need {<0x0, 0x2000> : [0x3f000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3f000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need {<0x0, 0x2000> : [0x41000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x41000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need {<0x0, 0x2000> : [0x43000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x43000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need {<0x0, 0x2000> : [0x45000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x45000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need {<0x0, 0x2000> : [0x47000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x47000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need {<0x0, 0x2000> : [0x49000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x49000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need {<0x0, 0x2000> : [0x13000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need {<0x0, 0x2000> : [0x15000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need {<0x0, 0x2000> : [0x17000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need {<0x0, 0x2000> : [0x19000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need {<0x0, 0x2000> : [0x1b000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need {<0x0, 0x2000> : [0x1d000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need {<0x0, 0x2000> : [0x1f000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need {<0x0, 0x2000> : [0x21000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need {<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need {<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x25000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 aio_wait 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need {<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x27000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need {<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x29000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 io_done 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need {<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2b000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need {<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2d000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need {<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2f000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need {<0x0, 0x2000> : [0x31000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x31000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need {<0x0, 0x2000> : [0x33000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x33000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need {<0x0, 0x2000> : [0x35000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x35000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need {<0x0, 0x2000> : [0x37000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x37000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need {<0x0, 0x2000> : [0x39000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x39000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need {<0x0, 0x2000> : [0x4b000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4b000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need {<0x0, 0x2000> : [0x4d000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4d000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need {<0x0, 0x2000> : [0x4f000:0~2000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4f000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) need {<0x1000, 0x1000> : [0x51000:1000~1000]} 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x51000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1988 < min_flush_size 524288 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2011 < min_flush_size 524288 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2011 < min_flush_size 524288 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x102efa~7db to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x102efa 0x102efa~7db to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd2efa 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xefa 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6d5 and padding block with 0x92b 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1036d5 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000141062s (0.000042993s flush + 0.000098069s kv commit) 2019-12-07T19:34:17.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6000> 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 kv_submitted 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6000 2019-12-07T19:34:17.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 finishing 2019-12-07T19:34:17.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6000 onodes 0x558c57e8ec80 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971eb8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c401b0 space 0x558c57971eb8 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a075f8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77dd0 space 0x558c57a075f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a07e48 in 0x558c53a59c00) _finish_write discard buffer(0x558c580cd9e0 space 0x558c57a07e48 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38a28 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40120 space 0x558c57c38a28 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49898 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40090 space 0x558c57f49898 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49978 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40000 space 0x558c57f49978 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49a58 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77ef0 space 0x558c57f49a58 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f49b38 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77e60 space 0x558c57f49b38 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b2e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40240 space 0x558c5809b2e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.639+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[a~1]] recovery_committed_to version 21'8 now ondisk 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 done 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6000 [] 2019-12-07T19:34:17.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.639+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 40/39 [PushReplyOp(1:db85c185:::smithi13140796-18:head)]) v3 -- 0x558c5763c600 con 0x558c54c5b400 2019-12-07T19:34:17.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x11000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beae18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c578ee3f0 space 0x558c57beae18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xf000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d75b38 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57a590e0 space 0x558c54d75b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xd000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d755f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c54e15950 space 0x558c54d755f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3b000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c577f2da8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c40d80 space 0x558c577f2da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3d000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c577f2e18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c40e10 space 0x558c577f2e18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3f000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c577f3048 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c40ea0 space 0x558c577f3048 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x41000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c577f3128 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c40f30 space 0x558c577f3128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x43000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c577f3208 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c40fc0 space 0x558c577f3208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x45000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c577f32e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41050 space 0x558c577f32e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x47000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c577f33c8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c410e0 space 0x558c577f33c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x49000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57880fd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41170 space 0x558c57880fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x13000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beaef8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41200 space 0x558c57beaef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x15000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beafd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41290 space 0x558c57beafd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x17000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb0b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41320 space 0x558c57beb0b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x19000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb198 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c413b0 space 0x558c57beb198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1b000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb278 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41440 space 0x558c57beb278 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1d000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb358 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c414d0 space 0x558c57beb358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1f000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb438 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41560 space 0x558c57beb438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x21000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb518 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c415f0 space 0x558c57beb518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb5f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41680 space 0x558c57beb5f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb6d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41710 space 0x558c57beb6d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb7b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c417a0 space 0x558c57beb7b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb898 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41830 space 0x558c57beb898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beb978 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c418c0 space 0x558c57beb978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57beba58 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41950 space 0x558c57beba58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebb38 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c419e0 space 0x558c57bebb38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x31000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 213 ==== MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=172, reqid=client.4540.0:316, at_version=40'13, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 121406+0+0 (crc 0 0 0) 0x558c57e7d000 con 0x558c54c64000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebc18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41a70 space 0x558c57bebc18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x33000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c578d3760 prio 127 cost 0 latency 0.000036 epoch 40 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=172, reqid=client.4540.0:316, at_version=40'13, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebcf8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41b00 space 0x558c57bebcf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=172, reqid=client.4540.0:316, at_version=40'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x35000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebdd8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41b90 space 0x558c57bebdd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x37000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebeb8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41c20 space 0x558c57bebeb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x39000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57bebf98 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41cb0 space 0x558c57bebf98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4b000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=172, reqid=client.4540.0:316, at_version=40'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcada8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41d40 space 0x558c57dcada8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4d000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 214 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=173, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c574a0280 con 0x558c54c64000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcae88 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=172, reqid=client.4540.0:316, at_version=40'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41dd0 space 0x558c57dcae88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57eff340 prio 127 cost 0 latency 0.000015 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=173, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4f000:0~2000]} 2019-12-07T19:34:17.640+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=173, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcaf68 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41e60 space 0x558c57dcaf68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x51000:1000~1000]} 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c578d3760 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=172, reqid=client.4540.0:316, at_version=40'13, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.7s4( v 40'12 (0'0,40'12] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.640+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcb048 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.640+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57c41ef0 space 0x558c57dcb048 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'12 (0'0,40'12] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=172, reqid=client.4540.0:316, at_version=40'13, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:17.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=173, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~52000 = 335872 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'12 (0'0,40'12] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] append_log log((0'0,40'12], crt=27'10) [40'13 (40'12) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:316 2019-12-07T19:34:16.630385+0000 0 ObjectCleanRegions clean_offsets: [0~770048,1245184~18446744073708306431], clean_omap: 1, new_object: 0] 2019-12-07T19:34:17.640+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=335872 r=335872 len=335872 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'13 (0'0,40'13] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'12 crt=27'10 lcod 40'12 active+remapped mbc={} ps=[a~1]] add_log_entry 40'13 (40'12) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:316 2019-12-07T19:34:16.630385+0000 0 ObjectCleanRegions clean_offsets: [0~770048,1245184~18446744073708306431], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'13 (0'0,40'13] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'12 crt=27'10 lcod 40'12 active+remapped mbc={} ps=[a~1]] append_log approx pg log length = 13 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'13 (0'0,40'13] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'12 crt=27'10 lcod 40'12 active+remapped mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:17.640+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 40'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.640+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.640+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:17.640+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:17.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740b800 seq 45 2019-12-07T19:34:17.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:17.640+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2f000~1d000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ec1285ab:::smithi13140796-19:head# 0x2f000~1d000 - have 0x2f000 (192512) bytes fadvise_flags 0x20 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x2f000~1d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740b800 15 blobs 2019-12-07T19:34:17.640+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=171, attrs_read=0)) v2 -- 0x558c57976c80 con 0x558c54c64000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c5740d4a0 finish 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1d000/1000,1d000,0 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x623000~1d000/1000,1d000,0 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x623000~1d000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5730 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c54e56460) get_ref 0x0~2000 Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5730 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c54e56460) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a5748 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c436c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43650 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c436c0) get_ref 0x0~2000 Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43650 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c436c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43668 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=173, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c43570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43500 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43570) get_ref 0x0~2000 Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43500 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43518 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c43490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43420 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43490) get_ref 0x0~2000 Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43420 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c57eff340 prio 127 cost 0 latency 0.000288 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=173, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43438 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c433b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43340 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c433b0) get_ref 0x0~2000 Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43340 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=173, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c433b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43358 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c432d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f04181700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43260 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~52000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c432d0) get_ref 0x0~2000 Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43260 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c432d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~52000 size 0x52000 (335872) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43278 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c431f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43180 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c431f0) get_ref 0x0~2000 Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43180 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c431f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43198 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c43110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c427e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c427e0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43110) get_ref 0x0~2000 Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c427e0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43110) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c427f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42700 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42770) get_ref 0x0~2000 Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42700 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42770) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42718 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42620 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42690) get_ref 0x0~2000 Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42620 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42638 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c425b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42540 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c425b0) get_ref 0x0~2000 Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42540 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c425b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42558 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c424d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c430a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c430a0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c424d0) get_ref 0x0~2000 Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c430a0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c424d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c430b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c43030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42fc0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43030) get_ref 0x0~2000 Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42fc0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43030) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42fd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42ee0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42f50) get_ref 0x0~2000 Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42ee0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42f50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42ef8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e00 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42e70) get_ref 0x1000~1000 Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e00 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42e70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~1000: 0x1000~1000 Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 sbid 0x0)) 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42e18 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4c000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2f000~1d000 = 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000040.00000000000000000013' <- 0000000040.00000000000000000013 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b800 cost 184938 (16 ios * 4000 + 120938 bytes) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b800 onodes 0x558c54e44000 shared_blobs 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update inline shard 835 bytes from 39 extents 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1414 (573 bytes onode + 2 bytes spanning blobs + 839 bytes inline extents) 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b800 allocated 0x[623000~1d000] released 0x[] 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 freelist allocate 0x623000~1d000 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 freelist _xor first_key 0x600000 last_key 0x600000 2019-12-07T19:34:17.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) need 0x1000~1000 cache has 0x[1000~1000] 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 prepare 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740b800 2019-12-07T19:34:17.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~52000 = 335872 2019-12-07T19:34:17.640+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=335872 r=335872 len=335872 2019-12-07T19:34:17.640+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.640+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:17.640+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:17.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:17.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:17.640+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.640+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=173, attrs_read=0)) v2 -- 0x558c57efa780 con 0x558c54c64000 2019-12-07T19:34:17.640+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c57eff340 finish 2019-12-07T19:34:17.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'13 (0'0,40'13] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'12 crt=27'10 lcod 40'12 active+remapped mbc={} ps=[a~1]] op_applied version 40'13 2019-12-07T19:34:17.640+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c578d3760 finish 2019-12-07T19:34:17.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 aio_wait 2019-12-07T19:34:17.641+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b800 2019-12-07T19:34:17.641+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 io_done 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2642 < min_flush_size 524288 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2665 < min_flush_size 524288 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2665 < min_flush_size 524288 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1036d5~a69 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1036d5 0x1036d5~a69 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd36d5 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6d5 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x13e and padding block with 0xec2 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x10413e 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000144034s (0.000051380s flush + 0.000092654s kv commit) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b800> 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 kv_submitted 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b800 2019-12-07T19:34:17.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 finishing 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b800 onodes 0x558c54e44000 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42558 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44480 space 0x558c57c42558 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42638 in 0x558c53a59880) _finish_write discard buffer(0x558c57c443f0 space 0x558c57c42638 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42718 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44360 space 0x558c57c42718 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c427f8 in 0x558c53a59880) _finish_write discard buffer(0x558c57c442d0 space 0x558c57c427f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42e18 in 0x558c53a59880) _finish_write discard buffer(0x558c57c446c0 space 0x558c57c42e18 0x1000~1000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42ef8 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44630 space 0x558c57c42ef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42fd8 in 0x558c53a59880) _finish_write discard buffer(0x558c57c445a0 space 0x558c57c42fd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c430b8 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44510 space 0x558c57c430b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 40/39 ECSubWriteReply(tid=172, last_complete=40'13, committed=1, applied=1)) v2 -- 0x558c55250e00 con 0x558c54c64000 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43198 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44240 space 0x558c57c43198 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43278 in 0x558c53a59880) _finish_write discard buffer(0x558c57c441b0 space 0x558c57c43278 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43358 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44120 space 0x558c57c43358 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43438 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44090 space 0x558c57c43438 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43518 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44000 space 0x558c57c43518 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43668 in 0x558c53a59880) _finish_write discard buffer(0x558c57a08c60 space 0x558c57c43668 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5748 in 0x558c53a59880) _finish_write discard buffer(0x558c54dbaea0 space 0x558c580a5748 0x0~2000 writing nocache) 2019-12-07T19:34:17.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b800 done 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:17.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b800 [] 2019-12-07T19:34:17.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 215 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=174, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c5793b900 con 0x558c54c64000 2019-12-07T19:34:17.642+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000022 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=174, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.642+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=174, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=174, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=174, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.642+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000063 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=174, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.642+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=174, to_read={1:9da56077:::smithi13140796-13:head=0,335872,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.642+0000 7f8f08189700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.642+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~52000 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~52000 size 0x52000 (335872) 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.642+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) need 0x1000~1000 cache has 0x[1000~1000] 2019-12-07T19:34:17.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x0~52000 = 335872 2019-12-07T19:34:17.642+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=335872 r=335872 len=335872 2019-12-07T19:34:17.642+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.642+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:17.642+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:17.642+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:17.642+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:17.642+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.642+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=174, attrs_read=0)) v2 -- 0x558c57572f00 con 0x558c54c64000 2019-12-07T19:34:17.642+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:17.642+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 216 ==== MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=175, reqid=client.4540.0:317, at_version=40'14, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 76350+0+0 (crc 0 0 0) 0x558c579e3000 con 0x558c54c64000 2019-12-07T19:34:17.643+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3e160 prio 127 cost 0 latency 0.000029 epoch 40 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=175, reqid=client.4540.0:317, at_version=40'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:17.643+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=175, reqid=client.4540.0:317, at_version=40'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.643+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.643+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=175, reqid=client.4540.0:317, at_version=40'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.643+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.643+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=175, reqid=client.4540.0:317, at_version=40'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.643+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3e160 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=175, reqid=client.4540.0:317, at_version=40'14, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.7s4( v 40'13 (0'0,40'13] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 152 ==== MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=64, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c575bac80 con 0x558c54c5b400 2019-12-07T19:34:17.643+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3e420 prio 3 cost 0 latency 0.000011 epoch 40 MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=64, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.643+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'13 (0'0,40'13] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=175, reqid=client.4540.0:317, at_version=40'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:17.643+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=64, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'13 (0'0,40'13] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] append_log log((0'0,40'13], crt=27'10) [40'14 (40'13) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:317 2019-12-07T19:34:16.632030+0000 0 ObjectCleanRegions clean_offsets: [0~1245184,1540096~18446744073708011519], clean_omap: 1, new_object: 0] 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'14 (0'0,40'14] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'13 crt=27'10 lcod 40'13 active+remapped mbc={} ps=[a~1]] add_log_entry 40'14 (40'13) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:317 2019-12-07T19:34:16.632030+0000 0 ObjectCleanRegions clean_offsets: [0~1245184,1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'14 (0'0,40'14] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'13 crt=27'10 lcod 40'13 active+remapped mbc={} ps=[a~1]] append_log approx pg log length = 14 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'14 (0'0,40'14] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'13 crt=27'10 lcod 40'13 active+remapped mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:17.644+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 40'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=64, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b94000 seq 46 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=64, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c54d9b000 2019-12-07T19:34:17.644+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:17.644+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:17.644+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4c000~12000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ec1285ab:::smithi13140796-19:head# 0x4c000~12000 - have 0x4c000 (311296) bytes fadvise_flags 0x20 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x4c000~12000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.644+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c57f3e420 prio 3 cost 0 latency 0.000107 MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=64, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57b94000 9 blobs 2019-12-07T19:34:17.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=64, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 25 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~40000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x33000 (208896) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac000 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57566d20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e424d0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e9a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1ab0 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdc70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fdc00 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdb90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2930 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b29a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2a10 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2a80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2af0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2b60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2bd0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2c40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2cb0 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2d90 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2e70 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x12000/1000,12000,0 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x640000~12000/1000,12000,0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2f50 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x640000~12000] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c435e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3030 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c435e0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3110 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b31f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3260 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57897d50) get_ref 0x0~2000 Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c435e0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57897d50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b32d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3340 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c435f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b33b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3490 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3570 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b35e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c43dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43e30 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3650 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b36c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43dc0) get_ref 0x0~2000 Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43e30 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3730 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3810 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43e48 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b38f0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c43f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b39d0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f80 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43f10) get_ref 0x0~2000 Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f80 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3ab0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3b20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43f10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3b90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579b3c00 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43f98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e424d0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e9a0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b96e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e70 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96e00) get_ref 0x0~2000 Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e70 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1ab0 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdc70 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96e00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b96e88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fdc00 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdb90 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ac000 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57566d20 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b96ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96f50 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2930 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b29a0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96ee0) get_ref 0x0~2000 Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96f50 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2a10 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2a80 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b96f68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2af0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2b60 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b96fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97030 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2bd0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2c40 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96fc0) get_ref 0x0~2000 Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97030 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2cb0 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b97048 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2d90 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b970a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97110 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2e70 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b970a0) get_ref 0x0~2000 Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97110 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b970a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2f50 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2fc0 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b97128 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3030 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b971f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3110 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3180 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b971f0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97180) get_ref 0x0~2000 Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b971f0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97180) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b31f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3260 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b97208 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b32d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3340 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b972d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b972d0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b33b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3420 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97260) get_ref 0x0~2000 Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b972d0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97260) get_ref init 0x2000, 1000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3490 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3500 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~2000: 0x0~2000 Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b972e8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3570 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b35e0 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5e000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3650 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b36c0 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4c000~12000 = 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3730 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3810 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b38f0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b39d0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000040.00000000000000000014' <- 0000000040.00000000000000000014 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3ab0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3b20 sbid 0x0)) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3b90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579b3c00 sbid 0x0)) need {<0x0, 0x1000> : [0x32000:0~1000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32000: 0x0 reading 0x0~1000 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94000 cost 115882 (10 ios * 4000 + 75882 bytes) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94000 onodes 0x558c54e44000 shared_blobs 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update inline shard 1007 bytes from 48 extents 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1586 (573 bytes onode + 2 bytes spanning blobs + 1011 bytes inline extents) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94000 allocated 0x[640000~12000] released 0x[] 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 freelist allocate 0x640000~12000 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 freelist _xor first_key 0x600000 last_key 0x600000 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 prepare 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57b94000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'14 (0'0,40'14] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'13 crt=27'10 lcod 40'13 active+remapped mbc={} ps=[a~1]] op_applied version 40'14 2019-12-07T19:34:17.644+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 aio_wait 2019-12-07T19:34:17.644+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94000 2019-12-07T19:34:17.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 io_done 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2814 < min_flush_size 524288 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2837 < min_flush_size 524288 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2837 < min_flush_size 524288 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x10413e~b15 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x10413e 0x10413e~b15 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd413e 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x13e 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc53 and padding block with 0x3ad 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x104c53 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000128117s (0.000036858s flush + 0.000091259s kv commit) 2019-12-07T19:34:17.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94000> 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 kv_submitted 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94000 2019-12-07T19:34:17.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 finishing 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94000 onodes 0x558c54e44000 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96e88 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44900 space 0x558c57b96e88 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96f68 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44990 space 0x558c57b96f68 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97048 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44a20 space 0x558c57b97048 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97128 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44ab0 space 0x558c57b97128 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97208 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44b40 space 0x558c57b97208 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b972e8 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44bd0 space 0x558c57b972e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c435f8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f77050 space 0x558c57c435f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43e48 in 0x558c53a59880) _finish_write discard buffer(0x558c57c447e0 space 0x558c57c43e48 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43f98 in 0x558c53a59880) _finish_write discard buffer(0x558c57c44870 space 0x558c57c43f98 0x0~2000 writing nocache) 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94000 done 2019-12-07T19:34:17.644+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 40/39 ECSubWriteReply(tid=175, last_complete=40'14, committed=1, applied=1)) v2 -- 0x558c5763c600 con 0x558c54c64000 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:17.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94000 [] 2019-12-07T19:34:17.644+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.644+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e424d0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e9a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f775f0 space 0x558c5757e9b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1ab0 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdc70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574fdc88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77680 space 0x558c574fdc88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fdc00 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdb90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574fdba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77710 space 0x558c574fdba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ac000 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57566d20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57566d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f777a0 space 0x558c57566d38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2930 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b29a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b29b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77830 space 0x558c579b29b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2a10 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2a80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2a98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f778c0 space 0x558c579b2a98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2af0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2b60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77950 space 0x558c579b2b78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2bd0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2c40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f779e0 space 0x558c579b2c58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2cb0 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77a70 space 0x558c579b2d38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2d90 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77b00 space 0x558c579b2e18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2e70 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77b90 space 0x558c579b2ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2f50 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77c20 space 0x558c579b2fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3030 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77cb0 space 0x558c579b30b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3110 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f77d40 space 0x558c579b3198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b31f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3260 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3278 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c45290 space 0x558c579b3278 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b32d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3340 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3358 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c45320 space 0x558c579b3358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b33b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3438 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c453b0 space 0x558c579b3438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3490 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3518 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c45440 space 0x558c579b3518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3570 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b35e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b35f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c454d0 space 0x558c579b35f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3650 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b36c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b36d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c45560 space 0x558c579b36d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3730 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b37b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c455f0 space 0x558c579b37b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3810 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c45680 space 0x558c579b3898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b38f0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c45710 space 0x558c579b3978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b39d0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c457a0 space 0x558c579b3a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3ab0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3b20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c45830 space 0x558c579b3b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3b90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579b3c00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x32000:0~1000]} 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57c458c0 space 0x558c579b3c18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~40000 = 208896 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=208896 len=208896 2019-12-07T19:34:17.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:17.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:17.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:17.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 208896 2019-12-07T19:34:17.645+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key 2019-12-07T19:34:17.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key = 0 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:17.645+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 40/39 ECSubReadReply(tid=64, attrs_read=0)) v2 -- 0x558c57e8ef00 con 0x558c54c5b400 2019-12-07T19:34:17.645+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c57f3e420 finish 2019-12-07T19:34:17.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 153 ==== MOSDPGPush(1.bs1 40/39 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 209958+0+0 (unknown 0 0 0) 0x558c55270800 con 0x558c54c5b400 2019-12-07T19:34:17.648+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3e9a0 prio 127 cost 209896 latency 0.000025 epoch 40 MOSDPGPush(1.bs1 40/39 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.648+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e40) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e40) queued 2019-12-07T19:34:17.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.648+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e40) pg 0x558c575ef000 2019-12-07T19:34:17.648+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57f3e9a0 prio 127 cost 209896 latency 0.000065 MOSDPGPush(1.bs1 40/39 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[a~1]] 2019-12-07T19:34:17.648+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.bs1 40/39 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.648+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[a~1]] on_local_recover: 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:17.648+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[a~1]] got missing 1:dce63cc3:::smithi13140796-50:head v 21'9 2019-12-07T19:34:17.648+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[a~1]] last_complete now 21'9 log.complete_to 21'10 2019-12-07T19:34:17.648+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c574fe900 seq 5 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:head# key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:head# key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.648+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1186 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:34:17.648+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~33000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574fe900 26 blobs 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.648+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x33000/1000,33000,0 2019-12-07T19:34:17.648+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x652000~33000/1000,33000,0 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x652000~33000] 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad650 blob([0x652000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c576ad650) get_ref 0x0~2000 Blob(0x558c576ad650 blob([0x652000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c576ad650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c576ad650 blob([0x652000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57819dd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897c70 blob([0x654000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279810 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c57897c70) get_ref 0x0~2000 Blob(0x558c57897c70 blob([0x654000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279810 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c57897c70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57897c70 blob([0x654000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55279828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fd960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a073b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fd960 blob([0x656000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a073b0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c574fd960) get_ref 0x0~2000 Blob(0x558c574fd960 blob([0x656000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a073b0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c574fd960) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c574fd960 blob([0x656000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57a073c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fd570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fd500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fd570 blob([0x658000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd500 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c574fd570) get_ref 0x0~2000 Blob(0x558c574fd570 blob([0x658000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd500 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c574fd570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c574fd570 blob([0x658000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fd518 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3570 blob([0x65a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881810 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c577f3570) get_ref 0x0~2000 Blob(0x558c577f3570 blob([0x65a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881810 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c577f3570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c577f3570 blob([0x65a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57881828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efc070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1810 blob([0x65c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc070 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c578f1810) get_ref 0x0~2000 Blob(0x558c578f1810 blob([0x65c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc070 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c578f1810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c578f1810 blob([0x65c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57efc088 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54677810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e63340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54677810 blob([0x65e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63340 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c54677810) get_ref 0x0~2000 Blob(0x558c54677810 blob([0x65e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63340 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c54677810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c54677810 blob([0x65e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57e63358 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fd6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2fc0 blob([0x660000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd6c0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c574d2fc0) get_ref 0x0~2000 Blob(0x558c574d2fc0 blob([0x660000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd6c0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c574d2fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([0x660000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fd6d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552790a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742af50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552790a0 blob([0x662000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742af50 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c552790a0) get_ref 0x0~2000 Blob(0x558c552790a0 blob([0x662000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742af50 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c552790a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c552790a0 blob([0x662000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5742af68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57567110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57567110 blob([0x664000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb20 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c57567110) get_ref 0x0~2000 Blob(0x558c57567110 blob([0x664000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb20 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c57567110) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57567110 blob([0x664000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5742bb38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f3490 blob([0x666000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3880 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c577f3490) get_ref 0x0~2000 Blob(0x558c577f3490 blob([0x666000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3880 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c577f3490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c577f3490 blob([0x666000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c577f3898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97f80 blob([0x668000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97f10 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c57b97f80) get_ref 0x0~2000 Blob(0x558c57b97f80 blob([0x668000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97f10 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c57b97f80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57b97f80 blob([0x668000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57b97f28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea000 blob([0x66a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea070 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea000) get_ref 0x0~2000 Blob(0x558c581ea000 blob([0x66a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea070 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c581ea000 blob([0x66a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea088 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea0e0 blob([0x66c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea150 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea0e0) get_ref 0x0~2000 Blob(0x558c581ea0e0 blob([0x66c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea150 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c581ea0e0 blob([0x66c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea168 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea1c0 blob([0x66e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea230 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea1c0) get_ref 0x0~2000 Blob(0x558c581ea1c0 blob([0x66e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea230 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c581ea1c0 blob([0x66e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea248 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea2a0 blob([0x670000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea310 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea2a0) get_ref 0x0~2000 Blob(0x558c581ea2a0 blob([0x670000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea310 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c581ea2a0 blob([0x670000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea380 blob([0x672000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea3f0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea380) get_ref 0x0~2000 Blob(0x558c581ea380 blob([0x672000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea3f0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea380) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c581ea380 blob([0x672000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea408 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea460 blob([0x674000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea4d0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea460) get_ref 0x0~2000 Blob(0x558c581ea460 blob([0x674000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea4d0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea460) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c581ea460 blob([0x674000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea4e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea540 blob([0x676000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea5b0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea540) get_ref 0x0~2000 Blob(0x558c581ea540 blob([0x676000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea5b0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea540) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c581ea540 blob([0x676000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea5c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea620 blob([0x678000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea690 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea620) get_ref 0x0~2000 Blob(0x558c581ea620 blob([0x678000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea690 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea620) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c581ea620 blob([0x678000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea6a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea700 blob([0x67a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea770 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea700) get_ref 0x0~2000 Blob(0x558c581ea700 blob([0x67a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea770 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea700) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c581ea700 blob([0x67a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea7e0 blob([0x67c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea850 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea7e0) get_ref 0x0~2000 Blob(0x558c581ea7e0 blob([0x67c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea850 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c581ea7e0 blob([0x67c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ea930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea8c0 blob([0x67e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea930 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea8c0) get_ref 0x0~2000 Blob(0x558c581ea8c0 blob([0x67e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea930 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c581ea8c0 blob([0x67e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ea948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581eaa10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea9a0 blob([0x680000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaa10 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea9a0) get_ref 0x0~2000 Blob(0x558c581ea9a0 blob([0x680000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaa10 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581ea9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c581ea9a0 blob([0x680000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581eaa28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581eaa80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581eaaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581eaa80 blob([0x682000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaaf0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581eaa80) get_ref 0x0~2000 Blob(0x558c581eaa80 blob([0x682000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaaf0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581eaa80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c581eaa80 blob([0x682000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581eab08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581eab60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581eabd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581eab60 blob([0x684000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eabd0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581eab60) get_ref 0x0~1000 Blob(0x558c581eab60 blob([0x684000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eabd0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.blob(0x558c581eab60) get_ref init 0x1000, 1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~1000: 0x0~1000 Blob(0x558c581eab60 blob([0x684000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 sbid 0x0)) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581eabe8 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x33000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 = 0 2019-12-07T19:34:17.648+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 4 keys 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 4 keys = 0 2019-12-07T19:34:17.648+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000499'.missing/0000000000000001.B376C33C.head.smithi13140796-50..' <- missing/0000000000000001.B376C33C.head.smithi13140796-50.. 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.648+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe900 cost 319133 (27 ios * 4000 + 211133 bytes) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe900 onodes 0x558c57f34c80 shared_blobs 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:17.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 217 ==== MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=176, reqid=client.4540.0:318, at_version=40'15, trim_to=0'0, roll_forward_to=40'12)) v2 ==== 2588+0+0 (crc 0 0 0) 0x558c57e7a000 con 0x558c54c64000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) update inline shard 494 bytes from 26 extents 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 985 (485 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:17.648+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe900 allocated 0x[652000~33000] released 0x[] 2019-12-07T19:34:17.648+0000 7f8f07187700 10 freelist allocate 0x652000~33000 2019-12-07T19:34:17.648+0000 7f8f07187700 20 freelist _xor first_key 0x600000 last_key 0x680000 2019-12-07T19:34:17.648+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3eb00 prio 127 cost 0 latency 0.000035 epoch 40 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=176, reqid=client.4540.0:318, at_version=40'15, trim_to=0'0, roll_forward_to=40'12)) v2 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 prepare 2019-12-07T19:34:17.648+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c574fe900 2019-12-07T19:34:17.648+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=176, reqid=client.4540.0:318, at_version=40'15, trim_to=0'0, roll_forward_to=40'12)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=176, reqid=client.4540.0:318, at_version=40'15, trim_to=0'0, roll_forward_to=40'12)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.648+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=176, reqid=client.4540.0:318, at_version=40'15, trim_to=0'0, roll_forward_to=40'12)) v2) prio 127 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.648+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3eb00 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=176, reqid=client.4540.0:318, at_version=40'15, trim_to=0'0, roll_forward_to=40'12)) v2 pg pg[1.7s4( v 40'14 (0'0,40'14] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'14 (0'0,40'14] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=176, reqid=client.4540.0:318, at_version=40'15, trim_to=0'0, roll_forward_to=40'12)) v2 2019-12-07T19:34:17.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'14 (0'0,40'14] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[a~1]] append_log log((0'0,40'14], crt=27'10) [40'15 (40'14) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:318 2019-12-07T19:34:16.632044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:17.648+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'14 crt=27'10 lcod 40'14 active+remapped mbc={} ps=[a~1]] add_log_entry 40'15 (40'14) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:318 2019-12-07T19:34:16.632044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.648+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'14 crt=40'12 lcod 40'14 active+remapped mbc={} ps=[a~1]] rollforward: entry=40'11 (21'4) clone 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [10,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'14 crt=40'12 lcod 40'14 active+remapped mbc={} ps=[a~1]] rollforward: entry=40'12 (21'4) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:315 2019-12-07T19:34:16.627511+0000 0 ObjectCleanRegions clean_offsets: [0~311296,770048~18446744073708781567], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'14 crt=40'12 lcod 40'14 active+remapped mbc={} ps=[a~1]] append_log approx pg log length = 15 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'14 crt=40'12 lcod 40'14 active+remapped mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:17.649+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 40'15, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b94300 seq 47 2019-12-07T19:34:17.649+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 3 keys 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 3 keys = 0 2019-12-07T19:34:17.649+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:17.649+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000040.00000000000000000015' <- 0000000040.00000000000000000015 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94300 cost 7028 (1 ios * 4000 + 3028 bytes) 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94300 onodes 0x558c54e44000 shared_blobs 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1586 (573 bytes onode + 2 bytes spanning blobs + 1011 bytes inline extents) 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 prepare 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94300 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 io_done 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 lua=40'14 crt=40'12 lcod 40'14 active+remapped mbc={} ps=[a~1]] op_applied version 40'15 2019-12-07T19:34:17.649+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3eb00 finish 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3682 < min_flush_size 524288 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3705 < min_flush_size 524288 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3705 < min_flush_size 524288 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x104c53~e79 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x104c53 0x104c53~e79 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd4c53 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc53 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xacc and padding block with 0x534 2019-12-07T19:34:17.649+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57f3e9a0 finish 2019-12-07T19:34:17.649+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x105acc 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.649+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000162365s (0.000000106s flush + 0.000162259s kv commit) 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94300> 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 kv_submitted 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94300 2019-12-07T19:34:17.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 finishing 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94300 onodes 0x558c54e44000 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94300 done 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:17.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94300 [] 2019-12-07T19:34:17.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.649+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.649+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 40/39 ECSubWriteReply(tid=176, last_complete=40'15, committed=1, applied=1)) v2 -- 0x558c54c1fe00 con 0x558c54c64000 2019-12-07T19:34:17.649+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.649+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 aio_wait 2019-12-07T19:34:17.649+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe900 2019-12-07T19:34:17.649+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 io_done 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2337 < min_flush_size 524288 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34c80 had 1 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2360 < min_flush_size 524288 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2360 < min_flush_size 524288 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x105acc~938 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 154 ==== MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=65, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558c57efa780 con 0x558c54c5b400 2019-12-07T19:34:17.649+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x105acc 0x105acc~938 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.649+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3ec60 prio 127 cost 0 latency 0.000018 epoch 40 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=65, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd5acc 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xacc 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.649+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x404 and padding block with 0xbfc 2019-12-07T19:34:17.649+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=65, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=65, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.649+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=65, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c5513e000 2019-12-07T19:34:17.649+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57f3ec60 prio 127 cost 0 latency 0.000074 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=65, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=65, to_read={1:cd22b62d:::smithi13140796-27:head=262144,262144,0}, subchunks={1:cd22b62d:::smithi13140796-27:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.649+0000 7f8f07187700 25 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e422a0 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42310 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x40e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x41e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0fc0 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e421c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x42e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x43e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42230 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42460 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x44e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x45e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e423f0 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42380 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x46e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x47e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48600~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48600~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48600~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48a00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48a00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48e00~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42690 blob([0x15c000~1000,0x25b000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e420e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x48e00~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49200~200 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49200~200 size 0x54000 (344064) 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.649+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.649+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49200~200 = 512 2019-12-07T19:34:17.649+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49a00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x49e00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4a600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4aa00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4aa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4aa00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ae00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42150 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42620 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ae00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4b200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4b600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4b600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ba00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x106404 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.650+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.650+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.650+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ba00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4be00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4be00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4be00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000633703s (0.000036090s flush + 0.000597613s kv commit) 2019-12-07T19:34:17.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4c600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ca00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ca00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ce00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42000 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42070 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ce00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4d200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4d600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4d600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4da00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4da00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4da00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4de00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4de00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4de00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4e600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ea00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ea00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ee00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43a40 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43f80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4ee00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4f200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4f600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4f600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fa00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4fa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fa00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fe00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4fe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x4fe00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50a00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c70 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43b90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x50e00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51a00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x51e00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52a00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43c00 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e439d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x52e00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53200~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53600~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53a00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43b20 blob([!~1000,0x266000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ab0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x53e00~200 = 512 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x54e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x55e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x56e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x57e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x58e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x59e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5a600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5aa00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5aa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5aa00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ae00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ae00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5b200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5b600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5b600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ba00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ba00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5be00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5be00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5be00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5c600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ca00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ca00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ce00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ce00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5d200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5d600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5d600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5da00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5da00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5da00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5de00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5de00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5de00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5e600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ea00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ea00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ee00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5ee00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5f200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5f600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5f600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fa00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5fa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fa00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fe00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5fe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x5fe00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x60e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x61e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x62e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x63e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x64e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x65e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x66e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x67e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x68e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x69e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6a600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6aa00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6aa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6aa00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ae00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ae00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6b200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6b600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6b600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ba00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ba00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6be00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6be00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6be00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6c600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ca00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ca00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ce00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ce00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6d200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6d600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6d600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6da00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6da00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6da00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6de00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6de00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6de00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6e600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ea00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ea00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ee00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6ee00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6f200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6f600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6f600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fa00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6fa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fa00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fe00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6fe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x6fe00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x70e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x71e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x72e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x73e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74e00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x74e00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75200~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75200~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75200~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75600~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75600~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75600~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75a00~200 2019-12-07T19:34:17.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75a00~200 = 0 2019-12-07T19:34:17.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75e00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x75e00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76a00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76a00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76e00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x76e00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77a00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77a00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77e00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x77e00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78a00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78a00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78e00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x78e00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79a00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79a00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79a00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79e00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79e00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x79e00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7a600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7aa00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7aa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7aa00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ae00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ae00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ae00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7b200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7b600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7b600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ba00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ba00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ba00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7be00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7be00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7be00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7c600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ca00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ca00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ca00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ce00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ce00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ce00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7d200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7d600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7d600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7da00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7da00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7da00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7de00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7de00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7de00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7e600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ea00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ea00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ea00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ee00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ee00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7ee00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f200~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7f200~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f200~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f600~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7f600~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7f600~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fa00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7fa00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fa00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fe00~200 2019-12-07T19:34:17.651+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7fe00~200 size 0x54000 (344064) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 0x7fe00~200 = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=40960 2019-12-07T19:34:17.651+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:17.651+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:17.651+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 344064 2019-12-07T19:34:17.651+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key 2019-12-07T19:34:17.651+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:cd22b62d:::smithi13140796-27:head# hinfo_key = 0 2019-12-07T19:34:17.651+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.3s0 40/39 ECSubReadReply(tid=65, attrs_read=0)) v2 -- 0x558c57f34f00 con 0x558c54c5b400 2019-12-07T19:34:17.651+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57f3ec60 finish 2019-12-07T19:34:17.651+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe900> 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.651+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 kv_submitted 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe900 2019-12-07T19:34:17.651+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 finishing 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe900 onodes 0x558c57f34c80 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279828 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40480 space 0x558c55279828 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742af68 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40870 space 0x558c5742af68 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bb38 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40900 space 0x558c5742bb38 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fd518 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c405a0 space 0x558c574fd518 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fd6d8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c407e0 space 0x558c574fd6d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3898 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40990 space 0x558c577f3898 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819dd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c403f0 space 0x558c57819dd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881828 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40630 space 0x558c57881828 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a073c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40510 space 0x558c57a073c8 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[a~1]] recovery_committed_to version 21'9 now ondisk 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97f28 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40a20 space 0x558c57b97f28 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e63358 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40750 space 0x558c57e63358 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efc088 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c406c0 space 0x558c57efc088 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea088 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40ab0 space 0x558c581ea088 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea168 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40b40 space 0x558c581ea168 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea248 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40bd0 space 0x558c581ea248 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea328 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40c60 space 0x558c581ea328 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 40/39 [PushReplyOp(1:dce63cc3:::smithi13140796-50:head)]) v3 -- 0x558c54dfbe00 con 0x558c54c5b400 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea408 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40cf0 space 0x558c581ea408 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea4e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45c20 space 0x558c581ea4e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea5c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45cb0 space 0x558c581ea5c8 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea6a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45d40 space 0x558c581ea6a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea788 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45dd0 space 0x558c581ea788 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea868 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45e60 space 0x558c581ea868 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ea948 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45ef0 space 0x558c581ea948 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581eaa28 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4000 space 0x558c581eaa28 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581eab08 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4090 space 0x558c581eab08 0x0~2000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581eabe8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4120 space 0x558c581eabe8 0x0~1000 writing nocache) 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe900 done 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.651+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe900 [] 2019-12-07T19:34:17.651+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.651+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.652+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 155 ==== MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=66, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57f34f00 con 0x558c54c5b400 2019-12-07T19:34:17.652+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3f600 prio 3 cost 0 latency 0.000015 epoch 40 MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=66, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.652+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=66, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=66, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=66, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c54d9b000 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c57f3f600 prio 3 cost 0 latency 0.000048 MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=66, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.bs3 40/39 ECSubRead(tid=66, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.652+0000 7f8f0317f700 25 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.652+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~40000 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x32000 (204800) 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~40000 = 204800 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=204800 len=204800 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:17.652+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:17.652+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 40/39 ECSubReadReply(tid=66, attrs_read=0)) v2 -- 0x558c575bac80 con 0x558c54c5b400 2019-12-07T19:34:17.652+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c57f3f600 finish 2019-12-07T19:34:17.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.654+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 668982 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.655+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 218 ==== MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=177, reqid=client.4540.0:318, at_version=0'0, trim_to=0'0, roll_forward_to=40'15)) v2 ==== 1028+0+0 (crc 0 0 0) 0x558c579e0000 con 0x558c54c64000 2019-12-07T19:34:17.655+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3f760 prio 127 cost 0 latency 0.000025 epoch 40 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=177, reqid=client.4540.0:318, at_version=0'0, trim_to=0'0, roll_forward_to=40'15)) v2 2019-12-07T19:34:17.655+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=177, reqid=client.4540.0:318, at_version=0'0, trim_to=0'0, roll_forward_to=40'15)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.655+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.655+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=177, reqid=client.4540.0:318, at_version=0'0, trim_to=0'0, roll_forward_to=40'15)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.655+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.655+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=177, reqid=client.4540.0:318, at_version=0'0, trim_to=0'0, roll_forward_to=40'15)) v2) prio 127 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.655+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3f760 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=177, reqid=client.4540.0:318, at_version=0'0, trim_to=0'0, roll_forward_to=40'15)) v2 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'12 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.655+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'12 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 40/39 ECSubWrite(tid=177, reqid=client.4540.0:318, at_version=0'0, trim_to=0'0, roll_forward_to=40'15)) v2 2019-12-07T19:34:17.655+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'12 active+remapped mbc={} ps=[a~1]] append_log log((0'0,40'15], crt=40'12) [] 2019-12-07T19:34:17.655+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] rollforward: entry=40'13 (40'12) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:316 2019-12-07T19:34:16.630385+0000 0 ObjectCleanRegions clean_offsets: [0~770048,1245184~18446744073708306431], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.655+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] rollforward: entry=40'14 (40'13) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:317 2019-12-07T19:34:16.632030+0000 0 ObjectCleanRegions clean_offsets: [0~1245184,1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.655+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] rollforward: entry=40'15 (40'14) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:318 2019-12-07T19:34:16.632044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:17.655+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] append_log approx pg log length = 15 2019-12-07T19:34:17.655+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:17.655+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:17.655+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b94600 seq 48 2019-12-07T19:34:17.656+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94600 cost 5421 (1 ios * 4000 + 1421 bytes) 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94600 onodes shared_blobs 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94600 allocated 0x[] released 0x[] 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 prepare 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94600 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 io_done 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3f760 finish 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1232 < min_flush_size 524288 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:17.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 156 ==== MOSDPGPush(1.bs1 40/39 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 205862+0+0 (unknown 0 0 0) 0x558c55250e00 con 0x558c54c5b400 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.656+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3f8c0 prio 127 cost 205800 latency 0.000042 epoch 40 MOSDPGPush(1.bs1 40/39 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x106404~4e7 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x106404 0x106404~4e7 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.656+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e40) 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd6404 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x404 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8eb and padding block with 0x715 2019-12-07T19:34:17.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e40) queued 2019-12-07T19:34:17.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.656+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 40/39 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e40) pg 0x558c575ef000 2019-12-07T19:34:17.656+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 prio 127 cost 205800 latency 0.000101 MOSDPGPush(1.bs1 40/39 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.656+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.bs1 40/39 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1068eb 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.656+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[a~1]] on_local_recover: 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.656+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[a~1]] got missing 1:d313ddd1:::smithi13140796-49:head v 21'10 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000190389s (0.000000135s flush + 0.000190254s kv commit) 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94600> 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 kv_submitted 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94600 2019-12-07T19:34:17.656+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[a~1]] last_complete now 21'11 log.complete_to 21'12 2019-12-07T19:34:17.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 finishing 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94600 onodes 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94600 done 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:17.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94600 [] 2019-12-07T19:34:17.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.656+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.656+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 40/39 ECSubWriteReply(tid=177, last_complete=40'15, committed=1, applied=1)) v2 -- 0x558c57612400 con 0x558c54c64000 2019-12-07T19:34:17.656+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c574ff200 seq 6 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:d313ddd1:::smithi13140796-49:head# key 0x818000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:d313ddd1:::smithi13140796-49:head# key 0x818000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1187 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# = 0 2019-12-07T19:34:17.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~32000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c574ff200 25 blobs 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 219 ==== MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=178, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c575bac80 con 0x558c54c64000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.656+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x32000/1000,32000,0 2019-12-07T19:34:17.656+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3fb80 prio 127 cost 0 latency 0.000018 epoch 40 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=178, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.656+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x685000~32000/1000,32000,0 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x685000~32000] 2019-12-07T19:34:17.656+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=178, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42150 blob([0x685000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2850 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57c42150) get_ref 0x0~2000 Blob(0x558c57c42150 blob([0x685000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2850 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57c42150) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57c42150 blob([0x685000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2850 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=178, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757f7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=178, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42000 blob([0x687000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f7a0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57c42000) get_ref 0x0~2000 Blob(0x558c57c42000 blob([0x687000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757f7a0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57c42000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57c42000 blob([0x687000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757f7a0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757f7b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3fb80 prio 127 cost 0 latency 0.000052 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=178, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581eb7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819ea0 blob([0x689000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eb7a0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=178, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57819ea0) get_ref 0x0~2000 Blob(0x558c57819ea0 blob([0x689000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eb7a0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57819ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57819ea0 blob([0x689000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eb7a0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 25 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581eb7b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~1000 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x5e000 (385024) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581eb810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581eb880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581eb810 blob([0x68b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eb880 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581eb810) get_ref 0x0~2000 Blob(0x558c581eb810 blob([0x68b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eb880 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581eb810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c581eb810 blob([0x68b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eb880 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581eb898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~1000 = 4096 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581eb8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581eb960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581eb8f0 blob([0x68d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eb960 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581eb8f0) get_ref 0x0~2000 Blob(0x558c581eb8f0 blob([0x68d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eb960 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581eb8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c581eb8f0 blob([0x68d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eb960 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581eb978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581eb9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581eba40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581eb9d0 blob([0x68f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eba40 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 385024 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581eb9d0) get_ref 0x0~2000 Blob(0x558c581eb9d0 blob([0x68f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eba40 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581eb9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c581eb9d0 blob([0x68f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eba40 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581eba58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ebab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ebb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 40/39 ECSubReadReply(tid=178, attrs_read=0)) v2 -- 0x558c5793bb80 con 0x558c54c64000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ebab0 blob([0x691000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebb20 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebab0) get_ref 0x0~2000 Blob(0x558c581ebab0 blob([0x691000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebb20 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c581ebab0 blob([0x691000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ebb20 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3fb80 finish 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ebb38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ebb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ebb90 blob([0x693000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8770 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebb90) get_ref 0x0~2000 Blob(0x558c581ebb90 blob([0x693000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8770 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebb90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c581ebb90 blob([0x693000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8770 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf8788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a07a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de850 blob([0x695000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07a40 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c578de850) get_ref 0x0~2000 Blob(0x558c578de850 blob([0x695000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07a40 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c578de850) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c578de850 blob([0x695000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07a40 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57a07a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552787e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552787e0 blob([0x697000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2150 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c552787e0) get_ref 0x0~2000 Blob(0x558c552787e0 blob([0x697000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2150 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c552787e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c552787e0 blob([0x697000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2150 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c577f2168 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2620 blob([0x699000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2770 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c577f2620) get_ref 0x0~2000 Blob(0x558c577f2620 blob([0x699000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2770 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c577f2620) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c577f2620 blob([0x699000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2770 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e2788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f25b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f25b0 blob([0x69b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2690 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c577f25b0) get_ref 0x0~2000 Blob(0x558c577f25b0 blob([0x69b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2690 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c577f25b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c577f25b0 blob([0x69b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2690 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e26a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f2540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e25b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f2540 blob([0x69d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e25b0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c577f2540) get_ref 0x0~2000 Blob(0x558c577f2540 blob([0x69d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e25b0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c577f2540) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c577f2540 blob([0x69d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e25b0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574e25c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97a40 blob([0x69f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97ab0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97a40) get_ref 0x0~2000 Blob(0x558c57b97a40 blob([0x69f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97ab0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97a40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57b97a40 blob([0x69f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97ab0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57b97ac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97b20 blob([0x6a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97b90 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97b20) get_ref 0x0~2000 Blob(0x558c57b97b20 blob([0x6a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97b90 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97b20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57b97b20 blob([0x6a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97b90 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57b97ba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97c00 blob([0x6a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97c70 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97c00) get_ref 0x0~2000 Blob(0x558c57b97c00 blob([0x6a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97c70 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97c00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57b97c00 blob([0x6a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97c70 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57b97c88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97ce0 blob([0x6a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97d50 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97ce0) get_ref 0x0~2000 Blob(0x558c57b97ce0 blob([0x6a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97d50 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57b97ce0 blob([0x6a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97d50 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57b97d68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97dc0 blob([0x6a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97e30 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97dc0) get_ref 0x0~2000 Blob(0x558c57b97dc0 blob([0x6a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97e30 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57b97dc0 blob([0x6a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97e30 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57b97e48 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ebc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97ea0 blob([0x6a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebc00 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97ea0) get_ref 0x0~2000 Blob(0x558c57b97ea0 blob([0x6a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebc00 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c57b97ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57b97ea0 blob([0x6a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ebc00 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ebc18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ebc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ebce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ebc70 blob([0x6ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebce0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebc70) get_ref 0x0~2000 Blob(0x558c581ebc70 blob([0x6ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebce0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebc70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c581ebc70 blob([0x6ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ebce0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ebcf8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ebd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ebd50 blob([0x6ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897ce0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebd50) get_ref 0x0~2000 Blob(0x558c581ebd50 blob([0x6ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897ce0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebd50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c581ebd50 blob([0x6ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897ce0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57897cf8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ebdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ebe30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ebdc0 blob([0x6af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebe30 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebdc0) get_ref 0x0~2000 Blob(0x558c581ebdc0 blob([0x6af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebe30 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c581ebdc0 blob([0x6af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ebe30 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ebe48 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ebea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581ebf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ebea0 blob([0x6b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebf10 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebea0) get_ref 0x0~2000 Blob(0x558c581ebea0 blob([0x6b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ebf10 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c581ebea0 blob([0x6b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ebf10 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c581ebf28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ebf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55136000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ebf80 blob([0x6b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136000 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebf80) get_ref 0x0~2000 Blob(0x558c581ebf80 blob([0x6b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136000 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c581ebf80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c581ebf80 blob([0x6b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55136000 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55136018 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55136070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551360e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55136070 blob([0x6b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551360e0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c55136070) get_ref 0x0~2000 Blob(0x558c55136070 blob([0x6b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551360e0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.blob(0x558c55136070) get_ref init 0x2000, 1000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c55136070 blob([0x6b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551360e0 sbid 0x0)) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c551360f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x32000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.extentmap(0x558c57efa888) dirty_range mark inline shard dirty 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 0 2019-12-07T19:34:17.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 4 keys 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 4 keys = 0 2019-12-07T19:34:17.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000499'.missing/0000000000000001.BC8CBBB8.head.smithi13140796-49..' <- missing/0000000000000001.BC8CBBB8.head.smithi13140796-49.. 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.656+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.656+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff200 cost 311037 (26 ios * 4000 + 207037 bytes) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff200 onodes 0x558c57efa780 shared_blobs 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.extentmap(0x558c57efa888) update 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore.extentmap(0x558c57efa888) update inline shard 478 bytes from 25 extents 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d313ddd1:::smithi13140796-49:head# is 969 (485 bytes onode + 2 bytes spanning blobs + 482 bytes inline extents) 2019-12-07T19:34:17.656+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff200 allocated 0x[685000~32000] released 0x[] 2019-12-07T19:34:17.656+0000 7f8f07187700 10 freelist allocate 0x685000~32000 2019-12-07T19:34:17.656+0000 7f8f07187700 20 freelist _xor first_key 0x680000 last_key 0x680000 2019-12-07T19:34:17.657+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 prepare 2019-12-07T19:34:17.657+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c574ff200 2019-12-07T19:34:17.657+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 finish 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.657+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.657+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.657+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 aio_wait 2019-12-07T19:34:17.657+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff200 2019-12-07T19:34:17.657+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 io_done 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2292 < min_flush_size 524288 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efa780 had 1 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2315 < min_flush_size 524288 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2315 < min_flush_size 524288 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1068eb~90b to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1068eb 0x1068eb~90b to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd68eb 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8eb 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1f6 and padding block with 0xe0a 2019-12-07T19:34:17.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 157 ==== MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=67, to_read={1:c8ce3553:::smithi13140796-29:e=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:e=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558c5793bb80 con 0x558c54c5b400 2019-12-07T19:34:17.657+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3fce0 prio 3 cost 0 latency 0.000018 epoch 40 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=67, to_read={1:c8ce3553:::smithi13140796-29:e=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:e=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.657+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=67, to_read={1:c8ce3553:::smithi13140796-29:e=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:e=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=67, to_read={1:c8ce3553:::smithi13140796-29:e=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:e=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1071f6 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=67, to_read={1:c8ce3553:::smithi13140796-29:e=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:e=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5513e000 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000189090s (0.000047202s flush + 0.000141888s kv commit) 2019-12-07T19:34:17.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c57f3fce0 prio 3 cost 0 latency 0.000075 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=67, to_read={1:c8ce3553:::smithi13140796-29:e=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:e=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=67, to_read={1:c8ce3553:::smithi13140796-29:e=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:e=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.657+0000 7f8f0317f700 25 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x200~200 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff200> 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 kv_submitted 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 finishing 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff200 onodes 0x558c57efa780 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8788 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f46c0 space 0x558c54cf8788 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55136018 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4fc0 space 0x558c55136018 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c551360f8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f5050 space 0x558c551360f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e25c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4990 space 0x558c574e25c8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e26a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4900 space 0x558c574e26a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2788 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4870 space 0x558c574e2788 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2868 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f42d0 space 0x558c574e2868 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757f7b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4360 space 0x558c5757f7b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2168 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f47e0 space 0x558c577f2168 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x200~200 = 512 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897cf8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4e10 space 0x558c57897cf8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a07a58 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4750 space 0x558c57a07a58 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97ac8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4a20 space 0x558c57b97ac8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x600~200 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97ba8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4ab0 space 0x558c57b97ba8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97c88 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4b40 space 0x558c57b97c88 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97d68 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4bd0 space 0x558c57b97d68 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97e48 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4c60 space 0x558c57b97e48 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581eb7b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f43f0 space 0x558c581eb7b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581eb898 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4480 space 0x558c581eb898 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581eb978 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4510 space 0x558c581eb978 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581eba58 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f45a0 space 0x558c581eba58 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ebb38 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4630 space 0x558c581ebb38 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ebc18 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4cf0 space 0x558c581ebc18 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ebcf8 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4d80 space 0x558c581ebcf8 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x600~200 = 512 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ebe48 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4ea0 space 0x558c581ebe48 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581ebf28 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4f30 space 0x558c581ebf28 0x0~2000 writing nocache) 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xa00~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xa00~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xe00~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 done 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff200 [] 2019-12-07T19:34:17.657+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xe00~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1200~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1200~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1600~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1600~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1a00~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1a00~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1e00~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1e00~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2200~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2200~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2600~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.657+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2600~200 = 512 2019-12-07T19:34:17.657+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2a00~200 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.657+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x4e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x5e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x6e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x7e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x8e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x9e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xa200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xa200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xa600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xa600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xaa00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xaa00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xae00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xae00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xb200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xb200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xb600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xb600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xba00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xba00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xbe00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xbe00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xc200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xc200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xc600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xc600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xca00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xca00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xce00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xce00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xd200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xd200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xd600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xd600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xda00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xda00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xde00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xde00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xe200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xe200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xe600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xe600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xea00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xea00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xee00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xee00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xf200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xf200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xf600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xf600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xfa00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xfa00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xfe00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0xfe00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x10e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x11e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x12e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x13e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x14e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x15e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x16e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x17e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x18e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19a00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19a00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19e00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x19e00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1a200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1a200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1a600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1a600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1aa00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1aa00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ae00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ae00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1b200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1b200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1b600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1b600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ba00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ba00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1be00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1be00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1c200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1c200~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1c600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1c600~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ca00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ca00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ce00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ce00~200 = 512 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1d200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1d200~200 = 0 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1d600~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1d600~200 = 0 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1da00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1da00~200 = 0 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1de00~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1de00~200 = 0 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1e200~200 2019-12-07T19:34:17.658+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.658+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1e200~200 = 0 2019-12-07T19:34:17.658+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1e600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1e600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ea00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ea00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ee00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1ee00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1f200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1f200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1f600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1f600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1fa00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[a~1]] recovery_committed_to version 21'11 now ondisk 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1fa00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1fe00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x1fe00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 40/39 [PushReplyOp(1:d313ddd1:::smithi13140796-49:head)]) v3 -- 0x558c55270800 con 0x558c54c5b400 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x20e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x21e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x22e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x23e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x24e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x25e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x26e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x27e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x28e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x29e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2a200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2a200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2a600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2a600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2aa00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2aa00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ae00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ae00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2b200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2b200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2b600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2b600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ba00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ba00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2be00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2be00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2c200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2c200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2c600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2c600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ca00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ca00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ce00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ce00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2d200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2d200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2d600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2d600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2da00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2da00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2de00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2de00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2e200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2e200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2e600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2e600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ea00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ea00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ee00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2ee00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2f200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2f200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2f600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2f600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2fa00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2fa00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2fe00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x2fe00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x30e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x31e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x32e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x33e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x34e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x35e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x36e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x37e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x38e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39a00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39a00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39e00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x39e00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3a200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3a200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3a600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3a600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3aa00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3aa00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ae00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ae00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3b200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3b200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3b600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3b600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ba00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ba00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3be00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3be00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3c200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3c200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3c600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3c600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ca00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ca00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ce00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ce00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3d200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3d200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3d600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3d600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3da00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3da00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3de00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3de00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3e200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3e200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3e600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3e600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ea00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ea00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ee00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3ee00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3f200~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3f200~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3f600~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3f600~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3fa00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3fa00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3fe00~200 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x1d000 (118784) 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 0x3fe00~200 = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=59392 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:c8ce3553:::smithi13140796-29:e 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:c8ce3553:::smithi13140796-29:e 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 118784 2019-12-07T19:34:17.659+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key = 0 2019-12-07T19:34:17.659+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.3s0 40/39 ECSubReadReply(tid=67, attrs_read=0)) v2 -- 0x558c57f34f00 con 0x558c54c5b400 2019-12-07T19:34:17.659+0000 7f8f0317f700 10 osd.7 40 dequeue_op 0x558c57f3fce0 finish 2019-12-07T19:34:17.659+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.660+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 158 ==== MOSDPGRecoveryDelete(1.bs1 e40,39 1:d07269f7:::smithi13140796-28:head,21'12) v2 ==== 123+0+0 (unknown 0 0 0) 0x558c55270800 con 0x558c54c5b400 2019-12-07T19:34:17.660+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57effb80 prio 3 cost 1000 latency 0.000014 epoch 40 MOSDPGRecoveryDelete(1.bs1 e40,39 1:d07269f7:::smithi13140796-28:head,21'12) v2 2019-12-07T19:34:17.660+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGRecoveryDelete(1.bs1 e40,39 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.660+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.660+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGRecoveryDelete(1.bs1 e40,39 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.660+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.660+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGRecoveryDelete(1.bs1 e40,39 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e40) pg 0x558c575ef000 2019-12-07T19:34:17.660+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57effb80 prio 3 cost 1000 latency 0.000041 MOSDPGRecoveryDelete(1.bs1 e40,39 1:d07269f7:::smithi13140796-28:head,21'12) v2 pg pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.660+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[a~1]] handle_recovery_delete 0x558c57effb80 2019-12-07T19:34:17.660+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[a~1]] remove_missing_object 1:d07269f7:::smithi13140796-28:head 21'12 2019-12-07T19:34:17.660+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.660+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c54ea0f00 seq 7 2019-12-07T19:34:17.660+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:d07269f7:::smithi13140796-28:head# key 0x818000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.660+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:17.660+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:17.660+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.660+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes shared_blobs 2019-12-07T19:34:17.660+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.660+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:17.660+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:17.660+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:17.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.660+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.660+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57effb80 finish 2019-12-07T19:34:17.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.660+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1071f6~2e to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1071f6 0x1071f6~2e to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd71f6 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1f6 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x224 and padding block with 0xddc 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x107224 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000114347s (0.000000106s flush + 0.000114241s kv commit) 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00> 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:17.661+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.661+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[a~1]] on_local_recover: 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:17.661+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[a~1]] got missing 1:d07269f7:::smithi13140796-28:head v 21'12 2019-12-07T19:34:17.661+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[a~1]] last_complete now 21'12 log.complete_to at end 2019-12-07T19:34:17.661+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.661+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5730b500 seq 8 2019-12-07T19:34:17.661+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000499'.missing/0000000000000001.B0E469FE.head.smithi13140796-28..' <- missing/0000000000000001.B0E469FE.head.smithi13140796-28.. 2019-12-07T19:34:17.661+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.661+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.661+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:17.661+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b500 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b500 onodes shared_blobs 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b500 allocated 0x[] released 0x[] 2019-12-07T19:34:17.661+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 prepare 2019-12-07T19:34:17.661+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b500 2019-12-07T19:34:17.661+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 io_done 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.661+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:17.661+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.661+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x107224~4b8 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x107224 0x107224~4b8 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd7224 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x224 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6dc and padding block with 0x924 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1076dc 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000105121s (0.000000073s flush + 0.000105048s kv commit) 2019-12-07T19:34:17.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b500> 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 kv_submitted 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b500 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 finishing 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b500 onodes 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 done 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b500 [] 2019-12-07T19:34:17.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.661+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.661+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[a~1]] recovery_committed_to version 21'12 now ondisk 2019-12-07T19:34:17.661+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_trim(1.bs0 21'12 e40/40) v2 -- 0x558c53a71a40 con 0x558c54c5b400 2019-12-07T19:34:17.661+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGRecoveryDeleteReply(1.bs0 e40,39 1:d07269f7:::smithi13140796-28:head,21'12) v2 -- 0x558c55250e00 con 0x558c54c5b400 2019-12-07T19:34:17.661+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.662+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 159 ==== MRecoveryReserve(1.bs1 RELEASE e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c53a71a40 con 0x558c54c5b400 2019-12-07T19:34:17.662+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.bs1 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.662+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.662+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.662+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.662+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.662+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c575ef000 2019-12-07T19:34:17.662+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.662+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.663712+0000 1 0.000028 2019-12-07T19:34:17.662+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.662+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.662+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.662+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 220 ==== MRecoveryReserve(1.4s0 GRANT e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c53a71a40 con 0x558c54c64000 2019-12-07T19:34:17.662+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.662+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.662+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.662+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:17.662+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MRecoveryReserve(1.4s4 REQUEST prio: 190 e40/40) v3 -- 0x558c54d39e00 con 0x558c54c19800 2019-12-07T19:34:17.662+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.662+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.663+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 221 ==== MRecoveryReserve(1.es4 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c53a71a40 con 0x558c54c64000 2019-12-07T19:34:17.663+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.es4 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.663+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.663+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.663+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.663+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.663+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c54d98000 2019-12-07T19:34:17.663+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.663+0000 7f8f0197c700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.664918+0000 3 0.000150 2019-12-07T19:34:17.663+0000 7f8f0197c700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.663+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:17.663+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.663+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.es4 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.663+0000 7f8f0b990700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c54d98000 2019-12-07T19:34:17.663+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.663+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.es0 GRANT e40/40) v3 -- 0x558c576dc000 con 0x558c54c64000 2019-12-07T19:34:17.663+0000 7f8f05984700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.664996+0000 1 0.000036 2019-12-07T19:34:17.663+0000 7f8f05984700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.663+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:17.663+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.664+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 160 ==== MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=68, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558c57f34f00 con 0x558c54c5b400 2019-12-07T19:34:17.664+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57effe40 prio 3 cost 0 latency 0.000019 epoch 40 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=68, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.664+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=68, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.664+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.664+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=68, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.664+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.664+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=68, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5513e000 2019-12-07T19:34:17.664+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57effe40 prio 3 cost 0 latency 0.000136 MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=68, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.664+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.3s4 40/39 ECSubRead(tid=68, to_read={1:c8ce3553:::smithi13140796-29:head=0,262144,0}, subchunks={1:c8ce3553:::smithi13140796-29:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.664+0000 7f8f07187700 25 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.664+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x200~200 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x26000 (155648) 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48018 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.664+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580cd7a0 space 0x558c57f48018 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.664+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x200~200 = 512 2019-12-07T19:34:17.664+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x600~200 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x26000 (155648) 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.664+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x600~200 = 512 2019-12-07T19:34:17.664+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa00~200 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.664+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa00~200 = 512 2019-12-07T19:34:17.664+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe00~200 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x26000 (155648) 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.664+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe00~200 = 512 2019-12-07T19:34:17.664+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1200~200 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x26000 (155648) 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.664+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48018 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.665+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5200 space 0x558c57f48018 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.665+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1200~200 = 512 2019-12-07T19:34:17.665+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1600~200 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x26000 (155648) 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.665+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1600~200 = 512 2019-12-07T19:34:17.665+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a00~200 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.665+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a00~200 = 512 2019-12-07T19:34:17.665+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e00~200 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bf80 blob([0x519000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48000 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.665+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e00~200 = 512 2019-12-07T19:34:17.665+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2200~200 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x26000 (155648) 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.665+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f480f8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5290 space 0x558c57f480f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2200~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2600~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2600~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3200~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f480f8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5320 space 0x558c57f480f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3200~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3600~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3600~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48070 blob([0x51b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f480e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4200~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f481d8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f53b0 space 0x558c57f481d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4200~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4600~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4600~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4a00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4a00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4e00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x4e00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5200~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f481d8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5440 space 0x558c57f481d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5200~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5600~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5600~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5a00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5a00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5e00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48150 blob([0x51d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f481c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x5e00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6200~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f482b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f54d0 space 0x558c57f482b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6200~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6600~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6600~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6a00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6a00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6e00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x6e00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7200~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f482b8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5560 space 0x558c57f482b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7200~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7600~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7600~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7a00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7a00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7e00~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48230 blob([0x51f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f482a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x7e00~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8200~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48398 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.666+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f55f0 space 0x558c57f48398 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.666+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8200~200 = 512 2019-12-07T19:34:17.666+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8600~200 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x26000 (155648) 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.666+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8a00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8a00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8e00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x8e00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9200~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48398 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5680 space 0x558c57f48398 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9200~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9600~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9a00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9a00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9e00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48310 blob([0x521000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48380 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x9e00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa200~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48478 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5710 space 0x558c57f48478 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa200~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa600~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xa600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xaa00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xaa00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xae00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xae00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb200~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48478 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f57a0 space 0x558c57f48478 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb200~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb600~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xb600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xba00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xba00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xbe00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f483f0 blob([0x523000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48460 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xbe00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc200~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48558 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5830 space 0x558c57f48558 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc200~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc600~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xc600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xca00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xca00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xce00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xce00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd200~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.667+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 222 ==== MOSDPGPush(1.es4 40/39 [PushOp(1:733d7474:::smithi13140796-5:head, version: 21'2, data_included: [0~20480], data_size: 20480, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:733d7474:::smithi13140796-5:head@21'2, size: 81920, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:81920, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 21537+0+0 (crc 0 0 0) 0x558c54dfbe00 con 0x558c54c64000 2019-12-07T19:34:17.667+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57efe160 prio 127 cost 21480 latency 0.000034 epoch 40 MOSDPGPush(1.es4 40/39 [PushOp(1:733d7474:::smithi13140796-5:head, version: 21'2, data_included: [0~20480], data_size: 20480, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:733d7474:::smithi13140796-5:head@21'2, size: 81920, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:81920, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.667+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDPGPush(1.es4 40/39 [PushOp(1:733d7474:::smithi13140796-5:head, version: 21'2, data_included: [0~20480], data_size: 20480, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:733d7474:::smithi13140796-5:head@21'2, size: 81920, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:81920, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 21480 e40) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDPGPush(1.es4 40/39 [PushOp(1:733d7474:::smithi13140796-5:head, version: 21'2, data_included: [0~20480], data_size: 20480, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:733d7474:::smithi13140796-5:head@21'2, size: 81920, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:81920, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 21480 e40) queued 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48558 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f58c0 space 0x558c57f48558 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd200~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd600~200 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDPGPush(1.es4 40/39 [PushOp(1:733d7474:::smithi13140796-5:head, version: 21'2, data_included: [0~20480], data_size: 20480, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:733d7474:::smithi13140796-5:head@21'2, size: 81920, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:81920, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 21480 e40) pg 0x558c54d98000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xd600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xda00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xda00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xde00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f484d0 blob([0x525000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48540 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 osd.7 40 dequeue_op 0x558c57efe160 prio 127 cost 21480 latency 0.000093 MOSDPGPush(1.es4 40/39 [PushOp(1:733d7474:::smithi13140796-5:head, version: 21'2, data_included: [0~20480], data_size: 20480, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:733d7474:::smithi13140796-5:head@21'2, size: 81920, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:81920, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xde00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe200~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.es4 40/39 [PushOp(1:733d7474:::smithi13140796-5:head, version: 21'2, data_included: [0~20480], data_size: 20480, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:733d7474:::smithi13140796-5:head@21'2, size: 81920, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:81920, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] on_local_recover: 1:733d7474:::smithi13140796-5:head 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'1 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=2 mbc={} ps=[a~1]] got missing 1:733d7474:::smithi13140796-5:head v 21'2 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'3 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=1 mbc={} ps=[a~1]] last_complete now 21'3 log.complete_to 21'4 2019-12-07T19:34:17.667+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c577fc600 seq 4 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48638 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5950 space 0x558c57f48638 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) get_onode oid 4#1:733d7474:::smithi13140796-5:head# key 0x848000000000000001733d74't!smithi13140796-5!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe200~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe600~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xe600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) r -2 v.len 0 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xea00~200 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) get_onode oid 4#1:733d7474:::smithi13140796-5:head# key 0x848000000000000001733d74't!smithi13140796-5!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) r -2 v.len 0 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xea00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xee00~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1188 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:733d7474:::smithi13140796-5:head# = 0 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x0~5000 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xee00~200 = 512 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:733d7474:::smithi13140796-5:head# 0x0~5000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf200~200 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~5000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc600 3 blobs 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x5000/1000,5000,0 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x6b7000~5000/1000,5000,0 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x6b7000~5000] 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896380 blob([0x6b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896310 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.blob(0x558c57896380) get_ref 0x0~2000 Blob(0x558c57896380 blob([0x6b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896310 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.blob(0x558c57896380) get_ref init 0x2000, 1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57896380 blob([0x6b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896328 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578962a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578962a0 blob([0x6b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8150 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.blob(0x558c578962a0) get_ref 0x0~2000 Blob(0x558c578962a0 blob([0x6b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8150 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.blob(0x558c578962a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c578962a0 blob([0x6b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54df8168 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896070 blob([0x6bb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896540 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.blob(0x558c57896070) get_ref 0x0~1000 Blob(0x558c57896070 blob([0x6bb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896540 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.blob(0x558c57896070) get_ref init 0x1000, 1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~1000: 0x0~1000 Blob(0x558c57896070 blob([0x6bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 sbid 0x0)) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896558 in 0x558c53a59ea0) _discard 0x0~1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3288) dirty_range mark inline shard dirty 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x0~5000 = 0 2019-12-07T19:34:17.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 4 keys 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 4 keys = 0 2019-12-07T19:34:17.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049d'.missing/0000000000000001.ECCBE2E2.head.smithi13140796-5..' <- missing/0000000000000001.ECCBE2E2.head.smithi13140796-5.. 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.667+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc600 cost 38713 (4 ios * 4000 + 22713 bytes) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc600 onodes 0x558c54ce3180 shared_blobs 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3288) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3288) update inline shard 57 bytes from 3 extents 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 546 (483 bytes onode + 2 bytes spanning blobs + 61 bytes inline extents) 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc600 allocated 0x[6b7000~5000] released 0x[] 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 freelist allocate 0x6b7000~5000 2019-12-07T19:34:17.667+0000 7f8f0197c700 20 freelist _xor first_key 0x680000 last_key 0x680000 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48638 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 prepare 2019-12-07T19:34:17.667+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fc600 2019-12-07T19:34:17.667+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f59e0 space 0x558c57f48638 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf200~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf600~200 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x26000 (155648) 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.667+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.667+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xf600~200 = 512 2019-12-07T19:34:17.667+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfa00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfa00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfe00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f485b0 blob([0x527000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48620 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0xfe00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f0197c700 10 osd.7 40 dequeue_op 0x558c57efe160 finish 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48718 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5a70 space 0x558c57f48718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10600~200 2019-12-07T19:34:17.668+0000 7f8f05984700 20 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'3 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=1 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x10e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 aio_wait 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc600 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48718 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5b00 space 0x558c57f48718 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 io_done 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48690 blob([0x529000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48700 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x11e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f487f8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5b90 space 0x558c57f487f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x12e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f487f8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5c20 space 0x558c57f487f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48770 blob([0x52b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f487e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x13e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f488d8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5cb0 space 0x558c57f488d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x14e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f488d8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c581f5d40 space 0x558c57f488d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48850 blob([0x52d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f488c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x15e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f489b8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46000 space 0x558c57f489b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x16e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f489b8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46090 space 0x558c57f489b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48930 blob([0x52f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f489a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x17e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48a98 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46120 space 0x558c57f48a98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x18e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48a98 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f461b0 space 0x558c57f48a98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19a00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19a00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19e00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48a10 blob([0x531000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48a80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x19e00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1867 < min_flush_size 524288 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3180 had 1 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1890 < min_flush_size 524288 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1890 < min_flush_size 524288 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1076dc~762 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1076dc 0x1076dc~762 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48b78 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd76dc 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6dc 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46240 space 0x558c57f48b78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe3e and padding block with 0x1c2 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1a600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1aa00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1aa00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ae00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ae00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x107e3e 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48b78 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f462d0 space 0x558c57f48b78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000188772s (0.000048025s flush + 0.000140747s kv commit) 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc600> 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b600~200 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 kv_submitted 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc600 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 finishing 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1b600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc600 onodes 0x558c54ce3180 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ba00~200 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8168 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4750 space 0x558c54df8168 0x0~2000 writing nocache) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896328 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4e10 space 0x558c57896328 0x0~2000 writing nocache) 2019-12-07T19:34:17.668+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896558 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4a20 space 0x558c57896558 0x0~1000 writing nocache) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc600 done 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ba00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc600 [] 2019-12-07T19:34:17.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1be00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48af0 blob([0x533000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48b60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1be00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'3 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 0'0 active m=1 mbc={} ps=[a~1]] recovery_committed_to version 21'3 now ondisk 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.668+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.es0 40/39 [PushReplyOp(1:733d7474:::smithi13140796-5:head)]) v3 -- 0x558c575b9a00 con 0x558c54c64000 2019-12-07T19:34:17.668+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48c58 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.668+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46360 space 0x558c57f48c58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c200~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c600~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1c600~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ca00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ca00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ce00~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.668+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ce00~200 = 512 2019-12-07T19:34:17.668+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d200~200 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x26000 (155648) 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.668+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48c58 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f463f0 space 0x558c57f48c58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1d600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1da00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1da00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1de00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48bd0 blob([0x535000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48c40 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1de00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need {<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48d38 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46480 space 0x558c57f48d38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1e600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ea00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ea00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ee00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1ee00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48d38 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46510 space 0x558c57f48d38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1f600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fa00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fa00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fe00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48cb0 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48d20 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x1fe00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need {<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48e18 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f465a0 space 0x558c57f48e18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20a00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x20e00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need {<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48e18 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46630 space 0x558c57f48e18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21a00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48d90 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48e00 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x21e00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need {<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48ef8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f466c0 space 0x558c57f48ef8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22a00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x22e00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need {<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48ef8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46750 space 0x558c57f48ef8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23a00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48e70 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48ee0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x23e00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need {<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48fd8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f467e0 space 0x558c57f48fd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24a00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x24e00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x25200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x25200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x25200:1200~200]} 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f48fd8 in 0x558c53a596c0) _discard 0x1000~1000 2019-12-07T19:34:17.669+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f46870 space 0x558c57f48fd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25200~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25600~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25a00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f48f50 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f48fc0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x25e00~200 = 512 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26200~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26600~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26a00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x26e00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27200~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27600~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27a00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x27e00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28200~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28600~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28a00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x28e00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29200~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29200~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29600~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29600~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29a00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29a00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29e00~200 2019-12-07T19:34:17.669+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.669+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x29e00~200 = 0 2019-12-07T19:34:17.669+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2a600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2aa00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2aa00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ae00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ae00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2b600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ba00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ba00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2be00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2be00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2c600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ca00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ca00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ce00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ce00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2d600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2da00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2da00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2de00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2de00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2e600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ea00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ea00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ee00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2ee00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2f600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fa00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fa00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fe00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x2fe00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x30e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x31e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x32e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x33e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x34e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x35e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x36e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x37e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x38e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39a00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39a00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39e00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x39e00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3a600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3aa00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3aa00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ae00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ae00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3b600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ba00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ba00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3be00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3be00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3c600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ca00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ca00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ce00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ce00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3d600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3da00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3da00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3de00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3de00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3e600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ea00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ea00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ee00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3ee00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f200~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f200~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f600~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3f600~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fa00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fa00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fe00~200 2019-12-07T19:34:17.670+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x26000 (155648) 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 0x3fe00~200 = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 20 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=77824 2019-12-07T19:34:17.670+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:c8ce3553:::smithi13140796-29:head 2019-12-07T19:34:17.670+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:c8ce3553:::smithi13140796-29:head 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:17.670+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 155648 2019-12-07T19:34:17.670+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key 2019-12-07T19:34:17.670+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key = 0 2019-12-07T19:34:17.670+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.3s0 40/39 ECSubReadReply(tid=68, attrs_read=0)) v2 -- 0x558c57f35180 con 0x558c54c5b400 2019-12-07T19:34:17.670+0000 7f8f07187700 10 osd.7 40 dequeue_op 0x558c57effe40 finish 2019-12-07T19:34:17.670+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.675+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 223 ==== MOSDPGPush(1.es4 40/39 [PushOp(1:76d0c63f:::smithi13140796-38:head, version: 21'4, data_included: [0~155648], data_size: 155648, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:76d0c63f:::smithi13140796-38:head@21'4, size: 622592, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:622592, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 156710+0+0 (crc 0 0 0) 0x558c55250e00 con 0x558c54c64000 2019-12-07T19:34:17.675+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57efe9a0 prio 127 cost 156648 latency 0.000037 epoch 40 MOSDPGPush(1.es4 40/39 [PushOp(1:76d0c63f:::smithi13140796-38:head, version: 21'4, data_included: [0~155648], data_size: 155648, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:76d0c63f:::smithi13140796-38:head@21'4, size: 622592, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:622592, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.675+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDPGPush(1.es4 40/39 [PushOp(1:76d0c63f:::smithi13140796-38:head, version: 21'4, data_included: [0~155648], data_size: 155648, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:76d0c63f:::smithi13140796-38:head@21'4, size: 622592, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:622592, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 156648 e40) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDPGPush(1.es4 40/39 [PushOp(1:76d0c63f:::smithi13140796-38:head, version: 21'4, data_included: [0~155648], data_size: 155648, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:76d0c63f:::smithi13140796-38:head@21'4, size: 622592, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:622592, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 156648 e40) queued 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.675+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 161 ==== MRecoveryReserve(1.3s4 RELEASE e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c576dc000 con 0x558c54c5b400 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDPGPush(1.es4 40/39 [PushOp(1:76d0c63f:::smithi13140796-38:head, version: 21'4, data_included: [0~155648], data_size: 155648, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:76d0c63f:::smithi13140796-38:head@21'4, size: 622592, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:622592, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 156648 e40) pg 0x558c54d98000 2019-12-07T19:34:17.675+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.3s4 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.675+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 osd.7 40 dequeue_op 0x558c57efe9a0 prio 127 cost 156648 latency 0.000098 MOSDPGPush(1.es4 40/39 [PushOp(1:76d0c63f:::smithi13140796-38:head, version: 21'4, data_included: [0~155648], data_size: 155648, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:76d0c63f:::smithi13140796-38:head@21'4, size: 622592, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:622592, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.es4( v 21'4 lc 21'3 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'3 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.es4 40/39 [PushOp(1:76d0c63f:::smithi13140796-38:head, version: 21'4, data_included: [0~155648], data_size: 155648, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:76d0c63f:::smithi13140796-38:head@21'4, size: 622592, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:622592, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'3 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active m=1 mbc={} ps=[a~1]] on_local_recover: 1:76d0c63f:::smithi13140796-38:head 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 lc 21'3 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active m=1 mbc={} ps=[a~1]] got missing 1:76d0c63f:::smithi13140796-38:head v 21'4 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 21'3 active mbc={} ps=[a~1]] last_complete now 21'4 log.complete_to at end 2019-12-07T19:34:17.675+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c577fc900 seq 5 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) get_onode oid 4#1:76d0c63f:::smithi13140796-38:head# key 0x84800000000000000176d0c6'?!smithi13140796-38!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) r -2 v.len 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) get_onode oid 4#1:76d0c63f:::smithi13140796-38:head# key 0x84800000000000000176d0c6'?!smithi13140796-38!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) r -2 v.len 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1189 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# = 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~26000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc900 19 blobs 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x26000/1000,26000,0 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x6bc000~26000/1000,26000,0 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x6bc000~26000] 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d75d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d75d50 blob([0x6bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42850 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75d50) get_ref 0x0~2000 Blob(0x558c54d75d50 blob([0x6bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42850 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c54d75d50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d75d50 blob([0x6bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42850 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57c42868 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c428c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578963f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c428c0 blob([0x6be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578963f0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57c428c0) get_ref 0x0~2000 Blob(0x558c57c428c0 blob([0x6be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578963f0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57c428c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57c428c0 blob([0x6be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578963f0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896408 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896620 blob([0x6c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896690 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896620) get_ref 0x0~2000 Blob(0x558c57896620 blob([0x6c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896690 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896620) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57896620 blob([0x6c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896690 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c578966a8 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896700 blob([0x6c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896770 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896700) get_ref 0x0~2000 Blob(0x558c57896700 blob([0x6c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896770 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896700) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57896700 blob([0x6c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896770 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896788 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578967e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578967e0 blob([0x6c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896850 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c578967e0) get_ref 0x0~2000 Blob(0x558c578967e0 blob([0x6c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896850 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c578967e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c578967e0 blob([0x6c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896850 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896868 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578968c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578968c0 blob([0x6c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896930 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c578968c0) get_ref 0x0~2000 Blob(0x558c578968c0 blob([0x6c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896930 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c578968c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c578968c0 blob([0x6c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896930 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896948 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578969a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578969a0 blob([0x6c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896a10 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c578969a0) get_ref 0x0~2000 Blob(0x558c578969a0 blob([0x6c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896a10 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c578969a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c578969a0 blob([0x6c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896a10 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896a28 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896a80 blob([0x6ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896af0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896a80) get_ref 0x0~2000 Blob(0x558c57896a80 blob([0x6ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896af0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896a80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57896a80 blob([0x6ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896af0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896b08 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896b60 blob([0x6cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896bd0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896b60) get_ref 0x0~2000 Blob(0x558c57896b60 blob([0x6cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896bd0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896b60) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57896b60 blob([0x6cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896bd0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896be8 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896c40 blob([0x6ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896cb0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896c40) get_ref 0x0~2000 Blob(0x558c57896c40 blob([0x6ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896cb0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896c40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57896c40 blob([0x6ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896cb0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896cc8 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896d20 blob([0x6d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896d90 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896d20) get_ref 0x0~2000 Blob(0x558c57896d20 blob([0x6d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896d90 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57896d20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57896d20 blob([0x6d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896d90 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896da8 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137c00 blob([0x6d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137c70 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137c00) get_ref 0x0~2000 Blob(0x558c55137c00 blob([0x6d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137c70 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137c00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c55137c00 blob([0x6d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137c70 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c55137c88 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137ce0 blob([0x6d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137d50 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137ce0) get_ref 0x0~2000 Blob(0x558c55137ce0 blob([0x6d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137d50 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c55137ce0 blob([0x6d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137d50 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c55137d68 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137dc0 blob([0x6d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137e30 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137dc0) get_ref 0x0~2000 Blob(0x558c55137dc0 blob([0x6d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137e30 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c55137dc0 blob([0x6d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137e30 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c55137e48 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137ea0 blob([0x6d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137f10 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137ea0) get_ref 0x0~2000 Blob(0x558c55137ea0 blob([0x6d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137f10 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c55137ea0 blob([0x6d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137f10 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c55137f28 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137f80 blob([0x6da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137f80) get_ref 0x0~2000 Blob(0x558c55137f80 blob([0x6da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c55137f80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c55137f80 blob([0x6da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dab0 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c54d3dac8 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f90000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f90000 blob([0x6dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90070 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90000) get_ref 0x0~2000 Blob(0x558c57f90000 blob([0x6dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90070 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57f90000 blob([0x6dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f90070 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57f90088 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f900e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f900e0 blob([0x6de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90150 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57f900e0) get_ref 0x0~2000 Blob(0x558c57f900e0 blob([0x6de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90150 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57f900e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57f900e0 blob([0x6de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f90150 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57f90168 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f901c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f901c0 blob([0x6e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90230 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57f901c0) get_ref 0x0~2000 Blob(0x558c57f901c0 blob([0x6e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90230 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.blob(0x558c57f901c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57f901c0 blob([0x6e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f90230 sbid 0x0)) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57f90248 in 0x558c53a59ea0) _discard 0x0~2000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x26000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 = 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 4 keys 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 4 keys = 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049d'.missing/0000000000000001.E6B036CF.head.smithi13140796-38..' <- missing/0000000000000001.E6B036CF.head.smithi13140796-38.. 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc900 cost 237885 (20 ios * 4000 + 157885 bytes) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc900 onodes 0x558c57f8e000 shared_blobs 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e108) update 4#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e108) update inline shard 364 bytes from 19 extents 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:76d0c63f:::smithi13140796-38:head# is 855 (485 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc900 allocated 0x[6bc000~26000] released 0x[] 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 freelist allocate 0x6bc000~26000 2019-12-07T19:34:17.675+0000 7f8f0197c700 20 freelist _xor first_key 0x680000 last_key 0x680000 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 prepare 2019-12-07T19:34:17.675+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fc900 2019-12-07T19:34:17.676+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.676+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.676+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.676+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c5513e000 2019-12-07T19:34:17.676+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.676+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.677239+0000 1 0.000037 2019-12-07T19:34:17.676+0000 7f8f0317f700 5 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.676+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:17.676+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:17.676+0000 7f8f0197c700 10 osd.7 40 dequeue_op 0x558c57efe9a0 finish 2019-12-07T19:34:17.676+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.676+0000 7f8f05984700 20 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 21'3 active mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.676+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.676+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 aio_wait 2019-12-07T19:34:17.676+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc900 2019-12-07T19:34:17.676+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 io_done 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 450 < min_flush_size 524288 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2189 < min_flush_size 524288 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e000 had 1 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2212 < min_flush_size 524288 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2212 < min_flush_size 524288 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x107e3e~8a4 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x107e3e 0x107e3e~8a4 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd7e3e 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe3e 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6e2 and padding block with 0x91e 2019-12-07T19:34:17.676+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 86 ==== MRecoveryReserve(1.5s4 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54d39e00 con 0x558c54c19800 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1086e2 2019-12-07T19:34:17.676+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.5s4 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.676+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.676+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000163279s (0.000049623s flush + 0.000113656s kv commit) 2019-12-07T19:34:17.676+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.676+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.676+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc900> 2019-12-07T19:34:17.676+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.676+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c57522000 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 kv_submitted 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc900 2019-12-07T19:34:17.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 finishing 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc900 onodes 0x558c57f8e000 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dac8 in 0x558c53a59ea0) _finish_write discard buffer(0x558c57f470e0 space 0x558c54d3dac8 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137c88 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f5e60 space 0x558c55137c88 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137d68 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f5ef0 space 0x558c55137d68 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.676+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137e48 in 0x558c53a59ea0) _finish_write discard buffer(0x558c57f46fc0 space 0x558c55137e48 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137f28 in 0x558c53a59ea0) _finish_write discard buffer(0x558c57f47050 space 0x558c55137f28 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896408 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4c60 space 0x558c57896408 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578966a8 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f43f0 space 0x558c578966a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896788 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4480 space 0x558c57896788 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896868 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4510 space 0x558c57896868 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896948 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f45a0 space 0x558c57896948 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896a28 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4630 space 0x558c57896a28 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896b08 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4cf0 space 0x558c57896b08 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896be8 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4d80 space 0x558c57896be8 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896cc8 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4ea0 space 0x558c57896cc8 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896da8 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f5dd0 space 0x558c57896da8 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 lcod 21'3 active mbc={} ps=[a~1]] recovery_committed_to version 21'4 now ondisk 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42868 in 0x558c53a59ea0) _finish_write discard buffer(0x558c581f4bd0 space 0x558c57c42868 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f06185700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.677728+0000 3 0.000176 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90088 in 0x558c53a59ea0) _finish_write discard buffer(0x558c57f47170 space 0x558c57f90088 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90168 in 0x558c53a59ea0) _finish_write discard buffer(0x558c57f47200 space 0x558c57f90168 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90248 in 0x558c53a59ea0) _finish_write discard buffer(0x558c57f47290 space 0x558c57f90248 0x0~2000 writing nocache) 2019-12-07T19:34:17.676+0000 7f8f06185700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.676+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_trim(1.es0 21'4 e40/40) v2 -- 0x558c576dda40 con 0x558c54c64000 2019-12-07T19:34:17.676+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc900 done 2019-12-07T19:34:17.676+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:17.676+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc900 [] 2019-12-07T19:34:17.676+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.676+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.676+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.es0 40/39 [PushReplyOp(1:76d0c63f:::smithi13140796-38:head)]) v3 -- 0x558c54dfbe00 con 0x558c54c64000 2019-12-07T19:34:17.676+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.676+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.5s4 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.676+0000 7f8f0b990700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.676+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.676+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.676+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.676+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c57522000 2019-12-07T19:34:17.676+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.676+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MRecoveryReserve(1.5s0 GRANT e40/40) v3 -- 0x558c575cd0e0 con 0x558c54c19800 2019-12-07T19:34:17.676+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.677810+0000 1 0.000024 2019-12-07T19:34:17.676+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.676+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.676+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.677+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 224 ==== MRecoveryReserve(1.es4 RELEASE e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c576dda40 con 0x558c54c64000 2019-12-07T19:34:17.677+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.es4 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.677+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.677+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.677+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.677+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.677+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c54d98000 2019-12-07T19:34:17.677+0000 7f8f0197c700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.677+0000 7f8f0197c700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.678691+0000 1 0.000039 2019-12-07T19:34:17.677+0000 7f8f0197c700 5 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.677+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:17.677+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:17.677+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 87 ==== MRecoveryReserve(1.4s0 GRANT e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c575cd0e0 con 0x558c54c19800 2019-12-07T19:34:17.677+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.677+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.677+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.677+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:17.677+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.4s1 REQUEST prio: 190 e40/40) v3 -- 0x558c576dcb40 con 0x558c54c5bc00 2019-12-07T19:34:17.677+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.677+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.679+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 225 ==== MRecoveryReserve(1.7s4 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c576dda40 con 0x558c54c64000 2019-12-07T19:34:17.679+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.7s4 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.679+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.679+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.679+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.679+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.679+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.679+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.679+0000 7f8f05183700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.680857+0000 3 0.000095 2019-12-07T19:34:17.679+0000 7f8f05183700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.679+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.679+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.679+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.7s4 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.679+0000 7f8f0b990700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.679+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.679+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.679+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.679+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.679+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.679+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.7s0 GRANT e40/40) v3 -- 0x558c57ede960 con 0x558c54c64000 2019-12-07T19:34:17.679+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.680944+0000 1 0.000032 2019-12-07T19:34:17.679+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.679+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:17.679+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.680+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 226 ==== MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=181, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 249+0+0 (crc 0 0 0) 0x558c57f35180 con 0x558c54c64000 2019-12-07T19:34:17.680+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c54e79340 prio 3 cost 0 latency 0.000024 epoch 40 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=181, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.680+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=181, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.680+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=181, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.680+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.680+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=181, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.680+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c54e79340 prio 3 cost 0 latency 0.000072 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=181, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=181, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.680+0000 7f8f05183700 25 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xc00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xc00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x4400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x4400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x4c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x4c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x5400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x5400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x5c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x5c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x6400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x6400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x6c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x6c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x7400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x7400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x7c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x7c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x8400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x8400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x8c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x8c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x9400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x9400~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x9c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x9c00~400 = 1024 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xa400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xa400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xac00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xac00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xac00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xb400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xb400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xbc00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xbc00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xc400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xc400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xcc00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xcc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xcc00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xd400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xd400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xdc00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xdc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xdc00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xe400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xe400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xec00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xec00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xec00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xf400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xf400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xfc00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0xfc00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x10400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x10400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x10c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x10c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x11400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x11400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x11c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x11c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x12400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x12400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x12c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x12c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x13400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x13400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x13c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x13c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x14400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x14400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x14c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x14c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x15400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x15400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x15c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x15c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x16400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x16400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x16c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x16c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x17400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x17400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x17c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x17c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x18400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x18400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x18c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x18c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x19400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x19400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x19c00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x19c00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1a400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1a400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1ac00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ac00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1ac00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1b400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1b400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1bc00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1bc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1bc00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1c400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c400~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1c400~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1cc00~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1cc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.680+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1cc00~400 = 0 2019-12-07T19:34:17.680+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1d400~400 2019-12-07T19:34:17.680+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1d400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1dc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1dc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1dc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1e400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1e400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1ec00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ec00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1ec00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1f400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1f400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1fc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x1fc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x20400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x20400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x20c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x20c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x21400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x21400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x21c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x21c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x22400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x22400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x22c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x22c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x23400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x23400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x23c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x23c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x24400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x24400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x24c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x24c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x25400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x25400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x25c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x25c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x26400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x26400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x26c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x26c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x27400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x27400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x27c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x27c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x28400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x28400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x28c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x28c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x29400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x29400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x29c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x29c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2a400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2a400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2ac00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ac00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2ac00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2b400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2b400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2bc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2bc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2bc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2c400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2c400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2cc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2cc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2cc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2d400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2d400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2dc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2dc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2dc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2e400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2e400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2ec00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ec00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2ec00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2f400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2f400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2fc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x2fc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x30400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x30400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x30c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x30c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x31400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x31400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x31c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x31c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x32400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x32400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x32c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x32c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x33400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x33400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x33c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x33c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x34400~400 2019-12-07T19:34:17.681+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 88 ==== MOSDPGPush(1.5s4 40/39 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 50214+0+0 (crc 0 0 0) 0x558c54c58200 con 0x558c54c19800 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x34400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x34c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c5740c000 prio 127 cost 50152 latency 0.000025 epoch 40 MOSDPGPush(1.5s4 40/39 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x34c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x35400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 50152 e40) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x35400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x35c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x35c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x36400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x36400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x36c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x36c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x37400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x37400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 50152 e40) queued 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x37c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x37c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x38400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x38400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 50152 e40) pg 0x558c57522000 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x38c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x38c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x39400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x39400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x39c00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39c00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f06185700 10 osd.7 40 dequeue_op 0x558c5740c000 prio 127 cost 50152 latency 0.000063 MOSDPGPush(1.5s4 40/39 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x39c00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3a400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3a400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3ac00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ac00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.5s4 40/39 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3ac00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3b400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3b400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3bc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3bc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3bc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3c400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] on_local_recover: 1:a9dea763:::smithi13140796-12:head 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3c400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3cc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3cc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3cc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=3 mbc={} ps=[a~1]] got missing 1:a9dea763:::smithi13140796-12:head v 21'2 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3d400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3d400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=2 mbc={} ps=[a~1]] last_complete now 21'3 log.complete_to 21'4 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3dc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3dc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3dc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3e400~400 2019-12-07T19:34:17.681+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3e400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3ec00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ec00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3ec00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3f400~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f400~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3f400~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3fc00~400 2019-12-07T19:34:17.681+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fc00~400 size 0xa000 (40960) 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x3fc00~400 = 0 2019-12-07T19:34:17.681+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=20480 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5730bb00 seq 4 2019-12-07T19:34:17.681+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) get_onode oid 4#1:a9dea763:::smithi13140796-12:head# key 0x848000000000000001a9dea7'c!smithi13140796-12!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.681+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:17.681+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 40960 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) r -2 v.len 0 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:34:17.681+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) get_onode oid 4#1:a9dea763:::smithi13140796-12:head# key 0x848000000000000001a9dea7'c!smithi13140796-12!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.681+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) r -2 v.len 0 2019-12-07T19:34:17.681+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1190 2019-12-07T19:34:17.681+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 40/39 ECSubReadReply(tid=181, attrs_read=0)) v2 -- 0x558c574a0780 con 0x558c54c64000 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 0x0~c000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:a9dea763:::smithi13140796-12:head# 0x0~c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.681+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c54e79340 finish 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730bb00 6 blobs 2019-12-07T19:34:17.681+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.681+0000 7f8f06185700 10 fbmap_alloc 0x558c53a72e00 allocate 0xc000/1000,c000,0 2019-12-07T19:34:17.681+0000 7f8f06185700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x6e2000~c000/1000,c000,0 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x6e2000~c000] 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5420 blob([0x6e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5b20 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5420) get_ref 0x0~2000 Blob(0x558c580a5420 blob([0x6e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5b20 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5420) get_ref init 0x2000, 1000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c580a5420 blob([0x6e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5b20 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c580a5b38 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5ab0 blob([0x6e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5880 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5ab0) get_ref 0x0~2000 Blob(0x558c580a5ab0 blob([0x6e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5880 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c580a5ab0 blob([0x6e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5880 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c580a5898 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a58f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5810 blob([0x6e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a58f0 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5810) get_ref 0x0~2000 Blob(0x558c580a5810 blob([0x6e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a58f0 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c580a5810 blob([0x6e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a58f0 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c580a5908 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5960 blob([0x6e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b960 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5960) get_ref 0x0~2000 Blob(0x558c580a5960 blob([0x6e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b960 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c580a5960) get_ref init 0x2000, 1000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c580a5960 blob([0x6e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3b960 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c57f3b978 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3b8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3b8f0 blob([0x6ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b880 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c57f3b8f0) get_ref 0x0~2000 Blob(0x558c57f3b8f0 blob([0x6ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3b880 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c57f3b8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57f3b8f0 blob([0x6ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3b880 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c57f3b898 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efdd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3b9d0 blob([0x6ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdd50 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c57f3b9d0) get_ref 0x0~2000 Blob(0x558c57f3b9d0 blob([0x6ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdd50 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.blob(0x558c57f3b9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57f3b9d0 blob([0x6ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdd50 sbid 0x0)) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.BufferSpace(0x558c57efdd68 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xc000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.extentmap(0x558c54d2c608) dirty_range mark inline shard dirty 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 0x0~c000 = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 4 keys 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 4 keys = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049a'.missing/0000000000000001.59B75E6C.head.smithi13140796-12..' <- missing/0000000000000001.59B75E6C.head.smithi13140796-12.. 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730bb00 cost 79389 (7 ios * 4000 + 51389 bytes) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730bb00 onodes 0x558c54d2c500 shared_blobs 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.extentmap(0x558c54d2c608) update 4#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore.extentmap(0x558c54d2c608) update inline shard 117 bytes from 6 extents 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:a9dea763:::smithi13140796-12:head# is 608 (485 bytes onode + 2 bytes spanning blobs + 121 bytes inline extents) 2019-12-07T19:34:17.681+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730bb00 allocated 0x[6e2000~c000] released 0x[] 2019-12-07T19:34:17.681+0000 7f8f06185700 10 freelist allocate 0x6e2000~c000 2019-12-07T19:34:17.681+0000 7f8f06185700 20 freelist _xor first_key 0x680000 last_key 0x680000 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 prepare 2019-12-07T19:34:17.681+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730bb00 2019-12-07T19:34:17.681+0000 7f8f06185700 10 osd.7 40 dequeue_op 0x558c5740c000 finish 2019-12-07T19:34:17.681+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.681+0000 7f8f06185700 20 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=2 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.681+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.681+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 aio_wait 2019-12-07T19:34:17.681+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730bb00 2019-12-07T19:34:17.681+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 io_done 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.681+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1931 < min_flush_size 524288 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c500 had 1 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:17.681+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1954 < min_flush_size 524288 2019-12-07T19:34:17.681+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1954 < min_flush_size 524288 2019-12-07T19:34:17.681+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.681+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1086e2~7a2 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.681+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1086e2 0x1086e2~7a2 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd86e2 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6e2 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.681+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe84 and padding block with 0x17c 2019-12-07T19:34:17.682+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x108e84 2019-12-07T19:34:17.682+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.682+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000248026s (0.000041467s flush + 0.000206559s kv commit) 2019-12-07T19:34:17.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730bb00> 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 kv_submitted 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730bb00 2019-12-07T19:34:17.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 finishing 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730bb00 onodes 0x558c54d2c500 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efdd68 in 0x558c53a59180) _finish_write discard buffer(0x558c54e14630 space 0x558c57efdd68 0x0~2000 writing nocache) 2019-12-07T19:34:17.682+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3b898 in 0x558c53a59180) _finish_write discard buffer(0x558c57a09320 space 0x558c57f3b898 0x0~2000 writing nocache) 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3b978 in 0x558c53a59180) _finish_write discard buffer(0x558c57a093b0 space 0x558c57f3b978 0x0~2000 writing nocache) 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5898 in 0x558c53a59180) _finish_write discard buffer(0x558c57a09ef0 space 0x558c580a5898 0x0~2000 writing nocache) 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5908 in 0x558c53a59180) _finish_write discard buffer(0x558c57a09440 space 0x558c580a5908 0x0~2000 writing nocache) 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5b38 in 0x558c53a59180) _finish_write discard buffer(0x558c580cdb00 space 0x558c580a5b38 0x0~2000 writing nocache) 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730bb00 done 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:17.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730bb00 [] 2019-12-07T19:34:17.682+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.682+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 0'0 active m=2 mbc={} ps=[a~1]] recovery_committed_to version 21'3 now ondisk 2019-12-07T19:34:17.682+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDPGPushReply(1.5s0 40/39 [PushReplyOp(1:a9dea763:::smithi13140796-12:head)]) v3 -- 0x558c575b8c00 con 0x558c54c19800 2019-12-07T19:34:17.682+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.685+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 227 ==== MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=182, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 249+0+0 (crc 0 0 0) 0x558c574a0780 con 0x558c54c64000 2019-12-07T19:34:17.685+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c54e79760 prio 3 cost 0 latency 0.000023 epoch 40 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=182, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.685+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=182, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=182, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=182, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c54e79760 prio 3 cost 0 latency 0.000059 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=182, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=182, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.685+0000 7f8f0117b700 25 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xc00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xc00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e579d0 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57500 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57960 blob([0xaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e578f0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x4400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x4400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x4c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x4c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x5400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x5400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x5c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e577a0 blob([0xb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56070 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x5c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x6400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x6400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x6c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x6c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x7400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x7400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x7c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e573b0 blob([0xb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d570 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x7c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x8400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x8400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x8c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x8c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x9400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x9400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x9c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d500 blob([0xb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d490 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x9c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xa400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xa400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xac00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xac00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xac00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xb400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xb400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xbc00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d420 blob([0xb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d3b0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xbc00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xc400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xc400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xcc00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xcc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xcc00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xd400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xd400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xdc00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xdc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57340 blob([0xb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56620 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xdc00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xe400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xe400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xec00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xec00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xec00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xf400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xf400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xfc00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57810 blob([0xbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57ab0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0xfc00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x10400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x10400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x10c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x10c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x11400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x11400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x11c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57b20 blob([0xbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57b90 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x11c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x12400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x12400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x12c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x12c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x13400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x13400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x13c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57c00 blob([0xbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57c70 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x13c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x14400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x14400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x14c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x14c00~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x15400~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15400~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x15400~400 = 1024 2019-12-07T19:34:17.685+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x15c00~400 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.685+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ce0 blob([0xc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57d50 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.685+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x15c00~400 = 1024 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x16400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x16400~400 = 1024 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x16c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x16c00~400 = 1024 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x17400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x17400~400 = 1024 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x17c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57dc0 blob([0xc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e57e30 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x17c00~400 = 1024 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x18400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e57f10 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x18400~400 = 1024 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x18c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e57ea0 blob([0xc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e57f10 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x18c00~400 = 1024 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x19400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x19400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x19c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x19c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1a400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1a400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1ac00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ac00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1ac00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1b400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1b400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1bc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1bc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1bc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1c400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1c400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1cc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1cc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1cc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1d400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1d400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1dc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1dc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1dc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1e400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1e400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1ec00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ec00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1ec00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1f400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1f400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1fc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x1fc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x20400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x20400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x20c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x20c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x21400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x21400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x21c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x21c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x22400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x22400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x22c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x22c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x23400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x23400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x23c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x23c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x24400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x24400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x24c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x24c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x25400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x25400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x25c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x25c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x26400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x26400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x26c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x26c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x27400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x27400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x27c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x27c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x28400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x28400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x28c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x28c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x29400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x29400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x29c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x29c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2a400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2a400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2ac00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ac00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2ac00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2b400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2b400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2bc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2bc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2bc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2c400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2c400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2cc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2cc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2cc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2d400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2d400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2dc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2dc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2dc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2e400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2e400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2ec00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ec00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2ec00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2f400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2f400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2fc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x2fc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x30400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x30400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x30c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x30c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x31400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x31400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x31c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x31c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x32400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x32400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x32c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x32c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x33400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x33400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x33c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x33c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x34400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x34400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x34c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x34c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x35400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x35400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x35c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x35c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x36400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x36400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x36c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x36c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x37400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x37400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x37c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x37c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x38400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x38400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x38c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x38c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x39400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x39400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x39c00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39c00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x39c00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3a400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3a400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3ac00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ac00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3ac00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3b400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3b400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3bc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3bc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3bc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3c400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3c400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3cc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3cc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3cc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3d400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3d400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3dc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3dc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3dc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3e400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3e400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3ec00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ec00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3ec00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3f400~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f400~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3f400~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3fc00~400 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fc00~400 size 0x19000 (102400) 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 0x3fc00~400 = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=51200 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 102400 2019-12-07T19:34:17.686+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:34:17.686+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 40/39 ECSubReadReply(tid=182, attrs_read=0)) v2 -- 0x558c575bac80 con 0x558c54c64000 2019-12-07T19:34:17.686+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c54e79760 finish 2019-12-07T19:34:17.686+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.687+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 89 ==== MOSDPGPush(1.5s4 40/39 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 99366+0+0 (crc 0 0 0) 0x558c575b8c00 con 0x558c54c19800 2019-12-07T19:34:17.687+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c5740c420 prio 127 cost 99304 latency 0.000025 epoch 40 MOSDPGPush(1.5s4 40/39 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.687+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 99304 e40) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 99304 e40) queued 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 99304 e40) pg 0x558c57522000 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 osd.7 40 dequeue_op 0x558c5740c420 prio 127 cost 99304 latency 0.000069 MOSDPGPush(1.5s4 40/39 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.5s4 40/39 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[a~1]] on_local_recover: 1:a8debf31:::smithi13140796-24:head 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[a~1]] got missing 1:a8debf31:::smithi13140796-24:head v 21'4 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 23'7 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 21'3 active m=1 mbc={} ps=[a~1]] last_complete now 23'7 log.complete_to 27'8 2019-12-07T19:34:17.687+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c577fc000 seq 5 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) get_onode oid 4#1:a8debf31:::smithi13140796-24:head# key 0x848000000000000001a8debf'1!smithi13140796-24!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) r -2 v.len 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) get_onode oid 4#1:a8debf31:::smithi13140796-24:head# key 0x848000000000000001a8debf'1!smithi13140796-24!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) r -2 v.len 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1191 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# = 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 0x0~18000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:a8debf31:::smithi13140796-24:head# 0x0~18000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~18000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fc000 12 blobs 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate 0x18000/1000,18000,0 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x6ee000~18000/1000,18000,0 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x6ee000~18000] 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757ec40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e56770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757ec40 blob([0x6ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56770 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c5757ec40) get_ref 0x0~2000 Blob(0x558c5757ec40 blob([0x6ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56770 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c5757ec40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c5757ec40 blob([0x6ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56770 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c54e56788 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a07c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264930 blob([0x6f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07c70 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c55264930) get_ref 0x0~2000 Blob(0x558c55264930 blob([0x6f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07c70 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c55264930) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c55264930 blob([0x6f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07c70 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57a07c88 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f15e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b420 blob([0x6f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f15e0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c5742b420) get_ref 0x0~2000 Blob(0x558c5742b420 blob([0x6f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f15e0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c5742b420) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c5742b420 blob([0x6f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f15e0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574f15f8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f916c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f916c0 blob([0x6f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91730 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f916c0) get_ref 0x0~2000 Blob(0x558c57f916c0 blob([0x6f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91730 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f916c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57f916c0 blob([0x6f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91730 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91748 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f917a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f917a0 blob([0x6f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91810 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f917a0) get_ref 0x0~2000 Blob(0x558c57f917a0 blob([0x6f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91810 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f917a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57f917a0 blob([0x6f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91810 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91828 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f91880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f918f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f91880 blob([0x6f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f918f0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91880) get_ref 0x0~2000 Blob(0x558c57f91880 blob([0x6f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f918f0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91880) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57f91880 blob([0x6f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f918f0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91908 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f91960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f919d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f91960 blob([0x6fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f919d0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91960) get_ref 0x0~2000 Blob(0x558c57f91960 blob([0x6fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f919d0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91960) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57f91960 blob([0x6fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f919d0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f919e8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f91a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f91a40 blob([0x6fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91ab0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91a40) get_ref 0x0~2000 Blob(0x558c57f91a40 blob([0x6fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91ab0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91a40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57f91a40 blob([0x6fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91ab0 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91ac8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f91b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f91b20 blob([0x6fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91b90 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91b20) get_ref 0x0~2000 Blob(0x558c57f91b20 blob([0x6fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91b90 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91b20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57f91b20 blob([0x6fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91b90 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91ba8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f91c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f91c00 blob([0x700000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91c70 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91c00) get_ref 0x0~2000 Blob(0x558c57f91c00 blob([0x700000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91c70 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91c00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57f91c00 blob([0x700000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91c70 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91c88 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f91ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f91ce0 blob([0x702000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91d50 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91ce0) get_ref 0x0~2000 Blob(0x558c57f91ce0 blob([0x702000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91d50 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57f91ce0 blob([0x702000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91d50 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91d68 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f91dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f91dc0 blob([0x704000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91e30 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91dc0) get_ref 0x0~2000 Blob(0x558c57f91dc0 blob([0x704000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91e30 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57f91dc0 blob([0x704000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91e30 sbid 0x0)) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f91e48 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x18000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e44d88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 0x0~18000 = 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 4 keys 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 4 keys = 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049a'.missing/0000000000000001.51B7DFC8.head.smithi13140796-24..' <- missing/0000000000000001.51B7DFC8.head.smithi13140796-24.. 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc000 cost 152541 (13 ios * 4000 + 100541 bytes) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc000 onodes 0x558c54e44c80 shared_blobs 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e44d88) update 4#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e44d88) update inline shard 231 bytes from 12 extents 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:a8debf31:::smithi13140796-24:head# is 722 (485 bytes onode + 2 bytes spanning blobs + 235 bytes inline extents) 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc000 allocated 0x[6ee000~18000] released 0x[] 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 freelist allocate 0x6ee000~18000 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 freelist _xor first_key 0x680000 last_key 0x700000 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 prepare 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fc000 2019-12-07T19:34:17.687+0000 7f8f0217d700 10 osd.7 40 dequeue_op 0x558c5740c420 finish 2019-12-07T19:34:17.687+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.687+0000 7f8f06185700 20 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 23'7 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 21'3 active m=1 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.687+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.687+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 aio_wait 2019-12-07T19:34:17.687+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc000 2019-12-07T19:34:17.687+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 io_done 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2074 < min_flush_size 524288 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44c80 had 1 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2097 < min_flush_size 524288 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2097 < min_flush_size 524288 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x108e84~831 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x108e84 0x108e84~831 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd8e84 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe84 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6b5 and padding block with 0x94b 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1096b5 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.688+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000167575s (0.000039319s flush + 0.000128256s kv commit) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.688+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc000> 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.688+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 kv_submitted 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc000 2019-12-07T19:34:17.688+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 finishing 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc000 onodes 0x558c54e44c80 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56788 in 0x558c53a59180) _finish_write discard buffer(0x558c573d8090 space 0x558c54e56788 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f15f8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86000 space 0x558c574f15f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a07c88 in 0x558c53a59180) _finish_write discard buffer(0x558c573d8000 space 0x558c57a07c88 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91748 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86090 space 0x558c57f91748 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91828 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86120 space 0x558c57f91828 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91908 in 0x558c53a59180) _finish_write discard buffer(0x558c57b861b0 space 0x558c57f91908 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f919e8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86240 space 0x558c57f919e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91ac8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b862d0 space 0x558c57f91ac8 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91ba8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86360 space 0x558c57f91ba8 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91c88 in 0x558c53a59180) _finish_write discard buffer(0x558c57b863f0 space 0x558c57f91c88 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91d68 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86480 space 0x558c57f91d68 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91e48 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86510 space 0x558c57f91e48 0x0~2000 writing nocache) 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc000 done 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:17.688+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc000 [] 2019-12-07T19:34:17.688+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.688+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 23'7 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 21'3 active m=1 mbc={} ps=[a~1]] recovery_committed_to version 23'7 now ondisk 2019-12-07T19:34:17.688+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.688+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDPGPushReply(1.5s0 40/39 [PushReplyOp(1:a8debf31:::smithi13140796-24:head)]) v3 -- 0x558c5763d200 con 0x558c54c19800 2019-12-07T19:34:17.688+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.694+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 228 ==== MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=183, to_read={1:ec1285ab:::smithi13140796-19:10=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:10=[2,2,6,2]}, attrs_to_read=)) v3 ==== 249+0+0 (crc 0 0 0) 0x558c575bac80 con 0x558c54c64000 2019-12-07T19:34:17.694+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57335080 prio 3 cost 0 latency 0.000027 epoch 40 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=183, to_read={1:ec1285ab:::smithi13140796-19:10=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:10=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.694+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=183, to_read={1:ec1285ab:::smithi13140796-19:10=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:10=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.694+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=183, to_read={1:ec1285ab:::smithi13140796-19:10=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:10=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.694+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.694+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=183, to_read={1:ec1285ab:::smithi13140796-19:10=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:10=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.694+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c57335080 prio 3 cost 0 latency 0.000072 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=183, to_read={1:ec1285ab:::smithi13140796-19:10=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:10=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=183, to_read={1:ec1285ab:::smithi13140796-19:10=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:10=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.694+0000 7f8f05183700 25 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xc00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xc00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x4400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x4400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x4c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x4c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x5400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x5400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x5c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x5c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x6400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x6400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x6c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x6c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x7400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x7400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x7c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x7c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x8400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x8400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x8c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x8c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x9400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x9400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x9c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x9c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xa400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xa400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xac00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xac00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xac00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xb400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xb400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xbc00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xbc00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xc400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xc400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xcc00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xcc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xcc00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xd400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xd400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xdc00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xdc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xdc00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xe400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xe400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xec00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xec00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xec00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xf400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xf400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xfc00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0xfc00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x10400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x10400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x10c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x10c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x11400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x11400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x11c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x11c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x12400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567650 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x12400~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x12c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567650 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x12c00~400 = 1024 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x13400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x13400~400 = 0 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x13c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x13c00~400 = 0 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x14400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x14400~400 = 0 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x14c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x14c00~400 = 0 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x15400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x15400~400 = 0 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x15c00~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x15c00~400 = 0 2019-12-07T19:34:17.694+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x16400~400 2019-12-07T19:34:17.694+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16400~400 size 0x13000 (77824) 2019-12-07T19:34:17.694+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x16400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x16c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x16c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x17400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x17400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x17c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x17c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x18400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x18400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x18c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x18c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x19400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x19400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x19c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x19c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1a400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1a400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1ac00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ac00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1ac00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1b400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1b400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1bc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1bc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1bc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1c400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1c400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1cc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1cc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1cc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1d400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1d400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1dc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1dc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1dc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1e400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1e400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1ec00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ec00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1ec00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1f400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1f400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1fc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x1fc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x20400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x20400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x20c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x20c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x21400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x21400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x21c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x21c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x22400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x22400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x22c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x22c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x23400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x23400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x23c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x23c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x24400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x24400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x24c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x24c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x25400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x25400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x25c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x25c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x26400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x26400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x26c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x26c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x27400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x27400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x27c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x27c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x28400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x28400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x28c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x28c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x29400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x29400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x29c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x29c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2a400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2a400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2ac00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ac00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2ac00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2b400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2b400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2bc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2bc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2bc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2c400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2c400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2cc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2cc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2cc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2d400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2d400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2dc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2dc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2dc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2e400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2e400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2ec00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ec00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2ec00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2f400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2f400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2fc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x2fc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x30400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x30400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x30c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x30c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x31400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x31400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x31c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x31c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x32400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x32400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x32c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x32c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x33400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x33400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x33c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x33c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x34400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x34400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x34c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x34c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x35400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x35400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x35c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x35c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x36400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x36400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x36c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x36c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x37400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x37400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x37c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x37c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x38400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x38400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x38c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x38c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x39400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x39400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x39c00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39c00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x39c00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3a400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3a400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3ac00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ac00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3ac00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3b400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3b400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3bc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3bc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3bc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3c400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3c400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3cc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3cc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3cc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3d400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3d400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3dc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3dc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3dc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3e400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3e400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3ec00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ec00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3ec00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3f400~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f400~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3f400~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3fc00~400 2019-12-07T19:34:17.695+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fc00~400 size 0x13000 (77824) 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 0x3fc00~400 = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=38912 2019-12-07T19:34:17.695+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:10 2019-12-07T19:34:17.695+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:10 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 2019-12-07T19:34:17.695+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:17.695+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key 2019-12-07T19:34:17.695+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key = 0 2019-12-07T19:34:17.695+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 40/39 ECSubReadReply(tid=183, attrs_read=0)) v2 -- 0x558c57f35180 con 0x558c54c64000 2019-12-07T19:34:17.695+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c57335080 finish 2019-12-07T19:34:17.695+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.698+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 90 ==== MOSDPGPush(1.5s4 40/39 [PushOp(1:a7686634:::smithi13140796-35:head, version: 27'8, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a7686634:::smithi13140796-35:head@27'8, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 177190+0+0 (crc 0 0 0) 0x558c5763d200 con 0x558c54c19800 2019-12-07T19:34:17.698+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c5740c6e0 prio 127 cost 177128 latency 0.000034 epoch 40 MOSDPGPush(1.5s4 40/39 [PushOp(1:a7686634:::smithi13140796-35:head, version: 27'8, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a7686634:::smithi13140796-35:head@27'8, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.698+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a7686634:::smithi13140796-35:head, version: 27'8, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a7686634:::smithi13140796-35:head@27'8, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e40) 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a7686634:::smithi13140796-35:head, version: 27'8, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a7686634:::smithi13140796-35:head@27'8, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e40) queued 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 40/39 [PushOp(1:a7686634:::smithi13140796-35:head, version: 27'8, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a7686634:::smithi13140796-35:head@27'8, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e40) pg 0x558c57522000 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 osd.7 40 dequeue_op 0x558c5740c6e0 prio 127 cost 177128 latency 0.000091 MOSDPGPush(1.5s4 40/39 [PushOp(1:a7686634:::smithi13140796-35:head, version: 27'8, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a7686634:::smithi13140796-35:head@27'8, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.5s4( v 27'8 lc 23'7 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 23'7 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.5s4 40/39 [PushOp(1:a7686634:::smithi13140796-35:head, version: 27'8, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a7686634:::smithi13140796-35:head@27'8, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 23'7 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[a~1]] on_local_recover: 1:a7686634:::smithi13140796-35:head 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 lc 23'7 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[a~1]] got missing 1:a7686634:::smithi13140796-35:head v 27'8 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 23'7 active mbc={} ps=[a~1]] last_complete now 27'8 log.complete_to at end 2019-12-07T19:34:17.698+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c577fd800 seq 6 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) get_onode oid 4#1:a7686634:::smithi13140796-35:head# key 0x848000000000000001a76866'4!smithi13140796-35!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) r -2 v.len 0 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 4#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) get_onode oid 4#1:a7686634:::smithi13140796-35:head# key 0x848000000000000001a76866'4!smithi13140796-35!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.5s4_head 0x558c54df65a0) r -2 v.len 0 2019-12-07T19:34:17.698+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1192 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# = 0 2019-12-07T19:34:17.698+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 0x0~2b000 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:a7686634:::smithi13140796-35:head# 0x0~2b000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.698+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c577fd800 22 blobs 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.698+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2b000/1000,2b000,0 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x706000~2b000/1000,2b000,0 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x706000~2b000] 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c577f20e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fddc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c577f20e0 blob([0x706000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fddc0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c577f20e0) get_ref 0x0~2000 Blob(0x558c577f20e0 blob([0x706000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fddc0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c577f20e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c577f20e0 blob([0x706000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fddc0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fddd8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f90e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f90e00 blob([0x708000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b570 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c57f90e00) get_ref 0x0~2000 Blob(0x558c57f90e00 blob([0x708000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b570 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c57f90e00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57f90e00 blob([0x708000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b570 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5742b588 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750e3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750e460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750e3f0 blob([0x70a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e460 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e3f0) get_ref 0x0~2000 Blob(0x558c5750e3f0 blob([0x70a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e460 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c5750e3f0 blob([0x70a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750e460 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750e478 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750e4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750e540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750e4d0 blob([0x70c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e540 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e4d0) get_ref 0x0~2000 Blob(0x558c5750e4d0 blob([0x70c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e540 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5750e4d0 blob([0x70c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750e540 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750e558 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750e5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750e620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750e5b0 blob([0x70e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e620 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e5b0) get_ref 0x0~2000 Blob(0x558c5750e5b0 blob([0x70e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e620 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5750e5b0 blob([0x70e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750e620 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750e638 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750e690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750e700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750e690 blob([0x710000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e700 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e690) get_ref 0x0~2000 Blob(0x558c5750e690 blob([0x710000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e700 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5750e690 blob([0x710000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750e700 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750e718 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750e770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750e7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750e770 blob([0x712000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e7e0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e770) get_ref 0x0~2000 Blob(0x558c5750e770 blob([0x712000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e7e0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e770) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5750e770 blob([0x712000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750e7e0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750e7f8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750e850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750e8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750e850 blob([0x714000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e8c0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e850) get_ref 0x0~2000 Blob(0x558c5750e850 blob([0x714000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e8c0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e850) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5750e850 blob([0x714000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750e8c0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750e8d8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750e930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750e9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750e930 blob([0x716000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e9a0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e930) get_ref 0x0~2000 Blob(0x558c5750e930 blob([0x716000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750e9a0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750e930) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5750e930 blob([0x716000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750e9a0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750e9b8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ea10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ea80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ea10 blob([0x718000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ea80 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ea10) get_ref 0x0~2000 Blob(0x558c5750ea10 blob([0x718000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ea80 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ea10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c5750ea10 blob([0x718000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ea80 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750ea98 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750eaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750eb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750eaf0 blob([0x71a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750eb60 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750eaf0) get_ref 0x0~2000 Blob(0x558c5750eaf0 blob([0x71a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750eb60 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750eaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c5750eaf0 blob([0x71a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750eb60 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750eb78 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ebd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ec40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ebd0 blob([0x71c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ec40 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ebd0) get_ref 0x0~2000 Blob(0x558c5750ebd0 blob([0x71c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ec40 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ebd0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c5750ebd0 blob([0x71c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ec40 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750ec58 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ecb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ed20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ecb0 blob([0x71e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ed20 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ecb0) get_ref 0x0~2000 Blob(0x558c5750ecb0 blob([0x71e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ed20 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ecb0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c5750ecb0 blob([0x71e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ed20 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750ed38 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ed90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ee00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ed90 blob([0x720000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ee00 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ed90) get_ref 0x0~2000 Blob(0x558c5750ed90 blob([0x720000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ee00 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ed90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c5750ed90 blob([0x720000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ee00 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750ee18 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ee70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750eee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ee70 blob([0x722000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750eee0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ee70) get_ref 0x0~2000 Blob(0x558c5750ee70 blob([0x722000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750eee0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ee70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c5750ee70 blob([0x722000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750eee0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750eef8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ef50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750efc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ef50 blob([0x724000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750efc0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ef50) get_ref 0x0~2000 Blob(0x558c5750ef50 blob([0x724000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750efc0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ef50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c5750ef50 blob([0x724000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750efc0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750efd8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750f030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750f0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750f030 blob([0x726000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f0a0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f030) get_ref 0x0~2000 Blob(0x558c5750f030 blob([0x726000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f0a0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f030) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c5750f030 blob([0x726000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750f0a0 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750f0b8 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750f110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750f180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750f110 blob([0x728000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f180 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f110) get_ref 0x0~2000 Blob(0x558c5750f110 blob([0x728000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f180 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f110) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c5750f110 blob([0x728000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750f180 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750f198 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750f1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750f260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750f1f0 blob([0x72a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f260 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f1f0) get_ref 0x0~2000 Blob(0x558c5750f1f0 blob([0x72a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f260 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c5750f1f0 blob([0x72a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750f260 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750f278 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750f2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750f340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750f2d0 blob([0x72c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f340 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f2d0) get_ref 0x0~2000 Blob(0x558c5750f2d0 blob([0x72c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f340 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c5750f2d0 blob([0x72c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750f340 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750f358 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750f3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750f420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750f3b0 blob([0x72e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f420 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f3b0) get_ref 0x0~2000 Blob(0x558c5750f3b0 blob([0x72e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f420 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c5750f3b0 blob([0x72e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750f420 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750f438 in 0x558c53a59180) _discard 0x0~2000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750f490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750f500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750f490 blob([0x730000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f500 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f490) get_ref 0x0~1000 Blob(0x558c5750f490 blob([0x730000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750f500 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.blob(0x558c5750f490) get_ref init 0x1000, 1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~1000: 0x0~1000 Blob(0x558c5750f490 blob([0x730000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5750f500 sbid 0x0)) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5750f518 in 0x558c53a59180) _discard 0x0~1000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2b000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57f8fa08) dirty_range mark inline shard dirty 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 0x0~2b000 = 0 2019-12-07T19:34:17.699+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 4 keys 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 4 keys = 0 2019-12-07T19:34:17.699+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049a'.missing/0000000000000001.5E6166C2.head.smithi13140796-35..' <- missing/0000000000000001.5E6166C2.head.smithi13140796-35.. 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.699+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd800 cost 270365 (23 ios * 4000 + 178365 bytes) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd800 onodes 0x558c57f8f900 shared_blobs 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57f8fa08) update 4#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57f8fa08) update inline shard 418 bytes from 22 extents 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:a7686634:::smithi13140796-35:head# is 909 (485 bytes onode + 2 bytes spanning blobs + 422 bytes inline extents) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd800 allocated 0x[706000~2b000] released 0x[] 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 freelist allocate 0x706000~2b000 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 freelist _xor first_key 0x700000 last_key 0x700000 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 prepare 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c577fd800 2019-12-07T19:34:17.699+0000 7f8f0217d700 10 osd.7 40 dequeue_op 0x558c5740c6e0 finish 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.699+0000 7f8f06185700 20 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 23'7 active mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.699+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.699+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 aio_wait 2019-12-07T19:34:17.699+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd800 2019-12-07T19:34:17.699+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 io_done 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2232 < min_flush_size 524288 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8f900 had 1 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2255 < min_flush_size 524288 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2255 < min_flush_size 524288 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1096b5~8cf to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1096b5 0x1096b5~8cf to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd96b5 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6b5 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf84 and padding block with 0x7c 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x109f84 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.699+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000139824s (0.000034970s flush + 0.000104854s kv commit) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.699+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd800> 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.699+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 kv_submitted 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd800 2019-12-07T19:34:17.699+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 finishing 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd800 onodes 0x558c57f8f900 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b588 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86750 space 0x558c5742b588 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fddd8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b866c0 space 0x558c574fddd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750e478 in 0x558c53a59180) _finish_write discard buffer(0x558c57b867e0 space 0x558c5750e478 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750e558 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86870 space 0x558c5750e558 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750e638 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86900 space 0x558c5750e638 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750e718 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86990 space 0x558c5750e718 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750e7f8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86a20 space 0x558c5750e7f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750e8d8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86ab0 space 0x558c5750e8d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750e9b8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86b40 space 0x558c5750e9b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ea98 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86bd0 space 0x558c5750ea98 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 lcod 23'7 active mbc={} ps=[a~1]] recovery_committed_to version 27'8 now ondisk 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750eb78 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86c60 space 0x558c5750eb78 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ec58 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86cf0 space 0x558c5750ec58 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ed38 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86d80 space 0x558c5750ed38 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ee18 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86e10 space 0x558c5750ee18 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750eef8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86ea0 space 0x558c5750eef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750efd8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86f30 space 0x558c5750efd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750f0b8 in 0x558c53a59180) _finish_write discard buffer(0x558c57b86fc0 space 0x558c5750f0b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_trim(1.5s0 27'8 e40/40) v2 -- 0x558c54d9ef00 con 0x558c54c19800 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750f198 in 0x558c53a59180) _finish_write discard buffer(0x558c57b87050 space 0x558c5750f198 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750f278 in 0x558c53a59180) _finish_write discard buffer(0x558c57b870e0 space 0x558c5750f278 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750f358 in 0x558c53a59180) _finish_write discard buffer(0x558c57b87170 space 0x558c5750f358 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750f438 in 0x558c53a59180) _finish_write discard buffer(0x558c57b87200 space 0x558c5750f438 0x0~2000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750f518 in 0x558c53a59180) _finish_write discard buffer(0x558c57b87290 space 0x558c5750f518 0x0~1000 writing nocache) 2019-12-07T19:34:17.699+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDPGPushReply(1.5s0 40/39 [PushReplyOp(1:a7686634:::smithi13140796-35:head)]) v3 -- 0x558c575b8c00 con 0x558c54c19800 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd800 done 2019-12-07T19:34:17.699+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:17.699+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.699+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd800 [] 2019-12-07T19:34:17.700+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.700+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.700+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 229 ==== MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=184, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 249+0+0 (crc 0 0 0) 0x558c57f35180 con 0x558c54c64000 2019-12-07T19:34:17.700+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3ec60 prio 3 cost 0 latency 0.000018 epoch 40 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=184, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.700+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=184, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=184, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=184, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3ec60 prio 3 cost 0 latency 0.000061 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=184, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=184, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.700+0000 7f8f0117b700 25 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xc00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xc00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d030 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d030 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d030 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d030 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3ccb0 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265a40 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265a40 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265a40 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265a40 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x8400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43030 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x8400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x8c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43030 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x8c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x9400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43030 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x9400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x9c00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43030 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x9c00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xa400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43110 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xa400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xac00~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43110 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xac00~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xb400~400 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.700+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43110 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.700+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xb400~400 = 1024 2019-12-07T19:34:17.700+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xbc00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43110 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xbc00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xc400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e431f0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xc400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xcc00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xcc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e431f0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xcc00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xd400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e431f0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xd400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xdc00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xdc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e431f0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xdc00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xe400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e432d0 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xe400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xec00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e432d0 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xec00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xf400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e432d0 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xf400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xfc00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e432d0 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0xfc00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x10400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e433b0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x10400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x10c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e433b0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x10c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x11400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e433b0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x11400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x11c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e433b0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x11c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x12400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43490 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x12400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x12c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e43490 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x12c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x13400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need {<0x0, 0x1000> : [0x13400:400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.701+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 91 ==== MRecoveryReserve(1.5s4 RELEASE e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54d9ef00 con 0x558c54c19800 2019-12-07T19:34:17.701+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.5s4 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.701+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c57522000 2019-12-07T19:34:17.701+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.701+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.702506+0000 1 0.000030 2019-12-07T19:34:17.701+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.701+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.701+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x13400:400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57567cf8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c44c60 space 0x558c57567cf8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x13400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x13c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x13c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x14400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need {<0x1000, 0x1000> : [0x14400:1400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.701+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 132 ==== MRecoveryReserve(1.4s0 GRANT e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c576dcb40 con 0x558c54c5bc00 2019-12-07T19:34:17.701+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.701+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x14400:1400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57567cf8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c44cf0 space 0x558c57567cf8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x14400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x14c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.701+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x14c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x15400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need {<0x0, 0x1000> : [0x15400:400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.701+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.4s5 REQUEST prio: 190 e40/40) v3 -- 0x558c575ccd20 con 0x558c54c5b400 2019-12-07T19:34:17.701+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.701+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x15400:400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5742b748 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c44d80 space 0x558c5742b748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x15400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x15c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x15c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x16400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need {<0x1000, 0x1000> : [0x16400:1400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x16400:1400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5742b748 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c44e10 space 0x558c5742b748 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x16400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x16c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x16c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x17400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need {<0x0, 0x1000> : [0x17400:400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.701+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 162 ==== MRecoveryReserve(1.4s0 GRANT e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c575ccd20 con 0x558c54c5b400 2019-12-07T19:34:17.701+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.701+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.701+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.701+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.701+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.701+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.701+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.701+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:17.701+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MRecoveryReserve(1.4s3 REQUEST prio: 190 e40/40) v3 -- 0x558c54d9eb40 con 0x558c54c19c00 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x17400:400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5742b908 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.701+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c44ea0 space 0x558c5742b908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.701+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x17400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x17c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x17c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x18400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x18400:1400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x18400:1400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5742b908 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c44f30 space 0x558c5742b908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x18400~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x18c00~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.701+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x18c00~400 = 1024 2019-12-07T19:34:17.701+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x19400~400 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need {<0x0, 0x1000> : [0x19400:400~400]} 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.701+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 69 ==== MRecoveryReserve(1.4s0 GRANT e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54d9eb40 con 0x558c54c19c00 2019-12-07T19:34:17.702+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.702+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x19400:400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5742b7b8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c44fc0 space 0x558c5742b7b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x19400~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x19c00~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x19c00~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1a400~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] exit Started/Primary/Active/WaitRemoteRecoveryReserved 2019-12-07T19:34:17.703307+0000 5 0.000149 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1a400:1400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.702+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] enter Started/Primary/Active/Recovering 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] queue_recovery -- queuing 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) 2019-12-07T19:34:17.702+0000 7f8f0297e700 10 osd.7 40 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) queued 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1a400:1400~400]} 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5742b7b8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c45050 space 0x558c5742b7b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1a400~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1ac00~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1ac00~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1b400~400 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need {<0x0, 0x1000> : [0x1b400:400~400]} 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 0 ml 2 missing shards 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1 ml 2 missing shards 1 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2 ml 2 missing shards 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3 ml 2 missing shards 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4 ml 2 missing shards 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5 ml 2 missing shards 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based degraded 2 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.702+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:70 2019-12-07T19:34:17.702+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) pg 0x558c548ab000 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 40 do_recovery starting 1 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1b400:400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c578977b8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c450e0 space 0x558c578977b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recover_replicas(1) 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1b400~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1bc00~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1bc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] peer osd.3(1) missing 2 objects. 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1bc00~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1c400~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1c400:1400~400]} 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] peer osd.3(1) missing {1:276d5164:::smithi13140796-22:e=37'3 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:276d5164:::smithi13140796-22:head=37'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recover_replicas: recover_object_replicas(1:276d5164:::smithi13140796-22:e) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prep_object_replica_pushes: on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_object_context: obc NOT found in cache: 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# _ 2019-12-07T19:34:17.702+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# _ = 0 2019-12-07T19:34:17.702+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# snapset 2019-12-07T19:34:17.702+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# snapset = 0 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] populate_obc_watchers 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:17.702+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# = 0 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_object_context: creating obc from disk: 0x558c57570fc0 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1c400:1400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c578977b8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c45170 space 0x558c578977b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_object_context: 0x558c57570fc0 1:276d5164:::smithi13140796-22:e rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:e(37'3 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1c400~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1cc00~400 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recovery got recovery read lock on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1cc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] start_recovery_op 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1cc00~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1d400~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 40 start_recovery_op pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:e (0/10 rops) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need {<0x0, 0x1000> : [0x1d400:400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] checking 1(2) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] checking 2(4) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] checking 3(1) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] checking 4(5) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] checking 5(3) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] checking 7(0) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1d400:400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c578976d8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] recover_object: built op RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=5 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57c45200 space 0x558c578976d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1d400~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1dc00~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1dc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] run_recovery_op: starting RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1dc00~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1e400~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1e400:1400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.702+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_hash_info: found on disk, size 90112 2019-12-07T19:34:17.702+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key 2019-12-07T19:34:17.702+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key = 0 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 4(5) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 5(3) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1e400:1400~400]} 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 1(2) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c578976d8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87200 space 0x558c578976d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 2(4) 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1e400~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 4(5) 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1ec00~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 5(3) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 7(0) 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1ec00~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1f400~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need {<0x0, 0x1000> : [0x1f400:400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] continue_recovery_op: IDLE return RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] start_read_op: starting ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_read_op: starting read ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:17.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.4s2 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 -- 0x558c5738ca00 con 0x558c54c64000 2019-12-07T19:34:17.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpRead(1.4s4 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 -- 0x558c5738cc80 con 0x558c54c19800 2019-12-07T19:34:17.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.4s5 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 -- 0x558c57cca000 con 0x558c54c5b400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1f400:400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c578975f8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.4s3 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 -- 0x558c57cca280 con 0x558c54c19c00 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b875f0 space 0x558c578975f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 40 send_incremental_map 38 -> 40 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1f400~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1fc00~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(39..40 src has 1..40) v4 -- 0x558c57570b40 con 0x558c53ae5400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 40 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 38 -> 40 (shared) 2019-12-07T19:34:17.702+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x1fc00~400 = 1024 2019-12-07T19:34:17.702+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x20400~400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.702+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 -- 0x558c57cca500 con 0x558c53ae5400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x20400:1400~400]} 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.702+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c57cca500 con 0x558c53ae5400 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f1119b700 15 osd.7 40 enqueue_op 0x558c57efe2c0 prio 3 cost 0 latency 0.000014 epoch 40 MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 2019-12-07T19:34:17.702+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.702+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(39..40 src has 1..40) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c57570b40 con 0x558c53ae5400 2019-12-07T19:34:17.702+0000 7f8f1199c700 20 osd.7 40 OSD::ms_dispatch: osd_map(39..40 src has 1..40) v4 2019-12-07T19:34:17.702+0000 7f8f1199c700 10 osd.7 40 do_waiters -- start 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.702+0000 7f8f1199c700 10 osd.7 40 do_waiters -- finish 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] do_read_op: started ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:e=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:e,2(4)=1:276d5164:::smithi13140796-22:e,4(5)=1:276d5164:::smithi13140796-22:e,5(3)=1:276d5164:::smithi13140796-22:e,7(0)=1:276d5164:::smithi13140796-22:e}, in_progress=1(2),2(4),4(5),5(3),7(0)) 2019-12-07T19:34:17.702+0000 7f8f1199c700 20 osd.7 40 _dispatch 0x558c57570b40 osd_map(39..40 src has 1..40) v4 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] started 1 2019-12-07T19:34:17.702+0000 7f8f1199c700 3 osd.7 40 handle_osd_map epochs [39,40], i have 40, src has [1,40] 2019-12-07T19:34:17.702+0000 7f8f1199c700 10 osd.7 40 no new maps here, dropping 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 40 do_recovery started 1/1 on pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.702+0000 7f8f06986700 10 osd.7 40 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:17.702+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.702+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.702+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c57efe2c0 prio 3 cost 0 latency 0.000087 MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.702+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=24, to_read={1:276d5164:::smithi13140796-22:e=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:e=[4,4]}, attrs_to_read=)) v3 2019-12-07T19:34:17.702+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x20400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c578975f8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87680 space 0x558c578975f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x20400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x20c00~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x20c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x21400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need {<0x0, 0x1000> : [0x21400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.702+0000 7f8f0297e700 25 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x800~800 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1)))) need 0x800~800 cache has 0x[800~800] 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1800~800 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1)))) need {<0x1000, 0x1000> : [0x1800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x21400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57897518 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87710 space 0x558c57897518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x21400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x21c00~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x21c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x22400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need {<0x1000, 0x1000> : [0x22400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x1800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e568d8 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x22400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c580cc870 space 0x558c54e568d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57897518 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b877a0 space 0x558c57897518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x22400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x22c00~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2800~800 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x22c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x23400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) need {<0x0, 0x1000> : [0x2800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need {<0x0, 0x1000> : [0x23400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x23400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a59e8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87830 space 0x558c580a59e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) need 0x{<0x0, 0x1000> : [0x2800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x23400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x23c00~400 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56b78 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c580cc900 space 0x558c54e56b78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x23c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x24400~400 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3800~800 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x24400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) need {<0x1000, 0x1000> : [0x3800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x3800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56b78 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57a090e0 space 0x558c54e56b78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x4800~800 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) need {<0x0, 0x1000> : [0x4800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x24400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a59e8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b878c0 space 0x558c580a59e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x24400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x24c00~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x24c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x25400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need {<0x0, 0x1000> : [0x25400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x25400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) need 0x{<0x0, 0x1000> : [0x4800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56a28 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54e14b40 space 0x558c54e56a28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x4800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x5800~800 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) need {<0x1000, 0x1000> : [0x5800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x25400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a56d8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87950 space 0x558c580a56d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x25400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x25c00~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x25c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x26400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x26400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x5800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56a28 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57a09e60 space 0x558c54e56a28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x5800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x6800~800 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) need {<0x0, 0x1000> : [0x6800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x26400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a56d8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b879e0 space 0x558c580a56d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x26400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x26c00~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x26c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x27400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need {<0x0, 0x1000> : [0x27400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x27400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) need 0x{<0x0, 0x1000> : [0x6800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56b08 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57a09200 space 0x558c54e56b08 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x6800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x7800~800 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) need {<0x1000, 0x1000> : [0x7800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x27400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a55f8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87a70 space 0x558c580a55f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x27400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x27c00~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x27c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x28400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x7800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x28400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56b08 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57a09290 space 0x558c54e56b08 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x7800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x8800~800 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) need {<0x0, 0x1000> : [0x8800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) need 0x{<0x0, 0x1000> : [0x8800:800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56408 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x28400:1400~400]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57b87cb0 space 0x558c54e56408 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a55f8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x8800~800 = 2048 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87b00 space 0x558c580a55f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.703+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x9800~800 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9800~800 size 0x16000 (90112) 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x28400~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x28c00~400 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) need {<0x1000, 0x1000> : [0x9800:1800~800]} 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.703+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x28c00~400 = 1024 2019-12-07T19:34:17.703+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x29400~400 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.703+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need {<0x0, 0x1000> : [0x29400:400~400]} 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x29400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.703+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x29400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a57b8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87b90 space 0x558c580a57b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x9800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x29400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56408 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x29c00~400 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57b87d40 space 0x558c54e56408 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x9800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xa800~800 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x29c00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2a400~400 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) need {<0x0, 0x1000> : [0xa800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x2a400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) need 0x{<0x0, 0x1000> : [0xa800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2a400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e42b08 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a57b8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57b87dd0 space 0x558c54e42b08 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xa800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57b87c20 space 0x558c580a57b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xb800~800 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2a400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2ac00~400 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) need {<0x1000, 0x1000> : [0xb800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2ac00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2b400~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need {<0x0, 0x1000> : [0x2b400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2b400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 722543 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) need 0x{<0x1000, 0x1000> : [0xb800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e42b08 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57b87e60 space 0x558c54e42b08 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xb800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xc800~800 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) need {<0x0, 0x1000> : [0xc800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2b400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a5c88 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc000 space 0x558c580a5c88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2b400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2bc00~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2bc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2bc00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) need 0x{<0x0, 0x1000> : [0xc800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2c400~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e43d68 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57b87ef0 space 0x558c54e43d68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need {<0x1000, 0x1000> : [0x2c400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xc800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xd800~800 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) need {<0x1000, 0x1000> : [0xd800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2c400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a5c88 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc090 space 0x558c580a5c88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2c400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2cc00~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2cc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2cc00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2d400~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need {<0x0, 0x1000> : [0x2d400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2d400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) need 0x{<0x1000, 0x1000> : [0xd800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e43d68 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57ccc7e0 space 0x558c54e43d68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xd800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xe800~800 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) need {<0x0, 0x1000> : [0xe800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2d400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54677978 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc120 space 0x558c54677978 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2d400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2dc00~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2dc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2dc00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2e400~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need {<0x1000, 0x1000> : [0x2e400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) need 0x{<0x0, 0x1000> : [0xe800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e42d38 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57ccc870 space 0x558c54e42d38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xe800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xf800~800 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) need {<0x1000, 0x1000> : [0xf800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2e400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54677978 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc1b0 space 0x558c54677978 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2e400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2ec00~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2ec00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2f400~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need {<0x0, 0x1000> : [0x2f400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2f400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) need 0x{<0x1000, 0x1000> : [0xf800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e42d38 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57ccc900 space 0x558c54e42d38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0xf800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x10800~800 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) need {<0x0, 0x1000> : [0x10800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) need 0x{<0x0, 0x1000> : [0x10800:800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56c58 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57ccc990 space 0x558c54e56c58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x10800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x11800~800 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2f400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a5748 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) need {<0x1000, 0x1000> : [0x11800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc240 space 0x558c580a5748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2f400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2fc00~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2fc00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x30400~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need {<0x1000, 0x1000> : [0x30400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x30400:1400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a5748 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc2d0 space 0x558c580a5748 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x30400~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x30c00~400 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x11800:1800~800]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56c58 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57ccca20 space 0x558c54e56c58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.704+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x30c00~400 = 1024 2019-12-07T19:34:17.704+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x31400~400 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x11800~800 = 2048 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.704+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x12800~800 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need {<0x0, 0x1000> : [0x31400:400~400]} 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x31400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12800~800 size 0x16000 (90112) 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.704+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.704+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) need {<0x0, 0x1000> : [0x12800:800~800]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x31400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43668 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc360 space 0x558c57c43668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x31400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x31c00~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x31c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x32400~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need {<0x1000, 0x1000> : [0x32400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) need 0x{<0x0, 0x1000> : [0x12800:800~800]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56d38 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57cccab0 space 0x558c54e56d38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x12800~800 = 2048 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x13800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) need {<0x1000, 0x1000> : [0x13800:1800~800]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 230 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 ==== 45221+0+0 (crc 0 0 0) 0x558c57cca280 con 0x558c54c64000 2019-12-07T19:34:17.705+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3f8c0 prio 127 cost 0 latency 0.000017 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.705+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x13800:1800~800]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56d38 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57cccb40 space 0x558c54e56d38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x32400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x13800~800 = 2048 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43668 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x14800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc3f0 space 0x558c57c43668 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x32400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x32c00~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) need {<0x0, 0x1000> : [0x14800:800~800]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x32c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x33400~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need {<0x0, 0x1000> : [0x33400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x33400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) need 0x{<0x0, 0x1000> : [0x14800:800~800]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56e18 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57cccbd0 space 0x558c54e56e18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x33400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x14800~800 = 2048 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43518 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x15800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc480 space 0x558c57c43518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x33400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x33c00~400 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) need {<0x1000, 0x1000> : [0x15800:1800~800]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x33c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x34400~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need {<0x1000, 0x1000> : [0x34400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x34400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x34400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43518 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc510 space 0x558c57c43518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x34400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x34c00~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x15800:1800~800]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c54e56e18 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c57cccc60 space 0x558c54e56e18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x15800~800 = 2048 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x34c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x16800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x35400~400 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x16800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x17800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x17800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x18800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x18800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x19800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need {<0x0, 0x1000> : [0x35400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x19800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x35400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1a800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1a800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1b800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1b800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1c800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1c800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1d800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1d800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1e800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1e800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1f800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x1f800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x20800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x20800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x21800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x21800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x22800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x22800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x23800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x23800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x24800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x24800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x25800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x25800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x26800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x26800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x27800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x27800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x28800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x28800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x29800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x29800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2a800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2a800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2b800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2b800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2c800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2c800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2d800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2d800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2e800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2e800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2f800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x2f800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x30800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x30800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x31800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x31800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x32800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x35400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x32800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x33800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43438 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x33800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc5a0 space 0x558c57c43438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x34800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x34800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x35800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x35400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x35800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x35c00~400 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x36800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x36800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x37800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x37800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x38800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x38800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x35c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x39800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x36400~400 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x39800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3a800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3a800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3b800~800 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3b800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3c800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need {<0x1000, 0x1000> : [0x36400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3c800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x36400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3d800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3d800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3e800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3e800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3f800~800 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f800~800 size 0x16000 (90112) 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 0x3f800~800 = 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read read request=262144 r=0 len=45056 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.705+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 -- 0x558c57e8f180 con 0x558c53ae5400 2019-12-07T19:34:17.705+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c57efe2c0 finish 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.705+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c57e8f180 con 0x558c53ae5400 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.705+0000 7f8f1119b700 15 osd.7 40 enqueue_op 0x558c57efe420 prio 127 cost 0 latency 0.000011 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.705+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 prio 127 cost 0 latency 0.000403 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=24, attrs_read=0) 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),45056]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:e=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:e,2(4)=1:276d5164:::smithi13140796-22:e,4(5)=1:276d5164:::smithi13140796-22:e,5(3)=1:276d5164:::smithi13140796-22:e,7(0)=1:276d5164:::smithi13140796-22:e}, in_progress=2(4),4(5),5(3),7(0)) 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 finish 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57efe420 prio 127 cost 0 latency 0.000091 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.705+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 163 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 ==== 45221+0+0 (unknown 0 0 0) 0x558c57cca000 con 0x558c54c5b400 2019-12-07T19:34:17.705+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3e160 prio 127 cost 0 latency 0.000011 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.705+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=24, attrs_read=0) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x36400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43438 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc630 space 0x558c57c43438 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x36400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),45056, 7(0),45056]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:e=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:e,2(4)=1:276d5164:::smithi13140796-22:e,4(5)=1:276d5164:::smithi13140796-22:e,5(3)=1:276d5164:::smithi13140796-22:e,7(0)=1:276d5164:::smithi13140796-22:e}, in_progress=2(4),4(5),5(3)) 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x36c00~400 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57efe420 finish 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x36c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x37400~400 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need {<0x0, 0x1000> : [0x37400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x37400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3e160 prio 127 cost 0 latency 0.000054 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=24, attrs_read=0) 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),45056, 4(5),45056, 7(0),45056]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:e=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:e,2(4)=1:276d5164:::smithi13140796-22:e,4(5)=1:276d5164:::smithi13140796-22:e,5(3)=1:276d5164:::smithi13140796-22:e,7(0)=1:276d5164:::smithi13140796-22:e}, in_progress=2(4),5(3)) 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.705+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x37400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43358 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc6c0 space 0x558c57c43358 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x37400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x37c00~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x37c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x38400~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need {<0x1000, 0x1000> : [0x38400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x38400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x38400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43358 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccc750 space 0x558c57c43358 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x38400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x38c00~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x38c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x39400~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need {<0x0, 0x1000> : [0x39400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x39400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 10 osd.7 40 tick_without_osd_lock 2019-12-07T19:34:17.705+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 92 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 ==== 45221+0+0 (crc 0 0 0) 0x558c5738cc80 con 0x558c54c19800 2019-12-07T19:34:17.705+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c5740c580 prio 127 cost 0 latency 0.000015 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.705+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f8cf000/0x40000000/0x1680000000, data 0x7739a7/0x72f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 25 osd.7 40 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674802+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 25 osd.7 40 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674781+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x39400:400~400]} 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 25 osd.7 40 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674735+0000 last_rx_front 2019-12-07T19:34:15.674747+0000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43278 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccd3b0 space 0x558c57c43278 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 25 osd.7 40 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674761+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x39400~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x39c00~400 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 25 osd.7 40 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674643+0000 last_rx_front 2019-12-07T19:34:15.674689+0000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 20 osd.7 40 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:17.705+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x39c00~400 = 1024 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 20 osd.7 40 sched_scrub not scheduling scrubs due to active recovery 2019-12-07T19:34:17.705+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3a400~400 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 10 osd.7 40 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 20 osd.7 40 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need {<0x1000, 0x1000> : [0x3a400:1400~400]} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3a400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.705+0000 7f8f1fbbe700 10 osd.7 40 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.705+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.705+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.705+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c5740c580 prio 127 cost 0 latency 0.000082 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.706+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.706+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=24, attrs_read=0) 2019-12-07T19:34:17.706+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),45056, 2(4),45056, 4(5),45056, 7(0),45056]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:e=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:e,2(4)=1:276d5164:::smithi13140796-22:e,4(5)=1:276d5164:::smithi13140796-22:e,5(3)=1:276d5164:::smithi13140796-22:e,7(0)=1:276d5164:::smithi13140796-22:e}, in_progress=5(3)) 2019-12-07T19:34:17.706+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c5740c580 finish 2019-12-07T19:34:17.706+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3a400:1400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43278 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccd440 space 0x558c57c43278 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3a400~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3ac00~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3ac00~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3b400~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need {<0x0, 0x1000> : [0x3b400:400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3b400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3b400:400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43198 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccd4d0 space 0x558c57c43198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3b400~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3bc00~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3bc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3bc00~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3c400~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need {<0x1000, 0x1000> : [0x3c400:1400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3c400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3c400:1400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43198 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccd560 space 0x558c57c43198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3c400~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3cc00~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3cc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3cc00~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3d400~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need {<0x0, 0x1000> : [0x3d400:400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3d400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3d400:400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c427f8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccd5f0 space 0x558c57c427f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3d400~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3dc00~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3dc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3dc00~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3e400~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3e400:1400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3e400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3e400:1400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c427f8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccd680 space 0x558c57c427f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3e400~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3ec00~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3ec00~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3f400~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need {<0x0, 0x1000> : [0x3f400:400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3f400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3f400:400~400]} 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42718 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 _add level 1 near 0 on buffer(0x558c57ccd710 space 0x558c57c42718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3f400~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3fc00~400 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x3fc00~400 = 1024 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=1024 len=131072 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 385024 2019-12-07T19:34:17.706+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:34:17.706+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 40/39 ECSubReadReply(tid=184, attrs_read=0)) v2 -- 0x558c574a0780 con 0x558c54c64000 2019-12-07T19:34:17.706+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3ec60 finish 2019-12-07T19:34:17.706+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.707+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 70 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 ==== 45221+0+0 (unknown 0 0 0) 0x558c5738ca00 con 0x558c54c19c00 2019-12-07T19:34:17.707+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3eb00 prio 127 cost 0 latency 0.000014 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.707+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c57f3eb00 prio 127 cost 0 latency 0.000037 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=24, attrs_read=0)) v2 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=24, attrs_read=0) 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=4 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=5 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply Complete: ReadOp(tid=24, to_read={1:276d5164:::smithi13140796-22:e=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:e=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),45056, 2(4),45056, 4(5),45056, 5(3),45056, 7(0),45056]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:e=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:e,2(4)=1:276d5164:::smithi13140796-22:e,4(5)=1:276d5164:::smithi13140796-22:e,5(3)=1:276d5164:::smithi13140796-22:e,7(0)=1:276d5164:::smithi13140796-22:e}, in_progress=) 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_recovery_read_complete: returned 1:276d5164:::smithi13140796-22:e (0, 1048576, [1(2),45056, 2(4),45056, 4(5),45056, 5(3),45056, 7(0),45056]) 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] handle_recovery_read_complete: [0,45056, 2,45056, 3,45056, 4,45056, 5,45056] 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:360448, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=90112, size=360448 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] continue_recovery_op: READING return RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:360448, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=3(1) extent_requested=0,1048576) 2019-12-07T19:34:17.707+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPush(1.4s1 40/39 [PushOp(1:276d5164:::smithi13140796-22:e, version: 37'3, data_included: [0~90112], data_size: 90112, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:360448, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558c57568600 con 0x558c54c5bc00 2019-12-07T19:34:17.707+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c57f3eb00 finish 2019-12-07T19:34:17.707+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.709+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 133 ==== MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:e)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558c57568600 con 0x558c54c5bc00 2019-12-07T19:34:17.709+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c54e78580 prio 127 cost 1049576 latency 0.000016 epoch 40 MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:e)]) v3 2019-12-07T19:34:17.709+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:e)]) v3) prio 127 cost 1049576 e40) 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:e)]) v3) prio 127 cost 1049576 e40) queued 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:e)]) v3) prio 127 cost 1049576 e40) pg 0x558c548ab000 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c54e78580 prio 127 cost 1049576 latency 0.000044 MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:e)]) v3 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:e)]) v3 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:360448, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] continue_recovery_op: on_peer_recover on 3(1), obj 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 2 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 0 ml 2 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1 ml 2 missing shards 1 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2 ml 2 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3 ml 2 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4 ml 2 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5 ml 2 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based degraded 2 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 2 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.709+0000 7f8f06986700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(0+0)=2},2={(1+0)=2},3={(1+0)=2},4={(1+0)=2},5={(1+0)=2}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:71 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 1 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=1} 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 0 ml 1 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1 ml 1 missing shards 1 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2 ml 1 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3 ml 1 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4 ml 1 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5 ml 1 missing shards 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based degraded 1 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 1 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.709+0000 7f8f06986700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:72 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] pushed 1:276d5164:::smithi13140796-22:e to all replicas 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] finish_recovery_op 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 40 finish_recovery_op pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:e dequeue=0 (1/10 rops) 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] queue_recovery -- queuing 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 40 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:17.709+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.709+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) queued 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] release_backoffs [1:276d5164:::smithi13140796-22:e,1:276d5164:::smithi13140796-22:e) 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] finish_degraded_object 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: WRITING return RecoveryOp(hoid=1:276d5164:::smithi13140796-22:e v=37'3 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:e@37'3, size: 360448, copy_subset: [], clone_subset: {}, snapset: e=[]:{e=[e,d,c]}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:360448, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=2 state=COMPLETE waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.709+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:17.709+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.709+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) pg 0x558c548ab000 2019-12-07T19:34:17.709+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 40 do_recovery starting 1 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] recover_replicas(1) 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] peer osd.3(1) missing 1 objects. 2019-12-07T19:34:17.709+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] peer osd.3(1) missing {1:276d5164:::smithi13140796-22:head=37'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] recover_replicas: recover_object_replicas(1:276d5164:::smithi13140796-22:head) 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] prep_object_replica_pushes: on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_object_context: obc NOT found in cache: 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.709+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# _ 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# _ = 0 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] populate_obc_watchers 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.709+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.709+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# = 0 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_object_context: creating obc from disk: 0x558c54d2efc0 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_object_context: 0x558c54d2efc0 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:17.709+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] recovery got recovery read lock on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] start_recovery_op 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 40 start_recovery_op pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:head (0/10 rops) 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] checking 1(2) 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] checking 2(4) 2019-12-07T19:34:17.709+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] checking 3(1) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] checking 4(5) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] checking 5(3) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] checking 7(0) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] recover_object: built op RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=5 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] run_recovery_op: starting RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:17.710+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 4(5) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 5(3) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 1(2) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 2(4) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 4(5) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 5(3) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_all_avail_shards: checking missing_loc 7(0) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: IDLE return RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] start_read_op: starting ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] do_read_op: starting read ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:17.710+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.4s2 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 -- 0x558c5738ca00 con 0x558c54c64000 2019-12-07T19:34:17.710+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpRead(1.4s4 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 -- 0x558c57cca500 con 0x558c54c19800 2019-12-07T19:34:17.710+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.4s5 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 -- 0x558c57e8f400 con 0x558c54c5b400 2019-12-07T19:34:17.710+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.4s3 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 -- 0x558c58092f00 con 0x558c54c19c00 2019-12-07T19:34:17.710+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 -- 0x558c58092c80 con 0x558c53ae5400 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] do_read_op: started ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:head=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:head,2(4)=1:276d5164:::smithi13140796-22:head,4(5)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=1(2),2(4),4(5),5(3),7(0)) 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] started 1 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 40 do_recovery started 1/1 on pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.710+0000 7f8f0297e700 10 osd.7 40 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:17.710+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.710+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c58092c80 con 0x558c53ae5400 2019-12-07T19:34:17.710+0000 7f8f1119b700 15 osd.7 40 enqueue_op 0x558c57efe580 prio 3 cost 0 latency 0.000206 epoch 40 MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 2019-12-07T19:34:17.710+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.710+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.710+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.710+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57efe580 prio 3 cost 0 latency 0.000232 MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.710+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpRead(1.4s0 40/39 ECSubRead(tid=25, to_read={1:276d5164:::smithi13140796-22:head=0,262144,0}, subchunks={1:276d5164:::smithi13140796-22:head=[4,4]}, attrs_to_read=)) v3 2019-12-07T19:34:17.710+0000 7f8f06986700 25 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.710+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x800~800 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x800~800 size 0x13000 (77824) 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a53b0 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a53b0 sbid 0x0)) need {<0x0, 0x1000> : [0x800:800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a53b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x800:800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a53c8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c551a8f30 space 0x558c580a53c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.710+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x800~800 = 2048 2019-12-07T19:34:17.710+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1800~800 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1800~800 size 0x13000 (77824) 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a53b0 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a53b0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1800:1800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5340 blob([0x58d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a53b0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1800:1800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a53c8 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e147e0 space 0x558c580a53c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.710+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1800~800 = 2048 2019-12-07T19:34:17.710+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2800~800 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2800~800 size 0x13000 (77824) 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4b60 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4b60 sbid 0x0)) need {<0x0, 0x1000> : [0x2800:800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4b60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2800:800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57ff4b78 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14870 space 0x558c57ff4b78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.710+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2800~800 = 2048 2019-12-07T19:34:17.710+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3800~800 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3800~800 size 0x13000 (77824) 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4b60 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4b60 sbid 0x0)) need {<0x1000, 0x1000> : [0x3800:1800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57ff4000 blob([0x58f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4b60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3800:1800~800]} 2019-12-07T19:34:17.710+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57ff4b78 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.710+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c573d9440 space 0x558c57ff4b78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.710+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x4800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07ea0 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07ea0 sbid 0x0)) need {<0x0, 0x1000> : [0x4800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07ea0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57a07eb8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14bd0 space 0x558c57a07eb8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.711+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x4800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x5800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07ea0 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07ea0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5800:1800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57ff4230 blob([0x591000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07ea0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5800:1800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c57a07eb8 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14900 space 0x558c57a07eb8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.711+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x5800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x6800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5dc0 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5dc0 sbid 0x0)) need {<0x0, 0x1000> : [0x6800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5dc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a5dd8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c54e14990 space 0x558c580a5dd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.711+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x6800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x7800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5dc0 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5dc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x7800:1800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5d50 blob([0x593000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5dc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7800:1800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a5dd8 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57a597a0 space 0x558c580a5dd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.711+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x7800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x8800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5ea0 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5ea0 sbid 0x0)) need {<0x0, 0x1000> : [0x8800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5ea0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a5eb8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c578eecf0 space 0x558c580a5eb8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.711+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x8800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x9800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5ea0 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5ea0 sbid 0x0)) need {<0x1000, 0x1000> : [0x9800:1800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 231 ==== MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=185, to_read={1:ec1285ab:::smithi13140796-19:head=262144,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 249+0+0 (crc 0 0 0) 0x558c57e8f400 con 0x558c54c64000 2019-12-07T19:34:17.711+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3f080 prio 127 cost 0 latency 0.000022 epoch 40 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=185, to_read={1:ec1285ab:::smithi13140796-19:head=262144,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.711+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=185, to_read={1:ec1285ab:::smithi13140796-19:head=262144,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.711+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.711+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=185, to_read={1:ec1285ab:::smithi13140796-19:head=262144,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5e30 blob([0x595000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5ea0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9800:1800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a5eb8 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57e64870 space 0x558c580a5eb8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.711+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x9800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=185, to_read={1:ec1285ab:::smithi13140796-19:head=262144,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xa800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5f80 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5f80 sbid 0x0)) need {<0x0, 0x1000> : [0xa800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c57f3f080 prio 127 cost 0 latency 0.000068 MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=185, to_read={1:ec1285ab:::smithi13140796-19:head=262144,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.711+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.7s4 40/39 ECSubRead(tid=185, to_read={1:ec1285ab:::smithi13140796-19:head=262144,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.711+0000 7f8f05183700 25 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.711+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x40400~400 2019-12-07T19:34:17.711+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.711+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need {<0x1000, 0x1000> : [0x40400:1400~400]} 2019-12-07T19:34:17.711+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x40400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5f80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa800:800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a5f98 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57ccdc20 space 0x558c580a5f98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.711+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xa800~800 = 2048 2019-12-07T19:34:17.711+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xb800~800 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb800~800 size 0x13000 (77824) 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5f80 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5f80 sbid 0x0)) need {<0x1000, 0x1000> : [0xb800:1800~800]} 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.711+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a5f10 blob([0x597000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5f80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb800:1800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c580a5f98 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57ccdcb0 space 0x558c580a5f98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xb800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xc800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a000 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a000 sbid 0x0)) need {<0x0, 0x1000> : [0xc800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x40400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42718 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c578ee000 space 0x558c57c42718 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x40400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x40c00~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x40c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x41400~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need {<0x0, 0x1000> : [0x41400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x41400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a000 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5809a018 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57ccdd40 space 0x558c5809a018 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xc800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xd800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a000 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a000 sbid 0x0)) need {<0x1000, 0x1000> : [0xd800:1800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x41400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42638 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c54c9a2d0 space 0x558c57c42638 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x41400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x41c00~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x41c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x42400~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need {<0x1000, 0x1000> : [0x42400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x42400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fd8f0 blob([0x599000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a000 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd800:1800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5809a018 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x42400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57ccddd0 space 0x558c5809a018 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42638 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xd800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57f770e0 space 0x558c57c42638 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xe800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x42400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a0e0 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x42c00~400 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a0e0 sbid 0x0)) need {<0x0, 0x1000> : [0xe800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x42c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x43400~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need {<0x0, 0x1000> : [0x43400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x43400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a0e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5809a0f8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57ccde60 space 0x558c5809a0f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xe800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xf800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a0e0 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a0e0 sbid 0x0)) need {<0x1000, 0x1000> : [0xf800:1800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5809a070 blob([0x59b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a0e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf800:1800~800]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x43400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5809a0f8 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42558 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57ccdef0 space 0x558c5809a0f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57f77170 space 0x558c57c42558 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x43400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0xf800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x43c00~400 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x10800~800 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a1c0 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x43c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x44400~400 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a1c0 sbid 0x0)) need {<0x0, 0x1000> : [0x10800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need {<0x1000, 0x1000> : [0x44400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x44400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a1c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5809a1d8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x44400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57668000 space 0x558c5809a1d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x10800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x11800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a1c0 sbid 0x0)) need 0x1800~800 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42558 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57f77200 space 0x558c57c42558 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a1c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x11800:1800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11800: 0x1800 reading 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x44400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x44c00~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x44c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x45400~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need {<0x0, 0x1000> : [0x45400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x45400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 93 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 ==== 39077+0+0 (crc 0 0 0) 0x558c57cca500 con 0x558c54c19800 2019-12-07T19:34:17.712+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c57f3fa20 prio 127 cost 0 latency 0.000018 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.712+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x45400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c430b8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57f77290 space 0x558c57c430b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x45400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x45c00~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x45c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x46400~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x46400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x46400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5809a150 blob([0x59d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a1c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11800:1800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5809a1d8 in 0x558c53a59260) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57668090 space 0x558c5809a1d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x11800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x12800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809a230 blob([0x59f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5809a2a0 sbid 0x0)) need 0x800~800 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5809a230 blob([0x59f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5809a2a0 sbid 0x0)) need {<0x0, 0x1000> : [0x12800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12800: 0x800 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 232 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 ==== 39077+0+0 (crc 0 0 0) 0x558c58092f00 con 0x558c54c64000 2019-12-07T19:34:17.712+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3fe40 prio 127 cost 0 latency 0.000014 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.712+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x46400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c430b8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57f77320 space 0x558c57c430b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x46400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x46c00~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x46c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x47400~400 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need {<0x0, 0x1000> : [0x47400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x47400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.712+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5809a230 blob([0x59f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5809a2a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12800:800~800]} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore.BufferSpace(0x558c5809a2b8 in 0x558c53a59260) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 20 _add level 1 near 0 on buffer(0x558c57668120 space 0x558c5809a2b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x12800~800 = 2048 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x13800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x13800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x14800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x14800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x15800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x15800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x16800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x16800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x17800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x17800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x18800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x18800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x19800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x19800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1a800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1a800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1b800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1b800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 71 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 ==== 39077+0+0 (unknown 0 0 0) 0x558c5738ca00 con 0x558c54c19c00 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1c800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1c800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1d800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1d800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740d080 prio 127 cost 0 latency 0.000016 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1e800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1e800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1f800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x1f800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x20800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x20800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x21800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x21800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x22800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x22800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x23800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x23800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x24800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x24800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x25800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x25800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x26800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x26800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x27800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x27800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x28800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x28800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x29800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x29800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2a800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2a800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2b800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2b800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2c800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2c800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2d800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2d800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x47400:400~400]} 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2e800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42fd8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2e800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2f800~800 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57f773b0 space 0x558c57c42fd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x2f800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x30800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x47400~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x30800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x31800~800 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x47c00~400 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x31800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x32800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x32800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x33800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x47c00~400 = 1024 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x33800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x34800~800 2019-12-07T19:34:17.712+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x48400~400 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x34800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x35800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x35800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x36800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x48400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x36800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x48400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x37800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x37800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x38800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x38800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x39800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x39800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3a800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3a800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3b800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3b800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3c800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3c800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3d800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3d800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3e800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3e800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3f800~800 2019-12-07T19:34:17.712+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f800~800 size 0x13000 (77824) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 0x3f800~800 = 0 2019-12-07T19:34:17.712+0000 7f8f06986700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read read request=262144 r=0 len=38912 2019-12-07T19:34:17.712+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.712+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 -- 0x558c5738cc80 con 0x558c53ae5400 2019-12-07T19:34:17.712+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57efe580 finish 2019-12-07T19:34:17.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.712+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c5738cc80 con 0x558c53ae5400 2019-12-07T19:34:17.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.712+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.712+0000 7f8f1119b700 15 osd.7 40 enqueue_op 0x558c57efe6e0 prio 127 cost 0 latency 0.000013 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.712+0000 7f8f1119b700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.712+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3fa20 prio 127 cost 0 latency 0.000356 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x48400:1400~400]} 2019-12-07T19:34:17.712+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.712+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42fd8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.712+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57f77440 space 0x558c57c42fd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x48400~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x48c00~400 2019-12-07T19:34:17.712+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=25, attrs_read=0) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x48c00~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x49400~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need {<0x0, 0x1000> : [0x49400:400~400]} 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [2(4),38912]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:head=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:head,2(4)=1:276d5164:::smithi13140796-22:head,4(5)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=1(2),4(5),5(3),7(0)) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x49400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3fa20 finish 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3fe40 prio 127 cost 0 latency 0.000330 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=25, attrs_read=0) 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),38912, 2(4),38912]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:head=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:head,2(4)=1:276d5164:::smithi13140796-22:head,4(5)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=4(5),5(3),7(0)) 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57f3fe40 finish 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c5740d080 prio 127 cost 0 latency 0.000282 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=25, attrs_read=0) 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),38912, 2(4),38912, 5(3),38912]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:head=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:head,2(4)=1:276d5164:::smithi13140796-22:head,4(5)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=4(5),7(0)) 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57efe6e0 prio 127 cost 0 latency 0.000217 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=25, attrs_read=0) 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply readop not complete: ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),38912, 2(4),38912, 5(3),38912, 7(0),38912]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:head=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:head,2(4)=1:276d5164:::smithi13140796-22:head,4(5)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=4(5)) 2019-12-07T19:34:17.713+0000 7f8f06986700 10 osd.7 40 dequeue_op 0x558c57efe6e0 finish 2019-12-07T19:34:17.713+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x49400:400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42ef8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668990 space 0x558c57c42ef8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.713+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 164 ==== MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 ==== 39077+0+0 (unknown 0 0 0) 0x558c574a0780 con 0x558c54c5b400 2019-12-07T19:34:17.713+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000011 epoch 40 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x49400~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x49c00~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x49c00~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4a400~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2) prio 127 cost 0 e40) pg 0x558c548ab000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need {<0x1000, 0x1000> : [0x4a400:1400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4a400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000032 MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDECSubOpReadReply(1.4s0 40/39 ECSubReadReply(tid=25, attrs_read=0)) v2 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply: reply ECSubReadReply(tid=25, attrs_read=0) 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=4 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=5 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4a400:1400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42ef8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668a20 space 0x558c57c42ef8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.713+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_sub_read_reply Complete: ReadOp(tid=25, to_read={1:276d5164:::smithi13140796-22:head=read_request_t(to_read=[0,1048576,0], need={1(2)=[4,4],2(4)=[4,4],4(5)=[4,4],5(3)=[4,4],7(0)=[4,4]}, want_attrs=0)}, complete={1:276d5164:::smithi13140796-22:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [1(2),38912, 2(4),38912, 4(5),38912, 5(3),38912, 7(0),38912]))}, priority=3, obj_to_source={1:276d5164:::smithi13140796-22:head=1(2),2(4),4(5),5(3),7(0)}, source_to_obj={1(2)=1:276d5164:::smithi13140796-22:head,2(4)=1:276d5164:::smithi13140796-22:head,4(5)=1:276d5164:::smithi13140796-22:head,5(3)=1:276d5164:::smithi13140796-22:head,7(0)=1:276d5164:::smithi13140796-22:head}, in_progress=) 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4a400~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4ac00~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_recovery_read_complete: returned 1:276d5164:::smithi13140796-22:head (0, 1048576, [1(2),38912, 2(4),38912, 4(5),38912, 5(3),38912, 7(0),38912]) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4ac00~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4b400~400 2019-12-07T19:34:17.713+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] handle_recovery_read_complete: [0,38912, 2,38912, 3,38912, 4,38912, 5,38912] 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4b400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 sbid 0x0)) need {<0x1000, 0x1000> : [0x4b400:1400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4b400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4b400:1400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42e18 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668ab0 space 0x558c57c42e18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4b400~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4bc00~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4bc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4bc00~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4c400~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need {<0x0, 0x1000> : [0x4c400:400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4c400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4c400:400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c435f8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668b40 space 0x558c57c435f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4c400~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4cc00~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4cc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.713+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4cc00~400 = 1024 2019-12-07T19:34:17.713+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4d400~400 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4d400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x4d400:1400~400]} 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4d400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.713+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4d400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c435f8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668bd0 space 0x558c57c435f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4d400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4dc00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4dc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4dc00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4e400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need {<0x0, 0x1000> : [0x4e400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4e400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4e400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c43e48 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668c60 space 0x558c57c43e48 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4e400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4ec00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:311296, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=77824, size=311296 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4ec00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4f400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4f400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need {<0x1000, 0x1000> : [0x4f400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4f400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: READING return RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:311296, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=3(1) extent_requested=0,1048576) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPush(1.4s1 40/39 [PushOp(1:276d5164:::smithi13140796-22:head, version: 37'4, data_included: [0~77824], data_size: 77824, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:311296, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558c575b9e00 con 0x558c54c5bc00 2019-12-07T19:34:17.714+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c5740d4a0 finish 2019-12-07T19:34:17.714+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4f400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c43e48 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668cf0 space 0x558c57c43e48 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4f400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4fc00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4fc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x4fc00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x50400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need {<0x0, 0x1000> : [0x50400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x50400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x50400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c43f98 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668d80 space 0x558c57c43f98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x50400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x50c00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x50c00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x51400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need {<0x1000, 0x1000> : [0x51400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x51400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x51400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c43f98 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668e10 space 0x558c57c43f98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x51400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x51c00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x51c00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x52400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need {<0x0, 0x1000> : [0x52400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x52400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x52400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b96e88 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668ea0 space 0x558c57b96e88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x52400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x52c00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x52c00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x53400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need {<0x1000, 0x1000> : [0x53400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x53400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x53400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b96e88 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668f30 space 0x558c57b96e88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x53400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x53c00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x53c00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x54400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need {<0x0, 0x1000> : [0x54400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x54400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x54400:400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b96f68 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57668fc0 space 0x558c57b96f68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x54400~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x54c00~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.714+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x54c00~400 = 1024 2019-12-07T19:34:17.714+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x55400~400 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need {<0x1000, 0x1000> : [0x55400:1400~400]} 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x55400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.714+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x55400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b96f68 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57669050 space 0x558c57b96f68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x55400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x55c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x55c00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x56400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need {<0x0, 0x1000> : [0x56400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x56400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x56400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b97048 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c576690e0 space 0x558c57b97048 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x56400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x56c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x56c00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x57400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need {<0x1000, 0x1000> : [0x57400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x57400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x57400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b97048 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57669170 space 0x558c57b97048 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x57400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x57c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x57c00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x58400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need {<0x0, 0x1000> : [0x58400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x58400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 134 ==== pg_trim(1.4s0 37'4 e40/40) v2 ==== 34+0+0 (unknown 0 0 0) 0x558c576dcb40 con 0x558c54c5bc00 2019-12-07T19:34:17.715+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 MTrim epoch 40 from 3 shard 0 trim_to 37'4 2019-12-07T19:34:17.715+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MTrim epoch 40 from 3 shard 0 trim_to 37'4) prio 255 cost 10 e40) 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 135 ==== MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:head)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558c575b9e00 con 0x558c54c5bc00 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MTrim epoch 40 from 3 shard 0 trim_to 37'4) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.715+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c54e794a0 prio 127 cost 1049576 latency 0.000008 epoch 40 MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:head)]) v3 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:head)]) v3) prio 127 cost 1049576 e40) 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 MTrim epoch 40 from 3 shard 0 trim_to 37'4) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x58400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b97128 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57669200 space 0x558c57b97128 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x58400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x58c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 MTrim epoch 40 from 3 shard 0 trim_to 37'4 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:head)]) v3) prio 127 cost 1049576 e40) queued 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x58c00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x59400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need {<0x1000, 0x1000> : [0x59400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x59400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] state: replica osd.3 lcod 37'4 2019-12-07T19:34:17.715+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:head)]) v3) prio 127 cost 1049576 e40) pg 0x558c548ab000 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c54e794a0 prio 127 cost 1049576 latency 0.000067 MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:head)]) v3 pg pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] _handle_message: MOSDPGPushReply(1.4s0 40/39 [PushReplyOp(1:276d5164:::smithi13140796-22:head)]) v3 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x59400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b97128 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57669290 space 0x558c57b97128 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x59400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x59c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x59c00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5a400~400 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:311296, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need {<0x0, 0x1000> : [0x5a400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5a400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] continue_recovery_op: on_peer_recover on 3(1), obj 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x5a400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b97208 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57669320 space 0x558c57b97208 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5a400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5ac00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5ac00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5b400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5b400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5b400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5b400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 1 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=1} 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 0 ml 1 missing shards 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1 ml 1 missing shards 1 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5b400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b97208 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c576693b0 space 0x558c57b97208 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5b400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2 ml 1 missing shards 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5bc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5bc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5bc00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5c400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3 ml 1 missing shards 0 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need {<0x0, 0x1000> : [0x5c400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5c400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4 ml 1 missing shards 0 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5 ml 1 missing shards 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based degraded 1 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 1 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x5c400:400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b972e8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c57669440 space 0x558c57b972e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5c400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5cc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5cc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5cc00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5d400~400 2019-12-07T19:34:17.715+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(0+0)=1},2={(1+0)=1},3={(1+0)=1},4={(1+0)=1},5={(1+0)=1}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:73 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5d400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5d400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5d400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based degraded 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.715+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:74 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] pushed 1:276d5164:::smithi13140796-22:head to all replicas 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] finish_recovery_op 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5d400:1400~400]} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57b972e8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 40 finish_recovery_op pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:head dequeue=0 (1/10 rops) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 _add level 1 near 0 on buffer(0x558c576694d0 space 0x558c57b972e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5d400~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5dc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5dc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5dc00~400 = 1024 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5e400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] queue_recovery -- queuing 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5e400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5ec00~400 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5ec00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5f400~400 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 40 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5f400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5f400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5fc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5fc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x5fc00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x60400~400 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) queued 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] release_backoffs [1:276d5164:::smithi13140796-22:head,1:276d5164:::smithi13140796-22:head) 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x60400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x60c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x60c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x61400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x61400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x61c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] finish_degraded_object 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x61c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x62400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x62400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x62c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x62c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x63400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x63400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] continue_recovery_op: WRITING return RecoveryOp(hoid=1:276d5164:::smithi13140796-22:head v=37'4 missing_on=3(1) missing_on_shards=1 recovery_info=ObjectRecoveryInfo(1:276d5164:::smithi13140796-22:head@37'4, size: 311296, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:311296, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=2 state=COMPLETE waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x63c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x63c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x64400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 40 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x64400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x64c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x64c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x65400~400 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGRecovery(pgid=1.4s0epoch_queued=40reserved_pushes=1) prio 5 cost 20971520 e40 reserved_pushes 1) pg 0x558c548ab000 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x65400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x65c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x65c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x66400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x66400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x66c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x66c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x67400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x67400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x67c00~400 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 40 do_recovery starting 1 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x67c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x68400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x68400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x68c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] recover_replicas(1) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x68c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x69400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x69400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x69c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x69c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6a400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6a400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] recover_primary recovering 0 in pg, missing missing(0 may_include_deletes = 1) 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6ac00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6ac00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6b400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6b400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6b400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f06986700 25 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] recover_primary {} 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6bc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6bc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6bc00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6c400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6c400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] started 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6cc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6cc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6cc00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6d400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6d400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] start_recovery_ops needs_recovery: {} 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6d400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6dc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6dc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6dc00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6e400~400 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] start_recovery_ops missing_loc: {} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6e400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6ec00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6ec00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6f400~400 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6f400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6f400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6fc00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6fc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x6fc00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] needs_backfill does not need backfill 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x70400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x70400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x70c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] recovery done, no backfill 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x70c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x71400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f06986700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 40 epoch_requested: 40 AllReplicasRecovered 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x71400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 AllReplicasRecovered) prio 255 cost 10 e40) 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x71c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x71c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x72400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x72400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x72c00~400 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 40 do_recovery started 0/1 on pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x72c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 AllReplicasRecovered) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.715+0000 7f8f06986700 10 osd.7 40 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x73400~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x73400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 AllReplicasRecovered) prio 255 cost 10 e40) pg 0x558c548ab000 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x73c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x73c00~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x74400~400 2019-12-07T19:34:17.715+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x74400~400 = 0 2019-12-07T19:34:17.715+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x74c00~400 2019-12-07T19:34:17.715+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.715+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x74c00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x75400~400 2019-12-07T19:34:17.715+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 AllReplicasRecovered 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x75400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x75c00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x75c00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x76400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x76400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x76c00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x76c00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x77400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x77400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.4s2 RELEASE e40/40) v3 -- 0x558c54df6780 con 0x558c54c64000 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x77c00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x77c00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x78400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x78400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x78c00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x78c00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x79400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x79400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x79c00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79c00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MRecoveryReserve(1.4s4 RELEASE e40/40) v3 -- 0x558c54df7c20 con 0x558c54c19800 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x79c00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7a400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7a400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7ac00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ac00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7ac00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7b400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7b400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7b400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7bc00~400 2019-12-07T19:34:17.716+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.4s1 RELEASE e40/40) v3 -- 0x558c54df7e00 con 0x558c54c5bc00 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7bc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7bc00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7c400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7c400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7cc00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7cc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7cc00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.4s5 RELEASE e40/40) v3 -- 0x558c581e8000 con 0x558c54c5b400 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7d400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7d400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7d400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7dc00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7dc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7dc00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7e400~400 2019-12-07T19:34:17.716+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MRecoveryReserve(1.4s3 RELEASE e40/40) v3 -- 0x558c581e81e0 con 0x558c54c19c00 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7e400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7ec00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ec00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7ec00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7f400~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7f400~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7f400~400 = 0 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7fc00~400 2019-12-07T19:34:17.716+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7fc00~400 size 0x5e000 (385024) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x7fc00~400 = 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] exit Started/Primary/Active/Recovering 2019-12-07T19:34:17.717260+0000 2 0.000258 2019-12-07T19:34:17.716+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=61440 2019-12-07T19:34:17.716+0000 7f8f0297e700 5 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] enter Started/Primary/Active/Recovered 2019-12-07T19:34:17.716+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:17.716+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:17.716+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 385024 2019-12-07T19:34:17.716+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:34:17.716+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:17.716+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 40/39 ECSubReadReply(tid=185, attrs_read=0)) v2 -- 0x558c575bac80 con 0x558c54c64000 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 3(1) objects 0 missing 0 2019-12-07T19:34:17.716+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c57f3f080 finish 2019-12-07T19:34:17.716+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),4(5),5(3),7(0)=2} 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:17.716+0000 7f8f0297e700 15 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] publish_stats_to_osd 40:75 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_acting all_info osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_acting all_info osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_acting all_info osd.3(1) 1.4s1( v 37'4 lc 0'0 (0'0,37'4] local-lis/les=39/40 n=0 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=0) 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_acting all_info osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_acting all_info osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_acting all_info osd.7(0) 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] find_best_info skipping osd.3(1) because it has missing while best is complete 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] calc_acting prefer osd.7(0) because it is current primary 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 3(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 4(5) 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_async_recovery_ec result want=[7,3,1,5,2,4] async_recovery= 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] choose_acting want [7,3,1,5,2,4] != acting [7,2147483647,1,5,2,4], requesting pg_temp change 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:17.716+0000 7f8f0297e700 10 osd.7 40 send_pg_temp {1.4=[]} 2019-12-07T19:34:17.716+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_pgtemp(e40 {1.4=[]} v40) v2 -- 0x558c575d4b40 con 0x558c5476d000 2019-12-07T19:34:17.716+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:17.717+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 233 ==== MRecoveryReserve(1.as5 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c581e81e0 con 0x558c54c64000 2019-12-07T19:34:17.717+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.as5 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.717+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.717+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.717+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.717+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.717+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c57e8b000 2019-12-07T19:34:17.717+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.717+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.718509+0000 3 0.000107 2019-12-07T19:34:17.717+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.717+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:17.717+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.717+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.as5 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.717+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.717+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.717+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.717+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.717+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c57e8b000 2019-12-07T19:34:17.717+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.717+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.as0 GRANT e40/40) v3 -- 0x558c54df6f00 con 0x558c54c64000 2019-12-07T19:34:17.717+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.718563+0000 1 0.000022 2019-12-07T19:34:17.717+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.717+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.717+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.719+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 234 ==== MRecoveryReserve(1.7s4 RELEASE e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54df6f00 con 0x558c54c64000 2019-12-07T19:34:17.719+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.7s4 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.719+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.719+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.719+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.719+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.719+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c54d5f000 2019-12-07T19:34:17.719+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.719+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.720535+0000 1 0.000040 2019-12-07T19:34:17.719+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.719+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:17.719+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.720+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 235 ==== MOSDPGPush(1.as5 40/39 [PushOp(1:51ccf92f:::smithi13140796-15:head, version: 21'2, data_included: [0~61440], data_size: 61440, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:51ccf92f:::smithi13140796-15:head@21'2, size: 245760, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:245760, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 62502+0+0 (crc 0 0 0) 0x558c57612400 con 0x558c54c64000 2019-12-07T19:34:17.720+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c54e786e0 prio 127 cost 62440 latency 0.000036 epoch 40 MOSDPGPush(1.as5 40/39 [PushOp(1:51ccf92f:::smithi13140796-15:head, version: 21'2, data_included: [0~61440], data_size: 61440, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:51ccf92f:::smithi13140796-15:head@21'2, size: 245760, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:245760, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.720+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDPGPush(1.as5 40/39 [PushOp(1:51ccf92f:::smithi13140796-15:head, version: 21'2, data_included: [0~61440], data_size: 61440, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:51ccf92f:::smithi13140796-15:head@21'2, size: 245760, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:245760, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 62440 e40) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.720+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDPGPush(1.as5 40/39 [PushOp(1:51ccf92f:::smithi13140796-15:head, version: 21'2, data_included: [0~61440], data_size: 61440, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:51ccf92f:::smithi13140796-15:head@21'2, size: 245760, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:245760, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 62440 e40) queued 2019-12-07T19:34:17.720+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.720+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDPGPush(1.as5 40/39 [PushOp(1:51ccf92f:::smithi13140796-15:head, version: 21'2, data_included: [0~61440], data_size: 61440, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:51ccf92f:::smithi13140796-15:head@21'2, size: 245760, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:245760, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 62440 e40) pg 0x558c57e8b000 2019-12-07T19:34:17.720+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c54e786e0 prio 127 cost 62440 latency 0.000085 MOSDPGPush(1.as5 40/39 [PushOp(1:51ccf92f:::smithi13140796-15:head, version: 21'2, data_included: [0~61440], data_size: 61440, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:51ccf92f:::smithi13140796-15:head@21'2, size: 245760, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:245760, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.720+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.as5 40/39 [PushOp(1:51ccf92f:::smithi13140796-15:head, version: 21'2, data_included: [0~61440], data_size: 61440, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:51ccf92f:::smithi13140796-15:head@21'2, size: 245760, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:245760, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.720+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] on_local_recover: 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:34:17.720+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 lc 21'1 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active m=1 mbc={} ps=[a~1]] got missing 1:51ccf92f:::smithi13140796-15:head v 21'2 2019-12-07T19:34:17.720+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active mbc={} ps=[a~1]] last_complete now 21'2 log.complete_to at end 2019-12-07T19:34:17.720+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.720+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 94 ==== MRecoveryReserve(1.ds2 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54df7c20 con 0x558c54c19800 2019-12-07T19:34:17.720+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.ds2 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:17.720+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c57ef9500 seq 4 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) get_onode oid 5#1:51ccf92f:::smithi13140796-15:head# key 0x85800000000000000151ccf9'/!smithi13140796-15!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.720+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.720+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.720+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.720+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c55141000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) r -2 v.len 0 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) get_onode oid 5#1:51ccf92f:::smithi13140796-15:head# key 0x85800000000000000151ccf9'/!smithi13140796-15!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) r -2 v.len 0 2019-12-07T19:34:17.720+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1193 2019-12-07T19:34:17.720+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# = 0 2019-12-07T19:34:17.720+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef9500 8 blobs 2019-12-07T19:34:17.720+0000 7f8f06185700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.721389+0000 4 0.000104 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.720+0000 7f8f06185700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.720+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.720+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.720+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.ds2 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.720+0000 7f8f0b990700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.720+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.720+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.720+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate 0xf000/1000,f000,0 2019-12-07T19:34:17.720+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.720+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x731000~f000/1000,f000,0 2019-12-07T19:34:17.720+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c55141000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x731000~f000] 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578816c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578816c0 blob([0x731000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c380 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c578816c0) get_ref 0x0~2000 Blob(0x558c578816c0 blob([0x731000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c380 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c578816c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.720+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c578816c0 blob([0x731000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c380 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c54e1c398 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:17.720+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MRecoveryReserve(1.ds0 GRANT e40/40) v3 -- 0x558c575ccb40 con 0x558c54c19800 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df490 blob([0x733000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c3f0 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c578df490) get_ref 0x0~2000 Blob(0x558c578df490 blob([0x733000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c3f0 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c578df490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c578df490 blob([0x733000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c3f0 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c54e1c408 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e1c460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d752d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e1c460 blob([0x735000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d752d0 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c54e1c460) get_ref 0x0~2000 Blob(0x558c54e1c460 blob([0x735000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d752d0 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c54e1c460) get_ref init 0x2000, 1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c54e1c460 blob([0x735000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d752d0 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c54d752e8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:17.720+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.721454+0000 1 0.000030 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578817a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578817a0 blob([0x737000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c578817a0) get_ref 0x0~2000 Blob(0x558c578817a0 blob([0x737000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c578817a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c578817a0 blob([0x737000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c54d3dc18 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:17.720+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800c000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.720+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800c000 blob([0x739000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c070 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c000) get_ref 0x0~2000 Blob(0x558c5800c000 blob([0x739000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c070 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5800c000 blob([0x739000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c070 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c088 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800c0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800c0e0 blob([0x73b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c150 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c0e0) get_ref 0x0~2000 Blob(0x558c5800c0e0 blob([0x73b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c150 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5800c0e0 blob([0x73b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c150 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c168 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800c1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800c1c0 blob([0x73d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c230 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c1c0) get_ref 0x0~2000 Blob(0x558c5800c1c0 blob([0x73d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c230 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5800c1c0 blob([0x73d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c230 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c248 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800c2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800c2a0 blob([0x73f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c310 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c2a0) get_ref 0x0~1000 Blob(0x558c5800c2a0 blob([0x73f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c310 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.blob(0x558c5800c2a0) get_ref init 0x1000, 1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~1000: 0x0~1000 Blob(0x558c5800c2a0 blob([0x73f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800c310 sbid 0x0)) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c328 in 0x558c53a59b20) _discard 0x0~1000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xf000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) dirty_range mark inline shard dirty 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 = 0 2019-12-07T19:34:17.720+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 4 keys 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 4 keys = 0 2019-12-07T19:34:17.720+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049e'.missing/0000000000000001.A833F94F.head.smithi13140796-15..' <- missing/0000000000000001.A833F94F.head.smithi13140796-15.. 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.720+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9500 cost 99677 (9 ios * 4000 + 63677 bytes) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9500 onodes 0x558c54e13180 shared_blobs 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) update 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) update inline shard 152 bytes from 8 extents 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:head# is 643 (485 bytes onode + 2 bytes spanning blobs + 156 bytes inline extents) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9500 allocated 0x[731000~f000] released 0x[] 2019-12-07T19:34:17.720+0000 7f8f03980700 10 freelist allocate 0x731000~f000 2019-12-07T19:34:17.720+0000 7f8f03980700 20 freelist _xor first_key 0x700000 last_key 0x700000 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 prepare 2019-12-07T19:34:17.720+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef9500 2019-12-07T19:34:17.720+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:34:17.720+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.720+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.720+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.720+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 aio_wait 2019-12-07T19:34:17.720+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9500 2019-12-07T19:34:17.720+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 io_done 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1966 < min_flush_size 524288 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13180 had 1 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1989 < min_flush_size 524288 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1989 < min_flush_size 524288 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x109f84~7c5 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x109f84 0x109f84~7c5 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd9f84 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf84 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x749 and padding block with 0x8b7 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x10a749 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.720+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124137s (0.000032847s flush + 0.000091290s kv commit) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.720+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9500> 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 kv_submitted 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9500 2019-12-07T19:34:17.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 finishing 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9500 onodes 0x558c54e13180 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dc18 in 0x558c53a59b20) _finish_write discard buffer(0x558c57669680 space 0x558c54d3dc18 0x0~2000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d752e8 in 0x558c53a59b20) _finish_write discard buffer(0x558c551a95f0 space 0x558c54d752e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c398 in 0x558c53a59b20) _finish_write discard buffer(0x558c551a9b90 space 0x558c54e1c398 0x0~2000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e1c408 in 0x558c53a59b20) _finish_write discard buffer(0x558c54dbac60 space 0x558c54e1c408 0x0~2000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c088 in 0x558c53a59b20) _finish_write discard buffer(0x558c57669710 space 0x558c5800c088 0x0~2000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c168 in 0x558c53a59b20) _finish_write discard buffer(0x558c576697a0 space 0x558c5800c168 0x0~2000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c248 in 0x558c53a59b20) _finish_write discard buffer(0x558c57669830 space 0x558c5800c248 0x0~2000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c328 in 0x558c53a59b20) _finish_write discard buffer(0x558c576698c0 space 0x558c5800c328 0x0~1000 writing nocache) 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9500 done 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:17.720+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9500 [] 2019-12-07T19:34:17.720+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.720+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 lcod 0'0 active mbc={} ps=[a~1]] recovery_committed_to version 21'2 now ondisk 2019-12-07T19:34:17.720+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.720+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_trim(1.as0 21'2 e40/40) v2 -- 0x558c54df70e0 con 0x558c54c64000 2019-12-07T19:34:17.720+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.as0 40/39 [PushReplyOp(1:51ccf92f:::smithi13140796-15:head)]) v3 -- 0x558c5763de00 con 0x558c54c64000 2019-12-07T19:34:17.720+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.721+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 95 ==== MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=51, to_read={1:bca18876:::smithi13140796-30:head=0,262144,0}, subchunks={1:bca18876:::smithi13140796-30:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (crc 0 0 0) 0x558c57efb180 con 0x558c54c19800 2019-12-07T19:34:17.721+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c54e798c0 prio 3 cost 0 latency 0.000019 epoch 40 MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=51, to_read={1:bca18876:::smithi13140796-30:head=0,262144,0}, subchunks={1:bca18876:::smithi13140796-30:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.721+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=51, to_read={1:bca18876:::smithi13140796-30:head=0,262144,0}, subchunks={1:bca18876:::smithi13140796-30:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.721+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=51, to_read={1:bca18876:::smithi13140796-30:head=0,262144,0}, subchunks={1:bca18876:::smithi13140796-30:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.721+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.721+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=51, to_read={1:bca18876:::smithi13140796-30:head=0,262144,0}, subchunks={1:bca18876:::smithi13140796-30:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c55141000 2019-12-07T19:34:17.721+0000 7f8f06185700 10 osd.7 40 dequeue_op 0x558c54e798c0 prio 3 cost 0 latency 0.000054 MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=51, to_read={1:bca18876:::smithi13140796-30:head=0,262144,0}, subchunks={1:bca18876:::smithi13140796-30:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=51, to_read={1:bca18876:::smithi13140796-30:head=0,262144,0}, subchunks={1:bca18876:::smithi13140796-30:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.721+0000 7f8f06185700 25 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xa00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xa00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xe00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xe00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x4e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x5e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x6e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x7e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x8e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x9e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xa200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xa200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xa600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xa600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xaa00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xaa00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xae00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xae00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xb200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xb200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xb600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xb600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xba00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xba00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xbe00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xbe00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xc200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xc200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xc600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xc600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xca00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xca00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xce00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xce00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xd200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xd200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xd600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xd600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xda00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xda00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xde00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xde00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xe200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xe200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xe600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xe600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xea00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xea00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xee00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xee00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xf200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xf200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xf600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xf600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xfa00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xfa00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xfe00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0xfe00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x10e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x11e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x12e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x13e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x14e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x15e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x16e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x17e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x18e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19a00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x19e00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1a200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1a200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1a600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1a600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1aa00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1aa00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ae00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ae00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1b200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1b200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1b600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1b600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ba00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ba00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1be00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1be00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1c200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1c200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1c600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1c600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ca00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ca00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ce00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ce00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1d200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1d200~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1d600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1d600~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1da00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1da00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1de00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1de00~200 = 512 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1e200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1e200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1e600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1e600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ea00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ea00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ee00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1ee00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1f200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1f200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1f600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1f600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1fa00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1fa00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1fe00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x1fe00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x20e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x21e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x22e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x23e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x24e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x25e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x26e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x27e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x28e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29a00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29a00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29e00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x29e00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2a200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2a200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2a600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2a600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2aa00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2aa00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ae00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ae00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2b200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2b200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2b600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2b600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ba00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ba00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2be00~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2be00~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2c200~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2c200~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2c600~200 2019-12-07T19:34:17.721+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.721+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2c600~200 = 0 2019-12-07T19:34:17.721+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ca00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ca00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ce00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ce00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2d200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2d200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2d600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2d600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2da00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2da00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2de00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2de00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2e200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2e200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2e600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2e600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ea00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ea00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ee00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2ee00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2f200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2f200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2f600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2f600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2fa00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2fa00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2fe00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x2fe00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x30e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x31e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x32e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x33e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x34e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x35e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x36e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x37e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x38e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39a00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39a00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39e00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x39e00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3a200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3a200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3a600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3a600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3aa00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3aa00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ae00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ae00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3b200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3b200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3b600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3b600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ba00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ba00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3be00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3be00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3c200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3c200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3c600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3c600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ca00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ca00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ce00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ce00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3d200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3d200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3d600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3d600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3da00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3da00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3de00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3de00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3e200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3e200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3e600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3e600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ea00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ea00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ee00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3ee00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3f200~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3f200~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3f600~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3f600~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3fa00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3fa00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3fe00~200 2019-12-07T19:34:17.722+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x1e000 (122880) 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x3fe00~200 = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=61440 2019-12-07T19:34:17.722+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:17.722+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:bca18876:::smithi13140796-30:head 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:17.722+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 122880 2019-12-07T19:34:17.722+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key 2019-12-07T19:34:17.722+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key = 0 2019-12-07T19:34:17.722+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 40/39 ECSubReadReply(tid=51, attrs_read=0)) v2 -- 0x558c57f8f400 con 0x558c54c19800 2019-12-07T19:34:17.722+0000 7f8f06185700 10 osd.7 40 dequeue_op 0x558c54e798c0 finish 2019-12-07T19:34:17.722+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.722+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 236 ==== MRecoveryReserve(1.as5 RELEASE e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54df70e0 con 0x558c54c64000 2019-12-07T19:34:17.722+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.as5 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.722+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.722+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.722+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.722+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.722+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c57e8b000 2019-12-07T19:34:17.722+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.722+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.723574+0000 1 0.000034 2019-12-07T19:34:17.722+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.722+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:17.722+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.723+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 136 ==== MRecoveryReserve(1.1s1 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c54df7e00 con 0x558c54c5bc00 2019-12-07T19:34:17.723+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.1s1 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.723+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.723+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.723+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.723+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.723+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c5736a000 2019-12-07T19:34:17.723+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.723+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.724998+0000 3 0.000173 2019-12-07T19:34:17.723+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.723+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:17.723+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.723+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.1s1 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.723+0000 7f8f0b990700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.723+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.723+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.723+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.723+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c5736a000 2019-12-07T19:34:17.723+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.723+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.1s0 GRANT e40/40) v3 -- 0x558c576ddc20 con 0x558c54c5bc00 2019-12-07T19:34:17.723+0000 7f8f08189700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.725059+0000 1 0.000020 2019-12-07T19:34:17.723+0000 7f8f08189700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.723+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.723+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.724+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 137 ==== MOSDPGRecoveryDelete(1.1s1 e40,39 1:891f0d13:::smithi13140796-14:4,29'8) v2 ==== 123+0+0 (unknown 0 0 0) 0x558c5763ce00 con 0x558c54c5bc00 2019-12-07T19:34:17.724+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c5740d600 prio 3 cost 1000 latency 0.000016 epoch 40 MOSDPGRecoveryDelete(1.1s1 e40,39 1:891f0d13:::smithi13140796-14:4,29'8) v2 2019-12-07T19:34:17.724+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGRecoveryDelete(1.1s1 e40,39 1:891f0d13:::smithi13140796-14:4,29'8) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGRecoveryDelete(1.1s1 e40,39 1:891f0d13:::smithi13140796-14:4,29'8) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGRecoveryDelete(1.1s1 e40,39 1:891f0d13:::smithi13140796-14:4,29'8) v2) prio 3 cost 1000 e40) pg 0x558c5736a000 2019-12-07T19:34:17.724+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c5740d600 prio 3 cost 1000 latency 0.000042 MOSDPGRecoveryDelete(1.1s1 e40,39 1:891f0d13:::smithi13140796-14:4,29'8) v2 pg pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] handle_recovery_delete 0x558c5740d600 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] remove_missing_object 1:891f0d13:::smithi13140796-14:4 29'8 2019-12-07T19:34:17.724+0000 7f8f04181700 20 snap_mapper.remove_oid 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:17.724+0000 7f8f04181700 20 snap_mapper._remove_oid 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:17.724+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.724+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.724+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.724+0000 7f8f04181700 20 snap_mapper.get_snaps 1:891f0d13:::smithi13140796-14:4 got.empty() 2019-12-07T19:34:17.724+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:17.724+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c57ef9b00 seq 4 2019-12-07T19:34:17.724+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) get_onode oid 1#1:891f0d13:::smithi13140796-14:4# key 0x818000000000000001891f0d1321736d'ithi13140796-14!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:17.724+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) r -2 v.len 0 2019-12-07T19:34:17.724+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 1#1:891f0d13:::smithi13140796-14:4# 2019-12-07T19:34:17.724+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9b00 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.724+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9b00 onodes shared_blobs 2019-12-07T19:34:17.724+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9b00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.724+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 prepare 2019-12-07T19:34:17.724+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9b00 2019-12-07T19:34:17.724+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 io_done 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.724+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c5740d600 finish 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.724+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x10a749~2e to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x10a749 0x10a749~2e to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xda749 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x749 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x777 and padding block with 0x889 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x10a777 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000106998s (0.000000094s flush + 0.000106904s kv commit) 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9b00> 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 kv_submitted 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9b00 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 finishing 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9b00 onodes 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9b00 done 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9b00 [] 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.724+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] on_local_recover: 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:17.724+0000 7f8f08189700 20 snap_mapper.remove_oid 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:17.724+0000 7f8f08189700 20 snap_mapper._remove_oid 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:17.724+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.724+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.724+0000 7f8f08189700 20 snap_mapper.get_snaps 1:891f0d13:::smithi13140796-14:4 got.empty() 2019-12-07T19:34:17.724+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 27'7 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=2 mbc={} ps=[a~1]] got missing 1:891f0d13:::smithi13140796-14:4 v 29'8 2019-12-07T19:34:17.724+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=1 mbc={} ps=[a~1]] last_complete now 29'8 log.complete_to 29'9 2019-12-07T19:34:17.724+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.724+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c574fe000 seq 5 2019-12-07T19:34:17.724+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049b'.missing/0000000000000001.198F0B8C.4.smithi13140796-14..' <- missing/0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:17.724+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.724+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.724+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.724+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe000 cost 5406 (1 ios * 4000 + 1406 bytes) 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe000 onodes shared_blobs 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe000 allocated 0x[] released 0x[] 2019-12-07T19:34:17.724+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 prepare 2019-12-07T19:34:17.724+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe000 2019-12-07T19:34:17.724+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 io_done 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.724+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=1 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.724+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1182 < min_flush_size 524288 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x10a777~4b5 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x10a777 0x10a777~4b5 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xda777 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x777 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc2c and padding block with 0x3d4 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x10ac2c 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.724+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000117080s (0.000000114s flush + 0.000116966s kv commit) 2019-12-07T19:34:17.724+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe000> 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 kv_submitted 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe000 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 finishing 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe000 onodes 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 done 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:17.724+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe000 [] 2019-12-07T19:34:17.724+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.724+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.724+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 0'0 active m=1 mbc={} ps=[a~1]] recovery_committed_to version 29'8 now ondisk 2019-12-07T19:34:17.724+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGRecoveryDeleteReply(1.1s0 e40,39 1:891f0d13:::smithi13140796-14:4,29'8) v2 -- 0x558c54c58a00 con 0x558c54c5bc00 2019-12-07T19:34:17.724+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.729+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 96 ==== MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=52, to_read={1:bf81b261:::smithi13140796-37:head=0,262144,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (crc 0 0 0) 0x558c57f8f400 con 0x558c54c19800 2019-12-07T19:34:17.729+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c54e79760 prio 3 cost 0 latency 0.000023 epoch 40 MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=52, to_read={1:bf81b261:::smithi13140796-37:head=0,262144,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.729+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=52, to_read={1:bf81b261:::smithi13140796-37:head=0,262144,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=52, to_read={1:bf81b261:::smithi13140796-37:head=0,262144,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=52, to_read={1:bf81b261:::smithi13140796-37:head=0,262144,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c55141000 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 osd.7 40 dequeue_op 0x558c54e79760 prio 3 cost 0 latency 0.000066 MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=52, to_read={1:bf81b261:::smithi13140796-37:head=0,262144,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=52, to_read={1:bf81b261:::smithi13140796-37:head=0,262144,0}, subchunks={1:bf81b261:::smithi13140796-37:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.729+0000 7f8f0217d700 25 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x200~200 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x200~200 = 512 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x600~200 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x600~200 = 512 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xa00~200 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xa00~200 = 512 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xe00~200 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xe00~200 = 512 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1200~200 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.729+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 138 ==== MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 24, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 317111+0+0 (unknown 0 0 0) 0x558c54c58a00 con 0x558c54c5bc00 2019-12-07T19:34:17.729+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c5740d760 prio 127 cost 263144 latency 0.000043 epoch 40 MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 24, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.729+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 24, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) 2019-12-07T19:34:17.729+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.729+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 24, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) queued 2019-12-07T19:34:17.729+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.729+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 24, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) pg 0x558c5736a000 2019-12-07T19:34:17.729+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c5740d760 prio 127 cost 263144 latency 0.000081 MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 24, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.729+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 24, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.729+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] get_temp_recovery_object -3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head 2019-12-07T19:34:17.729+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] handle_recovery_push: Adding oid -3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head in the temp collection 2019-12-07T19:34:17.729+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c54ddef00 seq 6 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c55278a98 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c551a9d40 space 0x558c55278a98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) get_onode oid 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# key 0x817ffffffffffffffd891f0d13217465'mp_recovering_1.1s1_29'9_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1200~200 = 512 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1600~200 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) r -2 v.len 0 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.729+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1600~200 = 512 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) get_onode oid 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# key 0x817ffffffffffffffd891f0d13217465'mp_recovering_1.1s1_29'9_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1a00~200 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) r -2 v.len 0 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.729+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1194 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1a00~200 = 512 2019-12-07T19:34:17.729+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# = 0 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1e00~200 2019-12-07T19:34:17.729+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 0x0~40000 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 0x0~40000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.729+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~40000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1e00~200 = 512 2019-12-07T19:34:17.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2200~200 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x25000 (151552) 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ddef00 32 blobs 2019-12-07T19:34:17.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.729+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538718 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57b87440 space 0x558c57538718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2200~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2600~200 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2600~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2a00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2a00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2e00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2e00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3200~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.730+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0x40000/1000,40000,0 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:17.730+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x740000~40000/1000,40000,0 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x740000~40000] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcb570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcb570 blob([0x740000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb500 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcb570) get_ref 0x0~2000 Blob(0x558c57dcb570 blob([0x740000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb500 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcb570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57dcb570 blob([0x740000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb500 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcb518 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcaa80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbb90 blob([0x742000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcaa80 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbb90) get_ref 0x0~2000 Blob(0x558c57dcbb90 blob([0x742000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcaa80 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbb90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57dcbb90 blob([0x742000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaa80 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcaa98 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbb20 blob([0x744000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d960 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbb20) get_ref 0x0~2000 Blob(0x558c57dcbb20 blob([0x744000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d960 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbb20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57dcbb20 blob([0x744000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d960 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d3d978 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d9d0 blob([0x746000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74f50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d9d0) get_ref 0x0~2000 Blob(0x558c54d3d9d0 blob([0x746000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74f50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c54d3d9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54d3d9d0 blob([0x746000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74f50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d74f68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad260 blob([0x748000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb650 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c576ad260) get_ref 0x0~2000 Blob(0x558c576ad260 blob([0x748000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb650 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c576ad260) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c576ad260 blob([0x748000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb650 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcb668 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d74ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbc70 blob([0x74a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74ee0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbc70) get_ref 0x0~2000 Blob(0x558c57dcbc70 blob([0x74a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74ee0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbc70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57dcbc70 blob([0x74a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74ee0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54d74ef8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcaa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcaa10 blob([0x74c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbc00 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcaa10) get_ref 0x0~2000 Blob(0x558c57dcaa10 blob([0x74c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbc00 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcaa10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57dcaa10 blob([0x74c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbc00 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcbc18 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbce0 blob([0x74e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbd50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbce0) get_ref 0x0~2000 Blob(0x558c57dcbce0 blob([0x74e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbd50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57dcbce0 blob([0x74e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbd50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcbd68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbe30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbdc0 blob([0x750000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbe30 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbdc0) get_ref 0x0~2000 Blob(0x558c57dcbdc0 blob([0x750000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbe30 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57dcbdc0 blob([0x750000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbe30 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcbe48 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbea0 blob([0x752000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbf10 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbea0) get_ref 0x0~2000 Blob(0x558c57dcbea0 blob([0x752000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcbf10 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57dcbea0 blob([0x752000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbf10 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcbf28 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcbf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcbf80 blob([0x754000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d730 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbf80) get_ref 0x0~2000 Blob(0x558c57dcbf80 blob([0x754000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d730 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57dcbf80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57dcbf80 blob([0x754000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d730 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800d748 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800d7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800d7a0 blob([0x756000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d810 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800d7a0) get_ref 0x0~2000 Blob(0x558c5800d7a0 blob([0x756000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d810 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800d7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c5800d7a0 blob([0x756000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d810 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800d828 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800d880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800d880 blob([0x758000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d8f0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800d880) get_ref 0x0~2000 Blob(0x558c5800d880 blob([0x758000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d8f0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800d880) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c5800d880 blob([0x758000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d8f0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800d908 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800d960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800d960 blob([0x75a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d9d0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800d960) get_ref 0x0~2000 Blob(0x558c5800d960 blob([0x75a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d9d0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800d960) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c5800d960 blob([0x75a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d9d0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800d9e8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800da40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800dab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800da40 blob([0x75c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800dab0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538718 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800da40) get_ref 0x0~2000 Blob(0x558c5800da40 blob([0x75c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800dab0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57b874d0 space 0x558c57538718 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800da40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c5800da40 blob([0x75c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dab0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800dac8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3200~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800db20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800db90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3600~200 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800db20 blob([0x75e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800db90 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800db20) get_ref 0x0~2000 Blob(0x558c5800db20 blob([0x75e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800db90 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800db20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c5800db20 blob([0x75e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800db90 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800dba8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800dc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800dc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3600~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3a00~200 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800dc00 blob([0x760000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800dc70 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800dc00) get_ref 0x0~2000 Blob(0x558c5800dc00 blob([0x760000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800dc70 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800dc00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c5800dc00 blob([0x760000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dc70 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3a00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800dc88 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3e00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800dce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800dd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800dce0 blob([0x762000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800dd50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800dce0) get_ref 0x0~2000 Blob(0x558c5800dce0 blob([0x762000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800dd50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800dce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c5800dce0 blob([0x762000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dd50 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800dd68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3e00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800ddc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800de30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4200~200 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800ddc0 blob([0x764000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800de30 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800ddc0) get_ref 0x0~2000 Blob(0x558c5800ddc0 blob([0x764000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800de30 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800ddc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c5800ddc0 blob([0x764000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800de30 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800de48 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800dea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800df10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800dea0 blob([0x766000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800df10 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800dea0) get_ref 0x0~2000 Blob(0x558c5800dea0 blob([0x766000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800df10 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800dea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c5800dea0 blob([0x766000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800df10 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800df28 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800df80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f66000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800df80 blob([0x768000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66000 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800df80) get_ref 0x0~2000 Blob(0x558c5800df80 blob([0x768000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66000 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c5800df80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c5800df80 blob([0x768000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66000 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f66018 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f660e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66070 blob([0x76a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f660e0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66070) get_ref 0x0~2000 Blob(0x558c57f66070 blob([0x76a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f660e0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66070) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57f66070 blob([0x76a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f660e0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f660f8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f661c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66150 blob([0x76c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f661c0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66150) get_ref 0x0~2000 Blob(0x558c57f66150 blob([0x76c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f661c0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66150) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57f66150 blob([0x76c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f661c0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f661d8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f662a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66230 blob([0x76e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f662a0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66230) get_ref 0x0~2000 Blob(0x558c57f66230 blob([0x76e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f662a0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66230) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57f66230 blob([0x76e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f662a0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f662b8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f66380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66310 blob([0x770000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66380 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66310) get_ref 0x0~2000 Blob(0x558c57f66310 blob([0x770000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66380 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66310) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57f66310 blob([0x770000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66380 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f66398 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f663f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f66460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f663f0 blob([0x772000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66460 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f663f0) get_ref 0x0~2000 Blob(0x558c57f663f0 blob([0x772000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66460 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f663f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c57f663f0 blob([0x772000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66460 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f66478 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f664d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f66540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f664d0 blob([0x774000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66540 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f664d0) get_ref 0x0~2000 Blob(0x558c57f664d0 blob([0x774000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66540 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f664d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c57f664d0 blob([0x774000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66540 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f66558 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f665b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f66620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f665b0 blob([0x776000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66620 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538868 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f665b0) get_ref 0x0~2000 Blob(0x558c57f665b0 blob([0x776000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66620 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f665b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57b87560 space 0x558c57538868 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c57f665b0 blob([0x776000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66620 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f66638 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4200~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f66700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4600~200 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66690 blob([0x778000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66700 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66690) get_ref 0x0~2000 Blob(0x558c57f66690 blob([0x778000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f66700 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c57f66690 blob([0x778000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66700 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f66718 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4600~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f667e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4a00~200 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66770 blob([0x77a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f667e0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66770) get_ref 0x0~2000 Blob(0x558c57f66770 blob([0x77a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f667e0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66770) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c57f66770 blob([0x77a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f667e0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f667f8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4a00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4e00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f668c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66850 blob([0x77c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f668c0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66850) get_ref 0x0~2000 Blob(0x558c57f66850 blob([0x77c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f668c0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66850) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x4e00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c57f66850 blob([0x77c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f668c0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f668d8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5200~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f66930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f669a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f66930 blob([0x77e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f669a0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66930) get_ref 0x0~2000 Blob(0x558c57f66930 blob([0x77e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f669a0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.blob(0x558c57f66930) get_ref init 0x2000, 1000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c57f66930 blob([0x77e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f669a0 sbid 0x0)) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f669b8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x40000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.extentmap(0x558c574a0388) dirty_range mark inline shard dirty 2019-12-07T19:34:17.730+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 0x0~40000 = 0 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.730+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 24 keys 2019-12-07T19:34:17.730+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 24 keys = 0 2019-12-07T19:34:17.730+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddef00 cost 448964 (33 ios * 4000 + 316964 bytes) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddef00 onodes 0x558c574a0280 shared_blobs 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.extentmap(0x558c574a0388) update 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore.extentmap(0x558c574a0388) update inline shard 611 bytes from 32 extents 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# is 55007 (54390 bytes onode + 2 bytes spanning blobs + 615 bytes inline extents) 2019-12-07T19:34:17.730+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddef00 allocated 0x[740000~40000] released 0x[] 2019-12-07T19:34:17.730+0000 7f8f04181700 10 freelist allocate 0x740000~40000 2019-12-07T19:34:17.730+0000 7f8f04181700 20 freelist _xor first_key 0x700000 last_key 0x700000 2019-12-07T19:34:17.730+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 prepare 2019-12-07T19:34:17.730+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ddef00 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538868 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57670f30 space 0x558c57538868 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5200~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5600~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5600~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5a00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5a00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5e00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x5e00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6200~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538948 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57670fc0 space 0x558c57538948 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6200~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6600~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6600~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6a00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6a00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6e00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x6e00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7200~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538948 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671050 space 0x558c57538948 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7200~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7600~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7600~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7a00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7a00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7e00~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x7e00~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8200~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x25000 (151552) 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.730+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c5740d760 finish 2019-12-07T19:34:17.730+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.730+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 aio_wait 2019-12-07T19:34:17.730+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddef00 2019-12-07T19:34:17.730+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 io_done 2019-12-07T19:34:17.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 21460 < min_flush_size 524288 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538a28 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c576710e0 space 0x558c57538a28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8200~200 = 512 2019-12-07T19:34:17.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8600~200 2019-12-07T19:34:17.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 54228 < min_flush_size 524288 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 55208 < min_flush_size 524288 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8a00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8a00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8e00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x8e00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0280 had 1 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 55231 < min_flush_size 524288 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 55231 < min_flush_size 524288 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x10ac2c~d7bf to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x10ac2c 0x10ac2c~d7bf to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:17.631456+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _allocate len 0x83eb from 0 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _allocate len 0x83eb from 1 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd240000 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd240000~10000/10000,10000,acd240000 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x120000 to include allocated 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 39 (was clean) 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:17.732260+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xdac2c 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc2c 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3eb and padding block with 0xc15 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x120000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57538a28 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671170 space 0x558c57538a28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9200~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9600~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9a00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9a00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9e00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x9e00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xa200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1183eb 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (39) on file(ino 16 size 0x120000 mtime 2019-12-07T19:34:17.732260+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]), flushing log 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x120000 mtime 2019-12-07T19:34:17.732260+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 39 len 0x49 crc 0x970b257d) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c575832e8 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671200 space 0x558c575832e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x26000~1000 to file(ino 1 size 0x26000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xa200~200 = 512 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x26000 0x26000~1000 to file(ino 1 size 0x26000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xa600~200 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x27000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x26000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xa600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xaa00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xaa00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xae00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xae00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xb200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x27000 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:17.731+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000020 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c575832e8 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671290 space 0x558c575832e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xb200~200 = 512 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 39 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xb600~200 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x120000 mtime 2019-12-07T19:34:17.732260+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000430134s (0.000039263s flush + 0.000390871s kv commit) 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xb600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xba00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xba00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xbe00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xbe00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xc200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddef00> 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 kv_submitted 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddef00 2019-12-07T19:34:17.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 finishing 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddef00 onodes 0x558c574a0280 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d978 in 0x558c53a59500) _finish_write discard buffer(0x558c57a08f30 space 0x558c54d3d978 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74ef8 in 0x558c53a59500) _finish_write discard buffer(0x558c57e28480 space 0x558c54d74ef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d74f68 in 0x558c53a59500) _finish_write discard buffer(0x558c57a08bd0 space 0x558c54d74f68 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcaa98 in 0x558c53a59500) _finish_write discard buffer(0x558c580cdd40 space 0x558c57dcaa98 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcb518 in 0x558c53a59500) _finish_write discard buffer(0x558c580cdef0 space 0x558c57dcb518 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcb668 in 0x558c53a59500) _finish_write discard buffer(0x558c57e283f0 space 0x558c57dcb668 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcbc18 in 0x558c53a59500) _finish_write discard buffer(0x558c57e28510 space 0x558c57dcbc18 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcbd68 in 0x558c53a59500) _finish_write discard buffer(0x558c57e285a0 space 0x558c57dcbd68 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcbe48 in 0x558c53a59500) _finish_write discard buffer(0x558c57669dd0 space 0x558c57dcbe48 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcbf28 in 0x558c53a59500) _finish_write discard buffer(0x558c57669e60 space 0x558c57dcbf28 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f66018 in 0x558c53a59500) _finish_write discard buffer(0x558c576703f0 space 0x558c57f66018 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f660f8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670480 space 0x558c57f660f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPushReply(1.1s0 40/39 [PushReplyOp(1:891f0d13:::smithi13140796-14:head)]) v3 -- 0x558c5763ce00 con 0x558c54c5bc00 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f661d8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670510 space 0x558c57f661d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f662b8 in 0x558c53a59500) _finish_write discard buffer(0x558c576705a0 space 0x558c57f662b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f66398 in 0x558c53a59500) _finish_write discard buffer(0x558c57670630 space 0x558c57f66398 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f66478 in 0x558c53a59500) _finish_write discard buffer(0x558c576706c0 space 0x558c57f66478 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f66558 in 0x558c53a59500) _finish_write discard buffer(0x558c57670750 space 0x558c57f66558 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f66638 in 0x558c53a59500) _finish_write discard buffer(0x558c576707e0 space 0x558c57f66638 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f66718 in 0x558c53a59500) _finish_write discard buffer(0x558c57670870 space 0x558c57f66718 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f667f8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670900 space 0x558c57f667f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f668d8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670990 space 0x558c57f668d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f669b8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670a20 space 0x558c57f669b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d748 in 0x558c53a59500) _finish_write discard buffer(0x558c57669ef0 space 0x558c5800d748 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d828 in 0x558c53a59500) _finish_write discard buffer(0x558c548df5f0 space 0x558c5800d828 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d908 in 0x558c53a59500) _finish_write discard buffer(0x558c57e28360 space 0x558c5800d908 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d9e8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670000 space 0x558c5800d9e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800dac8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670090 space 0x558c5800dac8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800dba8 in 0x558c53a59500) _finish_write discard buffer(0x558c57670120 space 0x558c5800dba8 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800dc88 in 0x558c53a59500) _finish_write discard buffer(0x558c576701b0 space 0x558c5800dc88 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800dd68 in 0x558c53a59500) _finish_write discard buffer(0x558c57670240 space 0x558c5800dd68 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800de48 in 0x558c53a59500) _finish_write discard buffer(0x558c576702d0 space 0x558c5800de48 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583208 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800df28 in 0x558c53a59500) _finish_write discard buffer(0x558c57670360 space 0x558c5800df28 0x0~2000 writing nocache) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671320 space 0x558c57583208 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddef00 done 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xc200~200 = 512 2019-12-07T19:34:17.731+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddef00 [] 2019-12-07T19:34:17.731+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xc600~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xc600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xca00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xca00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xce00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xce00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xd200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583208 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c576713b0 space 0x558c57583208 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xd200~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xd600~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xd600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xda00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xda00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xde00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xde00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xe200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583668 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671440 space 0x558c57583668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xe200~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xe600~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xe600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xea00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xea00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xee00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xee00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xf200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583668 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c576714d0 space 0x558c57583668 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xf200~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xf600~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xf600~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xfa00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xfa00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xfe00~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.731+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0xfe00~200 = 512 2019-12-07T19:34:17.731+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10200~200 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x25000 (151552) 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.731+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583748 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671560 space 0x558c57583748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10200~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10600~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10600~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10a00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10a00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10e00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x10e00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11200~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583748 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c576715f0 space 0x558c57583748 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11200~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11600~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11600~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11a00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11a00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11e00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x11e00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12200~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583828 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671680 space 0x558c57583828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12200~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12600~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12600~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12a00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12a00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12e00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x12e00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13200~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583828 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671710 space 0x558c57583828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13200~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13600~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13600~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13a00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13a00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13e00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x13e00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14200~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583908 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c576717a0 space 0x558c57583908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14200~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14600~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14600~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14a00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14a00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14e00~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.732+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x14e00~200 = 512 2019-12-07T19:34:17.732+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15200~200 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x25000 (151552) 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.732+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583908 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671830 space 0x558c57583908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15a00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15a00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15e00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x15e00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c54d3c558 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c576718c0 space 0x558c54d3c558 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16a00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16a00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16e00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x16e00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c54d3c558 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671950 space 0x558c54d3c558 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17a00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17a00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17e00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x17e00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c575839e8 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c576719e0 space 0x558c575839e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18a00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18a00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18e00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x18e00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 139 ==== MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [262144~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1261568, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 53848+0+0 (unknown 0 0 0) 0x558c5763ce00 con 0x558c54c5bc00 2019-12-07T19:34:17.733+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c5740db80 prio 127 cost 54248 latency 0.000020 epoch 40 MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [262144~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1261568, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.733+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [262144~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1261568, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 54248 e40) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.733+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [262144~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1261568, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 54248 e40) queued 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c575839e8 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [262144~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1261568, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 54248 e40) pg 0x558c5736a000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671a70 space 0x558c575839e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c5740db80 prio 127 cost 54248 latency 0.000065 MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [262144~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1261568, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19a00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.1s1 40/39 [PushOp(1:891f0d13:::smithi13140796-14:head, version: 29'9, data_included: [262144~53248], data_size: 53248, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:891f0d13:::smithi13140796-14:head@29'9, size: 1261568, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1261568, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19a00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19e00~200 2019-12-07T19:34:17.733+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] get_temp_recovery_object -3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x19e00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] handle_recovery_push: Removing oid -3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head from the temp collection 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1a200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] on_local_recover: 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 lc 29'8 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active m=1 mbc={} ps=[a~1]] got missing 1:891f0d13:::smithi13140796-14:head v 29'9 2019-12-07T19:34:17.733+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 29'8 active mbc={} ps=[a~1]] last_complete now 29'9 log.complete_to at end 2019-12-07T19:34:17.733+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c54ea1b00 seq 7 2019-12-07T19:34:17.733+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 0x40000~d000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 0x40000~d000 - have 0x40000 (262144) bytes fadvise_flags 0x0 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x40000~d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1b00 7 blobs 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.733+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0xd000/1000,d000,0 2019-12-07T19:34:17.733+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x780000~d000/1000,d000,0 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x780000~d000] 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576acfc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794dd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576acfc0 blob([0x780000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dd50 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c576acfc0) get_ref 0x0~2000 Blob(0x558c576acfc0 blob([0x780000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dd50 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c576acfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c576acfc0 blob([0x780000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dd50 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5794dd68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c000 blob([0x782000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c380 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5792c000) get_ref 0x0~2000 Blob(0x558c5792c000 blob([0x782000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c380 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5792c000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c5792c000 blob([0x782000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c380 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5792c398 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800cc40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8a10 blob([0x784000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cc40 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c54df8a10) get_ref 0x0~2000 Blob(0x558c54df8a10 blob([0x784000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cc40 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c54df8a10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c54df8a10 blob([0x784000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cc40 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800cc58 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800ccb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800cd20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583ac8 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800ccb0 blob([0x786000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cd20 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671b00 space 0x558c57583ac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800ccb0) get_ref 0x0~2000 Blob(0x558c5800ccb0 blob([0x786000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cd20 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800ccb0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c5800ccb0 blob([0x786000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cd20 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800cd38 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1a200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1a600~200 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800cd90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800ce00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800cd90 blob([0x788000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800ce00 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800cd90) get_ref 0x0~2000 Blob(0x558c5800cd90 blob([0x788000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800ce00 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800cd90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c5800cd90 blob([0x788000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ce00 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1a600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1aa00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800ce18 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800ce70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800cee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1aa00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800ce70 blob([0x78a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cee0 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800ce70) get_ref 0x0~2000 Blob(0x558c5800ce70 blob([0x78a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cee0 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800ce70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c5800ce70 blob([0x78a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cee0 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800cef8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ae00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800cf50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800cfc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800cf50 blob([0x78c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cfc0 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ae00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800cf50) get_ref 0x0~1000 Blob(0x558c5800cf50 blob([0x78c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cfc0 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.blob(0x558c5800cf50) get_ref init 0x1000, 1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1b200~200 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~1000: 0x0~1000 Blob(0x558c5800cf50 blob([0x78c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800cfc0 sbid 0x0)) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5800cfd8 in 0x558c53a59500) _discard 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4d000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.extentmap(0x558c574a0388) dirty_range mark inline shard dirty 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# 0x40000~d000 = 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) get_onode oid 1#1:891f0d13:::smithi13140796-14:head# key 0x818000000000000001891f0d1321736d'ithi13140796-14!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) r -2 v.len 0 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) get_onode oid 1#1:891f0d13:::smithi13140796-14:head# key 0x818000000000000001891f0d1321736d'ithi13140796-14!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.1s1_head 0x558c576dcf00) r -2 v.len 0 2019-12-07T19:34:17.733+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# -> 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.1s1_head 1#-3:891f0d13:::temp_recovering_1.1s1_29'9_39_head:head# -> 1#1:891f0d13:::smithi13140796-14:head# = 0 2019-12-07T19:34:17.733+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049b'.missing/0000000000000001.198F0B8C.head.smithi13140796-14..' <- missing/0000000000000001.198F0B8C.head.smithi13140796-14.. 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.733+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 87034 (8 ios * 4000 + 55034 bytes) 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes 0x558c574a0280 shared_blobs 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.extentmap(0x558c574a0388) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore.extentmap(0x558c574a0388) update inline shard 741 bytes from 39 extents 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 55137 (54390 bytes onode + 2 bytes spanning blobs + 745 bytes inline extents) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583ac8 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671b90 space 0x558c57583ac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[780000~d000] released 0x[] 2019-12-07T19:34:17.733+0000 7f8f04181700 10 freelist allocate 0x780000~d000 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1b200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f04181700 20 freelist _xor first_key 0x780000 last_key 0x780000 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1b600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1b600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ba00~200 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea1b00 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ba00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1be00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1be00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1c200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c5740db80 finish 2019-12-07T19:34:17.733+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.733+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 29'8 active mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.733+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583ba8 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671c20 space 0x558c57583ba8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1c200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1c600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1c600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 aio_wait 2019-12-07T19:34:17.733+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ca00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ca00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ce00~200 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ce00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1d200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 31765 < min_flush_size 524288 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 56543 < min_flush_size 524288 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583ba8 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671cb0 space 0x558c57583ba8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1d200~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1d600~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1d600~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1da00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1da00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1de00~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0280 had 1 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5793b680 had 1 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 56566 < min_flush_size 524288 2019-12-07T19:34:17.733+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1de00~200 = 512 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 56566 < min_flush_size 524288 2019-12-07T19:34:17.733+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1e200~200 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x25000 (151552) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:17.732260+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1183eb~dcf6 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:17.732260+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need {<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1183eb 0x1183eb~dcf6 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:17.732260+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _allocate len 0x60e1 from 0 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _allocate len 0x60e1 from 1 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd250000 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd250000~10000/10000,10000,acd250000 2019-12-07T19:34:17.733+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x130000 to include allocated 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 40 (was clean) 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xe83eb 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3eb 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe1 and padding block with 0xf1f 2019-12-07T19:34:17.733+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x130000 2019-12-07T19:34:17.733+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1260e1 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (40) on file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]), flushing log 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 40 len 0x49 crc 0x74a481ae) 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x27000~1000 to file(ino 1 size 0x27000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x27000 0x27000~1000 to file(ino 1 size 0x27000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x28000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x27000 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x28000 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:17.734+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000016 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 40 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000363062s (0.000034653s flush + 0.000328409s kv commit) 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.734+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1b00> 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.734+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:34:17.734+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 0x558c574a0280 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c398 in 0x558c53a59500) _finish_write discard buffer(0x558c57670d80 space 0x558c5792c398 0x0~2000 writing nocache) 2019-12-07T19:34:17.734+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794dd68 in 0x558c53a59500) _finish_write discard buffer(0x558c57670cf0 space 0x558c5794dd68 0x0~2000 writing nocache) 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800cc58 in 0x558c53a59500) _finish_write discard buffer(0x558c57670e10 space 0x558c5800cc58 0x0~2000 writing nocache) 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800cd38 in 0x558c53a59500) _finish_write discard buffer(0x558c57670ea0 space 0x558c5800cd38 0x0~2000 writing nocache) 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800ce18 in 0x558c53a59500) _finish_write discard buffer(0x558c578ac000 space 0x558c5800ce18 0x0~2000 writing nocache) 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800cef8 in 0x558c53a59500) _finish_write discard buffer(0x558c578ac090 space 0x558c5800cef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800cfd8 in 0x558c53a59500) _finish_write discard buffer(0x558c578ac120 space 0x558c5800cfd8 0x0~1000 writing nocache) 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:17.734+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:34:17.734+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.734+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 lcod 29'8 active mbc={} ps=[a~1]] recovery_committed_to version 29'9 now ondisk 2019-12-07T19:34:17.734+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.734+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_trim(1.1s0 29'9 e40/40) v2 -- 0x558c581e8b40 con 0x558c54c5bc00 2019-12-07T19:34:17.734+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPushReply(1.1s0 40/39 [PushReplyOp(1:891f0d13:::smithi13140796-14:head)]) v3 -- 0x558c54c58a00 con 0x558c54c5bc00 2019-12-07T19:34:17.734+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583c88 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671d40 space 0x558c57583c88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1e200~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1e600~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1e600~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ea00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ea00~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ee00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1ee00~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1f200~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583c88 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671dd0 space 0x558c57583c88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1f200~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1f600~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1f600~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1fa00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1fa00~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1fe00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x1fe00~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20200~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need {<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583d68 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671e60 space 0x558c57583d68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20200~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20600~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20600~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20a00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20a00~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20e00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x20e00~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21200~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need {<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.734+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 140 ==== MRecoveryReserve(1.1s1 RELEASE e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c581e8b40 con 0x558c54c5bc00 2019-12-07T19:34:17.734+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.1s1 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.734+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.734+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.734+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.734+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.734+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c5736a000 2019-12-07T19:34:17.734+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.734+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.736124+0000 1 0.000029 2019-12-07T19:34:17.734+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.734+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:17.734+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583d68 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c57671ef0 space 0x558c57583d68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21200~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21600~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21600~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21a00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.734+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21a00~200 = 512 2019-12-07T19:34:17.734+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21e00~200 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.734+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x21e00~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need {<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583e48 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c578ac870 space 0x558c57583e48 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22200~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22600~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22a00~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x22e00~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need {<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583e48 in 0x558c53a59dc0) _discard 0x1000~1000 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c578ac900 space 0x558c57583e48 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23200~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23600~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23a00~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x23e00~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) need {<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24200: 0x200 reading 0x0~1000 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57583f28 in 0x558c53a59dc0) _discard 0x0~1000 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 _add level 1 near 0 on buffer(0x558c578ac990 space 0x558c57583f28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24200~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24600~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24a00~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x24e00~200 = 512 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x25e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x26e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x27e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x28e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x29e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2a200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2a200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2a600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2a600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2aa00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2aa00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ae00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ae00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2b200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2b200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2b600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2b600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ba00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ba00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2be00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2be00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2c200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2c200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2c600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2c600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ca00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ca00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ce00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ce00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2d200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2d200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2d600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2d600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2da00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2da00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2de00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2de00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2e200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2e200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2e600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2e600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ea00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ea00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ee00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2ee00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2f200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2f200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2f600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2f600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2fa00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2fa00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2fe00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x2fe00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x30e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x31e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x32e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x33e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x34e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x35e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x36e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x37e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x38e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39a00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39a00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39e00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x39e00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3a200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3a200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3a600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3a600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3aa00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3aa00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ae00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ae00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3b200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3b200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3b600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3b600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ba00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ba00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3be00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3be00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3c200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3c200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3c600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3c600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ca00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ca00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ce00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ce00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3d200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3d200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3d600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3d600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3da00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3da00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3de00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3de00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3e200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3e200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3e600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3e600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ea00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ea00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ee00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3ee00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3f200~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3f200~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3f600~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3f600~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3fa00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3fa00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3fe00~200 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x25000 (151552) 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 0x3fe00~200 = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=75776 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:bf81b261:::smithi13140796-37:head 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:bf81b261:::smithi13140796-37:head 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 151552 2019-12-07T19:34:17.735+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key = 0 2019-12-07T19:34:17.735+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 40/39 ECSubReadReply(tid=52, attrs_read=0)) v2 -- 0x558c57f8fb80 con 0x558c54c19800 2019-12-07T19:34:17.735+0000 7f8f0217d700 10 osd.7 40 dequeue_op 0x558c54e79760 finish 2019-12-07T19:34:17.735+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 237 ==== MRecoveryReserve(1.9s4 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c54df70e0 con 0x558c54c64000 2019-12-07T19:34:17.735+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.735+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.9s4 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.735+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.735+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.735+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.735+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.735+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c5500a000 2019-12-07T19:34:17.735+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.735+0000 7f8f08189700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.736879+0000 3 0.000104 2019-12-07T19:34:17.735+0000 7f8f08189700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.735+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:17.735+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.735+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.9s4 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.735+0000 7f8f0b990700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.735+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.735+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.735+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.735+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c5500a000 2019-12-07T19:34:17.735+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.735+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.9s0 GRANT e40/40) v3 -- 0x558c576dc1e0 con 0x558c54c64000 2019-12-07T19:34:17.735+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.736929+0000 1 0.000019 2019-12-07T19:34:17.735+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.735+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:17.735+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.736+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 238 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=187, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 249+0+0 (crc 0 0 0) 0x558c575bac80 con 0x558c54c64000 2019-12-07T19:34:17.736+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c54e78b00 prio 3 cost 0 latency 0.000021 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=187, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.736+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=187, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.736+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=187, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.736+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.736+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=187, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.736+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c54e78b00 prio 3 cost 0 latency 0.000064 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=187, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=187, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.736+0000 7f8f08189700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55278d90 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75c00 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75730 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d757a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75880 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d758f0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d75960 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d759d0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x8c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8540 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf85b0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x9c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xa400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xac00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xac00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xb400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xb400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xbc00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf84d0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8460 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xbc00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xc400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xcc00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xcc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf83f0 blob([0x62000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55265ab0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xcc00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xd400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xdc00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xdc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xdc00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xe400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xec00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2d20 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d755e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xec00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xf400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xf400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xfc00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0xfc00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac770 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75b20 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x10c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x11c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55265f80 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beae00 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x12c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x13c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beae70 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beaee0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x14c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x15c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beaf50 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beafc0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x16c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x17c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18400~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18400~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18400~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18c00~400 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.736+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb030 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb0a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.736+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x18c00~400 = 1024 2019-12-07T19:34:17.736+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x19c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1a400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ac00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb110 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb180 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ac00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1b400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1b400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1bc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1bc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1bc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1c400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1cc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1cc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb1f0 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb260 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1cc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1d400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1d400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1dc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1dc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1dc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1e400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ec00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb2d0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb340 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1ec00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1f400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1f400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1fc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x1fc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb3b0 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb420 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x20c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x21c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb490 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb500 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x22c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x23c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb570 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb5e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x24c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x25c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb650 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb6c0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x26c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x27c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb730 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb7a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x28c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x29c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2a400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ac00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb810 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb880 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ac00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2b400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2b400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2bc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2bc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2bc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2c400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2cc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2cc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb8f0 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beb960 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2cc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2d400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2d400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2dc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2dc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2dc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2e400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ec00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57beb9d0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beba40 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2ec00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2f400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2f400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2fc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x2fc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebab0 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebb20 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x30c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x31c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebb90 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebc00 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x32c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x33c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebc70 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebce0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x34c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x35c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebd50 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebdc0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x36c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x37c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebe30 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebea0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x38c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39c00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x39c00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3a400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ac00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57bebf10 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bebf80 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ac00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3b400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3bc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3bc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3bc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3c400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3cc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3cc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2e70 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2d90 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3cc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3d400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3d400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3dc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3dc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3dc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3e400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ec00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2ee0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2e00 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3ec00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3f400~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f400~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3f400~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3fc00~400 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.737+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x3fc00~400 = 1024 2019-12-07T19:34:17.737+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=1024 len=131072 2019-12-07T19:34:17.737+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.737+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:17.737+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:17.737+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:17.737+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:17.737+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=187, attrs_read=0)) v2 -- 0x558c5793b900 con 0x558c54c64000 2019-12-07T19:34:17.737+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:17.737+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.741+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 97 ==== MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=53, to_read={1:be42858c:::smithi13140796-46:head=0,262144,0}, subchunks={1:be42858c:::smithi13140796-46:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (crc 0 0 0) 0x558c57f8fb80 con 0x558c54c19800 2019-12-07T19:34:17.741+0000 7f8f24c58700 15 osd.7 40 enqueue_op 0x558c54e79a20 prio 3 cost 0 latency 0.000023 epoch 40 MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=53, to_read={1:be42858c:::smithi13140796-46:head=0,262144,0}, subchunks={1:be42858c:::smithi13140796-46:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.741+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=53, to_read={1:be42858c:::smithi13140796-46:head=0,262144,0}, subchunks={1:be42858c:::smithi13140796-46:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.741+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=53, to_read={1:be42858c:::smithi13140796-46:head=0,262144,0}, subchunks={1:be42858c:::smithi13140796-46:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.741+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.741+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=53, to_read={1:be42858c:::smithi13140796-46:head=0,262144,0}, subchunks={1:be42858c:::smithi13140796-46:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c55141000 2019-12-07T19:34:17.741+0000 7f8f06185700 10 osd.7 40 dequeue_op 0x558c54e79a20 prio 3 cost 0 latency 0.000063 MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=53, to_read={1:be42858c:::smithi13140796-46:head=0,262144,0}, subchunks={1:be42858c:::smithi13140796-46:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.ds2 40/39 ECSubRead(tid=53, to_read={1:be42858c:::smithi13140796-46:head=0,262144,0}, subchunks={1:be42858c:::smithi13140796-46:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:17.741+0000 7f8f06185700 25 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xa00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xa00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xe00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xe00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1570 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b1f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1420 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1490 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x4e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1650 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f16c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x5e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x6e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1730 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb7a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x7e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x8e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb730 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb6c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x9e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xa200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xa200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xa600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xa600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xaa00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xaa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xaa00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xae00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xae00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xae00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xb200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xb200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xb600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xb600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xba00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xba00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xba00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xbe00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbe00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fb650 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fb5e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xbe00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xc200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xc200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xc600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xc600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xca00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xca00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xca00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xce00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xce00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xce00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xd200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xd200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xd600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xd600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xda00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xda00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xda00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xde00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xde00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbb20 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbb90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xde00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xe200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xe200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xe600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xe600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xea00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xea00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xea00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xee00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xee00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xee00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xf200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xf200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xf600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xf600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xfa00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xfa00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xfe00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfe00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57583420 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f17a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0xfe00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x10e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbc00 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbc70 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x11e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x12e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbce0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbd50 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x13e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x14e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbdc0 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbe30 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x15e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x16e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbea0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fbf10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x17e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x18e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fbf80 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc000 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x19e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1a200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1a200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1a600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1a600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1aa00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1aa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1aa00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ae00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ae00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ae00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1b200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1b200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1b600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1b600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ba00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ba00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ba00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1be00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1be00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc070 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc0e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1be00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1c200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1c200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1c600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1c600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ca00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ca00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ca00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ce00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ce00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ce00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1d200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1d200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1d600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1d600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1da00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1da00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1da00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1de00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1de00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc150 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc1c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1de00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1e200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1e200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1e600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1e600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ea00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ea00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ea00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ee00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ee00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1ee00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1f200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1f200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1f600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1f600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1fa00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1fa00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1fe00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fe00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc230 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc2a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x1fe00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x20e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc310 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc380 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x21e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x22e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc3f0 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc460 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x23e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x24e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc4d0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc540 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x25e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x26e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc5b0 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc620 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x27e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x28e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29600~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29a00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29a00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29e00~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc690 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc700 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x29e00~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2a200~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.741+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2a200~200 = 512 2019-12-07T19:34:17.741+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2a600~200 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.741+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2a600~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2aa00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2aa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2aa00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ae00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ae00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ae00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2b200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2b200~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2b600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2b600~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ba00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ba00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ba00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2be00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2be00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc770 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc7e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2be00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2c200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2c200~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2c600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2c600~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ca00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ca00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ca00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ce00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ce00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ce00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2d200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2d200~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2d600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2d600~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2da00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2da00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2da00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2de00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2de00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc850 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fc8c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2de00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2e200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2e200~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2e600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2e600~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ea00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ea00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ea00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ee00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ee00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fc930 blob([0x22a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fc9a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2ee00~200 = 512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2f200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2f200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2f600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2f600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2fa00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2fa00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2fe00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fe00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x2fe00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x30e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x31e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x32e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x33e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x34e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x35e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x36e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x37e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x38e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39a00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39a00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39a00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39e00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39e00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x39e00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3a200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3a200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3a600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3a600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3aa00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3aa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3aa00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ae00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ae00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ae00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3b200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3b200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3b600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3b600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ba00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ba00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ba00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3be00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3be00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3be00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3c200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3c200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3c600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3c600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ca00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ca00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ca00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ce00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ce00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ce00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3d200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3d200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3d600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3d600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3da00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3da00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3da00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3de00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3de00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3de00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3e200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3e200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3e600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3e600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ea00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ea00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ea00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ee00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ee00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3ee00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3f200~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f200~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3f200~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3f600~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f600~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3f600~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3fa00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fa00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3fa00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3fe00~200 2019-12-07T19:34:17.742+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fe00~200 size 0x2f000 (192512) 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 0x3fe00~200 = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 20 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=96256 2019-12-07T19:34:17.742+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:be42858c:::smithi13140796-46:head 2019-12-07T19:34:17.742+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:be42858c:::smithi13140796-46:head 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:34:17.742+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 192512 2019-12-07T19:34:17.742+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key 2019-12-07T19:34:17.742+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.ds2_head 2#1:be42858c:::smithi13140796-46:head# hinfo_key = 0 2019-12-07T19:34:17.742+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.ds0 40/39 ECSubReadReply(tid=53, attrs_read=0)) v2 -- 0x558c57efb180 con 0x558c54c19800 2019-12-07T19:34:17.742+0000 7f8f06185700 10 osd.7 40 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:34:17.742+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.745+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 239 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=188, to_read={1:9da56077:::smithi13140796-13:head=262144,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 249+0+0 (crc 0 0 0) 0x558c5793b900 con 0x558c54c64000 2019-12-07T19:34:17.745+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57335b80 prio 127 cost 0 latency 0.000018 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=188, to_read={1:9da56077:::smithi13140796-13:head=262144,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.745+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=188, to_read={1:9da56077:::smithi13140796-13:head=262144,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.745+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=188, to_read={1:9da56077:::smithi13140796-13:head=262144,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.745+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.745+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=188, to_read={1:9da56077:::smithi13140796-13:head=262144,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.745+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c57335b80 prio 127 cost 0 latency 0.000076 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=188, to_read={1:9da56077:::smithi13140796-13:head=262144,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=188, to_read={1:9da56077:::smithi13140796-13:head=262144,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.745+0000 7f8f04181700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x40400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x40400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x40c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f2fc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3030 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x40c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x41400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x41400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x41c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x41c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x42400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x42400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x42c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f30a0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3110 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x42c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x43400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x43400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x43c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x43c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x44400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x44400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x44c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3180 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f31f0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x44c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x45400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x45400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x45c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x45c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x46400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x46400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x46c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3260 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f32d0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x46c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x47400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x47400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x47c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x47c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x48400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x48400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x48c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3340 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f33b0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x48c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x49400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x49400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x49c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x49c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4a400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4a400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4ac00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3420 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880fc0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4ac00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4b400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4b400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4b400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4bc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4bc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4bc00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4c400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4c400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4cc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4cc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcacb0 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcad90 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4cc00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4d400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4d400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4d400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4dc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4dc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4dc00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4e400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4e400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4ec00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcae00 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcae70 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4ec00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4f400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4f400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4f400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4fc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4fc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x4fc00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x50400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x50400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x50c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaee0 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaf50 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x50c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x51400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x51400~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x51c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcafc0 blob([!~1000,0x488000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57dcb030 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x51c00~400 = 1024 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x52400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x52400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x52c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x52c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x53400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x53400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x53c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x53c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x54400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x54400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x54c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x54c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x55400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x55400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x55c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x55c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x56400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x56400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x56c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x56c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x57400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x57400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x57c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x57c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x58400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x58400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x58c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x58c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x59400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x59400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x59c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x59c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5a400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5a400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5ac00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5ac00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5b400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5b400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5b400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5bc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5bc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5bc00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5c400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5c400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5cc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5cc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5cc00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5d400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5d400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5d400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5dc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5dc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5dc00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5e400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5e400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5ec00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5ec00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5f400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5f400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5f400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5fc00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5fc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x5fc00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x60400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x60400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x60c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x60c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x61400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x61400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x61c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x61c00~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x62400~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62400~400 size 0x52000 (335872) 2019-12-07T19:34:17.745+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x62400~400 = 0 2019-12-07T19:34:17.745+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x62c00~400 2019-12-07T19:34:17.745+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x62c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x63400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x63400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x63c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x63c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x64400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x64400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x64c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x64c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x65400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x65400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x65c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x65c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x66400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x66400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x66c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x66c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x67400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x67400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x67c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x67c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x68400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x68400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x68c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x68c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x69400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x69400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x69c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x69c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6a400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6a400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6ac00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6ac00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6b400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6b400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6b400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6bc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6bc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6bc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6c400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6c400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6cc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6cc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6cc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6d400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6d400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6d400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6dc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6dc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6dc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6e400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6e400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6ec00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6ec00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6f400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6f400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6f400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6fc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6fc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x6fc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x70400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x70400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x70c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x70c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x71400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x71400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x71c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x71c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x72400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x72400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x72c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x72c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x73400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x73400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x73c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x73c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x74400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x74400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x74c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x74c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x75400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x75400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x75c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x75c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x76400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x76400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x76c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x76c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x77400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x77400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x77c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x77c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x78400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x78400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x78c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x78c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x79400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x79400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x79c00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79c00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x79c00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7a400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7a400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7ac00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ac00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7ac00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7b400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7b400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7b400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7bc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7bc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7bc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7c400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7c400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7cc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7cc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7cc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7d400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7d400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7d400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7dc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7dc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7dc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7e400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7e400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7ec00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ec00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7ec00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7f400~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7f400~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7f400~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7fc00~400 2019-12-07T19:34:17.746+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7fc00~400 size 0x52000 (335872) 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 0x7fc00~400 = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=36864 2019-12-07T19:34:17.746+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.746+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:17.746+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:17.746+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:17.746+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:17.746+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=188, attrs_read=0)) v2 -- 0x558c58092c80 con 0x558c54c64000 2019-12-07T19:34:17.746+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c57335b80 finish 2019-12-07T19:34:17.746+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.748+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 98 ==== MRecoveryReserve(1.ds2 RELEASE e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c575ccb40 con 0x558c54c19800 2019-12-07T19:34:17.748+0000 7f8f24c58700 15 osd.7 40 enqueue_peering_evt 1.ds2 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.748+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.748+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.748+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.748+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.748+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c55141000 2019-12-07T19:34:17.748+0000 7f8f0217d700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.748+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.749933+0000 1 0.000031 2019-12-07T19:34:17.748+0000 7f8f0217d700 5 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.748+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:17.748+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:17.749+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 240 ==== MRecoveryReserve(1.8s5 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c581e8000 con 0x558c54c64000 2019-12-07T19:34:17.749+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.8s5 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.749+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.749+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.749+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.749+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.749+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c57380000 2019-12-07T19:34:17.749+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.749+0000 7f8f0898a700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.751060+0000 3 0.000281 2019-12-07T19:34:17.749+0000 7f8f0898a700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.749+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:17.749+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.749+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.8s5 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.749+0000 7f8f0b990700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.749+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.749+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.749+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.749+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c57380000 2019-12-07T19:34:17.749+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.749+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.8s0 GRANT e40/40) v3 -- 0x558c576dd4a0 con 0x558c54c64000 2019-12-07T19:34:17.749+0000 7f8f04982700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.751131+0000 1 0.000026 2019-12-07T19:34:17.749+0000 7f8f04982700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.749+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:17.749+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.751+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 241 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=190, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 247+0+0 (crc 0 0 0) 0x558c58092c80 con 0x558c54c64000 2019-12-07T19:34:17.751+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57334b00 prio 3 cost 0 latency 0.000025 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=190, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.751+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=190, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.752+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=190, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.752+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.752+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=190, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.752+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c57334b00 prio 3 cost 0 latency 0.000748 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=190, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=190, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.752+0000 7f8f08189700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x4c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x5c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x6c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x7c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x8c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x9c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xa400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xac00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xac00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xac00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xb400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xb400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xbc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xbc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xc400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xcc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xcc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xcc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xd400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xd400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xdc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xdc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xdc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xe400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xec00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xec00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xec00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xf400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xf400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xfc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0xfc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x10c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x11c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x12c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x13c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x14c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x15c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x16c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x17c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x18c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x19c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1a400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ac00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ac00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ac00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1b400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1b400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1bc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1bc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1bc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1c400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1cc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1cc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1cc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1d400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1d400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1dc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1dc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1dc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1e400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ec00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ec00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1ec00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1f400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1f400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1fc00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x1fc00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x20c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x21c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x400~400 cache has 0x[400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x22c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x1400~400 cache has 0x[1400~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23400~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x23c00~400 = 1024 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24400~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24c00~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25400~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x25c00~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26400~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x26c00~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27400~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27c00~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x27c00~400 = 0 2019-12-07T19:34:17.752+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28400~400 2019-12-07T19:34:17.752+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28400~400 size 0x24000 (147456) 2019-12-07T19:34:17.752+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x28c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x29c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2a400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ac00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ac00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ac00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2b400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2b400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2bc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2bc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2bc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2c400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2cc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2cc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2cc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2d400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2d400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2dc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2dc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2dc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2e400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ec00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ec00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2ec00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2f400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2f400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2fc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x2fc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x30c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x31c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x32c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x33c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x34c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x35c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x36c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x37c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x38c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39c00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39c00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x39c00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3a400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ac00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ac00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ac00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3b400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3b400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3bc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3bc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3bc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3c400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3cc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3cc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3cc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3d400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3d400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3dc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3dc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3dc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3e400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ec00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ec00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3ec00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3f400~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f400~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3f400~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3fc00~400 2019-12-07T19:34:17.753+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fc00~400 size 0x24000 (147456) 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x3fc00~400 = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=73728 2019-12-07T19:34:17.753+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:34:17.753+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:17.753+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 147456 2019-12-07T19:34:17.753+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key 2019-12-07T19:34:17.753+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key = 0 2019-12-07T19:34:17.753+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=190, attrs_read=0)) v2 -- 0x558c575bac80 con 0x558c54c64000 2019-12-07T19:34:17.753+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c57334b00 finish 2019-12-07T19:34:17.753+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.754+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 819315 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.754+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 242 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 46118+0+0 (crc 0 0 0) 0x558c54dfbe00 con 0x558c54c64000 2019-12-07T19:34:17.754+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57334c60 prio 127 cost 46056 latency 0.000030 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.754+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 46056 e40) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 46056 e40) queued 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 46056 e40) pg 0x558c57380000 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 osd.7 40 dequeue_op 0x558c57334c60 prio 127 cost 46056 latency 0.000067 MOSDPGPush(1.8s5 40/39 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] on_local_recover: 1:1353159e:::smithi13140796-11:head 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'3 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=9 mbc={} ps=[a~1]] got missing 1:1353159e:::smithi13140796-11:head v 21'4 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'11 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=8 mbc={} ps=[a~1]] last_complete now 21'11 log.complete_to 21'12 2019-12-07T19:34:17.754+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c54ea1500 seq 4 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:1353159e:::smithi13140796-11:head# key 0x8580000000000000011353159e21736d'ithi13140796-11!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:1353159e:::smithi13140796-11:head# key 0x8580000000000000011353159e21736d'ithi13140796-11!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1195 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# = 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea1500 6 blobs 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0xb000/1000,b000,0 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x78d000~b000/1000,b000,0 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x78d000~b000] 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578815e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578815e0 blob([0x78d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881650 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c578815e0) get_ref 0x0~2000 Blob(0x558c578815e0 blob([0x78d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881650 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c578815e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c578815e0 blob([0x78d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881650 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57881668 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742b490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df82a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742b490 blob([0x78f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df82a0 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c5742b490) get_ref 0x0~2000 Blob(0x558c5742b490 blob([0x78f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df82a0 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c5742b490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c5742b490 blob([0x78f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df82a0 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c54df82b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396000 blob([0x791000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dd50 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c57396000) get_ref 0x0~2000 Blob(0x558c57396000 blob([0x791000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dd50 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c57396000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57396000 blob([0x791000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dd50 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c54d3dd68 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57566b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57566930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57566b60 blob([0x793000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57566930 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c57566b60) get_ref 0x0~2000 Blob(0x558c57566b60 blob([0x793000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57566930 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c57566b60) get_ref init 0x2000, 1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57566b60 blob([0x793000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57566930 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57566948 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575bd570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575bd570 blob([0x795000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4070 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c575bd570) get_ref 0x0~2000 Blob(0x558c575bd570 blob([0x795000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4070 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c575bd570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c575bd570 blob([0x795000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4070 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57ff4088 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f110 blob([0x797000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee00 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c5757f110) get_ref 0x0~1000 Blob(0x558c5757f110 blob([0x797000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757ee00 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.blob(0x558c5757f110) get_ref init 0x1000, 1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~1000: 0x0~1000 Blob(0x558c5757f110 blob([0x797000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5757ee00 sbid 0x0)) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5757ee18 in 0x558c53a59960) _discard 0x0~1000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xb000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57f34608) dirty_range mark inline shard dirty 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 = 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 4 keys 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 4 keys = 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.8CAC8A97.head.smithi13140796-11..' <- missing/0000000000000001.8CAC8A97.head.smithi13140796-11.. 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 75293 (7 ios * 4000 + 47293 bytes) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes 0x558c57f34500 shared_blobs 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57f34608) update 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57f34608) update inline shard 114 bytes from 6 extents 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:1353159e:::smithi13140796-11:head# is 605 (485 bytes onode + 2 bytes spanning blobs + 118 bytes inline extents) 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[78d000~b000] released 0x[] 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 freelist allocate 0x78d000~b000 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 freelist _xor first_key 0x780000 last_key 0x780000 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea1500 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 osd.7 40 dequeue_op 0x558c57334c60 finish 2019-12-07T19:34:17.754+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'11 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=8 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.754+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.754+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 aio_wait 2019-12-07T19:34:17.754+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:17.754+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1928 < min_flush_size 524288 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34500 had 1 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1951 < min_flush_size 524288 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1951 < min_flush_size 524288 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1260e1~79f to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1260e1 0x1260e1~79f to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf60e1 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe1 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x880 and padding block with 0x780 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x126880 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.754+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000143433s (0.000041724s flush + 0.000101709s kv commit) 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.754+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.754+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:17.754+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 0x558c57f34500 2019-12-07T19:34:17.754+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3dd68 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad290 space 0x558c54d3dd68 0x0~2000 writing nocache) 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df82b8 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad200 space 0x558c54df82b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57566948 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad320 space 0x558c57566948 0x0~2000 writing nocache) 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757ee18 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad440 space 0x558c5757ee18 0x0~1000 writing nocache) 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881668 in 0x558c53a59960) _finish_write discard buffer(0x558c57e65290 space 0x558c57881668 0x0~2000 writing nocache) 2019-12-07T19:34:17.754+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff4088 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad3b0 space 0x558c57ff4088 0x0~2000 writing nocache) 2019-12-07T19:34:17.754+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'11 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 0'0 active m=8 mbc={} ps=[a~1]] recovery_committed_to version 21'11 now ondisk 2019-12-07T19:34:17.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:17.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.755+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:17.755+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.755+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.755+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:1353159e:::smithi13140796-11:head)]) v3 -- 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.755+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.760+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 243 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 177190+0+0 (crc 0 0 0) 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.760+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57334dc0 prio 127 cost 177128 latency 0.000038 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.760+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e40) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.760+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e40) queued 2019-12-07T19:34:17.760+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.760+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e40) pg 0x558c57380000 2019-12-07T19:34:17.760+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57334dc0 prio 127 cost 177128 latency 0.000096 MOSDPGPush(1.8s5 40/39 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 21'11 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=8 mbc={} ps=[a~1]] 2019-12-07T19:34:17.760+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'11 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=8 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.760+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'11 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=8 mbc={} ps=[a~1]] on_local_recover: 1:10f769a3:::smithi13140796-42:head 2019-12-07T19:34:17.760+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 21'11 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=8 mbc={} ps=[a~1]] got missing 1:10f769a3:::smithi13140796-42:head v 21'12 2019-12-07T19:34:17.760+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 21'11 active m=7 mbc={} ps=[a~1]] last_complete now 27'23 log.complete_to 29'24 2019-12-07T19:34:17.760+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.760+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57ff1800 seq 5 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10f769a3:::smithi13140796-42:head# key 0x85800000000000000110f769a321736d'ithi13140796-42!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.760+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10f769a3:::smithi13140796-42:head# key 0x85800000000000000110f769a321736d'ithi13140796-42!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.760+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1196 2019-12-07T19:34:17.760+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# = 0 2019-12-07T19:34:17.760+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.760+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ff1800 22 blobs 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.760+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2b000/1000,2b000,0 2019-12-07T19:34:17.760+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x798000~2b000/1000,2b000,0 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x798000~2b000] 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a06d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a06930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a06d20 blob([0x798000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06930 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a06d20) get_ref 0x0~2000 Blob(0x558c57a06d20 blob([0x798000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06930 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a06d20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57a06d20 blob([0x798000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06930 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57a06948 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57397f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d2d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57397f80 blob([0x79a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2d90 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57397f80) get_ref 0x0~2000 Blob(0x558c57397f80 blob([0x79a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2d90 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57397f80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57397f80 blob([0x79a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2d90 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c574d2da8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55264690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f3500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55264690 blob([0x79c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3500 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c55264690) get_ref 0x0~2000 Blob(0x558c55264690 blob([0x79c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3500 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c55264690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c55264690 blob([0x79c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3500 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f3518 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a067e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a068c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a067e0 blob([0x79e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a068c0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a067e0) get_ref 0x0~2000 Blob(0x558c57a067e0 blob([0x79e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a068c0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a067e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57a067e0 blob([0x79e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a068c0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57a068d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a069a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a06a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a069a0 blob([0x7a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06a10 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a069a0) get_ref 0x0~2000 Blob(0x558c57a069a0 blob([0x7a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06a10 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a069a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57a069a0 blob([0x7a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06a10 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57a06a28 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a06a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a06af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a06a80 blob([0x7a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06af0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a06a80) get_ref 0x0~2000 Blob(0x558c57a06a80 blob([0x7a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06af0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a06a80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57a06a80 blob([0x7a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06af0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57a06b08 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a06b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ece00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a06b60 blob([0x7a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ece00 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a06b60) get_ref 0x0~2000 Blob(0x558c57a06b60 blob([0x7a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ece00 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c57a06b60) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57a06b60 blob([0x7a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ece00 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ece18 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ece70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ecee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ece70 blob([0x7a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecee0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ece70) get_ref 0x0~2000 Blob(0x558c580ece70 blob([0x7a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecee0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ece70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c580ece70 blob([0x7a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecee0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ecef8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ecf50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ecfc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ecf50 blob([0x7a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecfc0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ecf50) get_ref 0x0~2000 Blob(0x558c580ecf50 blob([0x7a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecfc0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ecf50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c580ecf50 blob([0x7a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecfc0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ecfd8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed030 blob([0x7aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed0a0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed030) get_ref 0x0~2000 Blob(0x558c580ed030 blob([0x7aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed0a0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed030) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c580ed030 blob([0x7aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed0a0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed0b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed110 blob([0x7ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed180 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed110) get_ref 0x0~2000 Blob(0x558c580ed110 blob([0x7ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed180 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed110) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c580ed110 blob([0x7ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed180 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed198 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed1f0 blob([0x7ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed260 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed1f0) get_ref 0x0~2000 Blob(0x558c580ed1f0 blob([0x7ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed260 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c580ed1f0 blob([0x7ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed260 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed278 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed2d0 blob([0x7b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed340 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed2d0) get_ref 0x0~2000 Blob(0x558c580ed2d0 blob([0x7b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed340 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c580ed2d0 blob([0x7b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed340 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed358 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed3b0 blob([0x7b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed420 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed3b0) get_ref 0x0~2000 Blob(0x558c580ed3b0 blob([0x7b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed420 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c580ed3b0 blob([0x7b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed420 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed438 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed490 blob([0x7b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed500 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed490) get_ref 0x0~2000 Blob(0x558c580ed490 blob([0x7b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed500 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c580ed490 blob([0x7b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed500 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed518 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed570 blob([0x7b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed5e0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed570) get_ref 0x0~2000 Blob(0x558c580ed570 blob([0x7b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed5e0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c580ed570 blob([0x7b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed5e0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed5f8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed650 blob([0x7b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed6c0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed650) get_ref 0x0~2000 Blob(0x558c580ed650 blob([0x7b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed6c0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c580ed650 blob([0x7b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed6c0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed6d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed730 blob([0x7ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed7a0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed730) get_ref 0x0~2000 Blob(0x558c580ed730 blob([0x7ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed7a0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed730) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c580ed730 blob([0x7ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed7a0 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed7b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 244 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=192, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 247+0+0 (crc 0 0 0) 0x558c575bac80 con 0x558c54c64000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed810 blob([0x7bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed880 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed810) get_ref 0x0~2000 Blob(0x558c580ed810 blob([0x7bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed880 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3fb80 prio 3 cost 0 latency 0.000021 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=192, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c580ed810 blob([0x7bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed880 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed898 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=192, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ed960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed8f0 blob([0x7be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed960 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed8f0) get_ref 0x0~2000 Blob(0x558c580ed8f0 blob([0x7be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ed960 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c580ed8f0 blob([0x7be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ed960 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ed978 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ed9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580eda40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.760+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ed9d0 blob([0x7c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580eda40 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed9d0) get_ref 0x0~2000 Blob(0x558c580ed9d0 blob([0x7c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580eda40 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580ed9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c580ed9d0 blob([0x7c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580eda40 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=192, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580eda58 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580edab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580edb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.760+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580edab0 blob([0x7c2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580edb20 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580edab0) get_ref 0x0~1000 Blob(0x558c580edab0 blob([0x7c2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580edb20 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=192, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.blob(0x558c580edab0) get_ref init 0x1000, 1000 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~1000: 0x0~1000 Blob(0x558c580edab0 blob([0x7c2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580edb20 sbid 0x0)) 2019-12-07T19:34:17.760+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580edb38 in 0x558c53a59960) _discard 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2b000 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore.extentmap(0x558c57efa108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.761+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 = 0 2019-12-07T19:34:17.760+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c57f3fb80 prio 3 cost 0 latency 0.000070 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=192, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.761+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 4 keys 2019-12-07T19:34:17.761+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 4 keys = 0 2019-12-07T19:34:17.761+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.761+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=192, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.80FE695C.head.smithi13140796-42..' <- missing/0000000000000001.80FE695C.head.smithi13140796-42.. 2019-12-07T19:34:17.761+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.761+0000 7f8f04181700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.761+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x400~400 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.761+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1800 cost 270365 (23 ios * 4000 + 178365 bytes) 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1800 onodes 0x558c57efa000 shared_blobs 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need {<0x0, 0x1000> : [0x400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore.extentmap(0x558c57efa108) update 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore.extentmap(0x558c57efa108) update inline shard 418 bytes from 22 extents 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10f769a3:::smithi13140796-42:head# is 909 (485 bytes onode + 2 bytes spanning blobs + 422 bytes inline extents) 2019-12-07T19:34:17.761+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1800 allocated 0x[798000~2b000] released 0x[] 2019-12-07T19:34:17.761+0000 7f8f04982700 10 freelist allocate 0x798000~2b000 2019-12-07T19:34:17.761+0000 7f8f04982700 20 freelist _xor first_key 0x780000 last_key 0x780000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 prepare 2019-12-07T19:34:17.761+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ff1800 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcb6d8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac1b0 space 0x558c57dcb6d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc00~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcb6d8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac240 space 0x558c57dcb6d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c00~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need {<0x0, 0x1000> : [0x2400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcac58 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac2d0 space 0x558c57dcac58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c00~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need {<0x1000, 0x1000> : [0x3400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57334dc0 finish 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.761+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 21'11 active m=7 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.761+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57dcac58 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac360 space 0x558c57dcac58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c00~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need {<0x0, 0x1000> : [0x4400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 aio_wait 2019-12-07T19:34:17.761+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1800 2019-12-07T19:34:17.761+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 io_done 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57bea088 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac3f0 space 0x558c57bea088 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4c00~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4c00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need {<0x1000, 0x1000> : [0x5400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2232 < min_flush_size 524288 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57bea088 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac480 space 0x558c57bea088 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efa000 had 1 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5c00~400 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2255 < min_flush_size 524288 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2255 < min_flush_size 524288 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5c00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x126880~8cf to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x126880 0x126880~8cf to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need {<0x0, 0x1000> : [0x6400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf6880 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x880 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x14f and padding block with 0xeb1 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12714f 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57881198 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac510 space 0x558c57881198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6c00~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000179269s (0.000049242s flush + 0.000130027s kv commit) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.761+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6c00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need {<0x1000, 0x1000> : [0x7400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x7400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7400:1400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57881198 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac5a0 space 0x558c57881198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7400~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7c00~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.761+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7c00~400 = 1024 2019-12-07T19:34:17.761+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8400~400 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need {<0x0, 0x1000> : [0x8400:400~400]} 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.761+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff1800> 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 kv_submitted 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1800 2019-12-07T19:34:17.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 finishing 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1800 onodes 0x558c57efa000 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d2da8 in 0x558c53a59960) _finish_write discard buffer(0x558c57a08870 space 0x558c574d2da8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3518 in 0x558c53a59960) _finish_write discard buffer(0x558c57a08900 space 0x558c577f3518 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a068d8 in 0x558c53a59960) _finish_write discard buffer(0x558c57a08990 space 0x558c57a068d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57881438 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a06948 in 0x558c53a59960) _finish_write discard buffer(0x558c57e65170 space 0x558c57a06948 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a06a28 in 0x558c53a59960) _finish_write discard buffer(0x558c57a08a20 space 0x558c57a06a28 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac630 space 0x558c57881438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a06b08 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad8c0 space 0x558c57a06b08 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ece18 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad950 space 0x558c580ece18 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ecef8 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad9e0 space 0x558c580ecef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8c00~400 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ecfd8 in 0x558c53a59960) _finish_write discard buffer(0x558c578ada70 space 0x558c580ecfd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x8c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed0b8 in 0x558c53a59960) _finish_write discard buffer(0x558c578adb00 space 0x558c580ed0b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed198 in 0x558c53a59960) _finish_write discard buffer(0x558c578adb90 space 0x558c580ed198 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 21'11 active m=7 mbc={} ps=[a~1]] recovery_committed_to version 27'23 now ondisk 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed278 in 0x558c53a59960) _finish_write discard buffer(0x558c578adc20 space 0x558c580ed278 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed358 in 0x558c53a59960) _finish_write discard buffer(0x558c578adcb0 space 0x558c580ed358 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed438 in 0x558c53a59960) _finish_write discard buffer(0x558c578add40 space 0x558c580ed438 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed518 in 0x558c53a59960) _finish_write discard buffer(0x558c578addd0 space 0x558c580ed518 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed5f8 in 0x558c53a59960) _finish_write discard buffer(0x558c578ade60 space 0x558c580ed5f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x8c00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed6d8 in 0x558c53a59960) _finish_write discard buffer(0x558c578adef0 space 0x558c580ed6d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed7b8 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb8000 space 0x558c580ed7b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9400~400 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed898 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb8090 space 0x558c580ed898 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ed978 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb8120 space 0x558c580ed978 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580eda58 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb81b0 space 0x558c580eda58 0x0~2000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580edb38 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb8240 space 0x558c580edb38 0x0~1000 writing nocache) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:10f769a3:::smithi13140796-42:head)]) v3 -- 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need {<0x1000, 0x1000> : [0x9400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1800 done 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x9400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.762+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1800 [] 2019-12-07T19:34:17.762+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.762+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.762+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57881438 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac6c0 space 0x558c57881438 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9c00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x9c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x9c00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xa400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need {<0x0, 0x1000> : [0xa400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57881358 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac750 space 0x558c57881358 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xa400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xac00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xac00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xb400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xb400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need {<0x1000, 0x1000> : [0xb400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xb400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57881358 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c578ac7e0 space 0x558c57881358 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xb400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xbc00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xbc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xbc00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xc400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need {<0x0, 0x1000> : [0xc400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57880f68 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8750 space 0x558c57880f68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xc400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xcc00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xcc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xcc00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xd400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xd400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need {<0x1000, 0x1000> : [0xd400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xd400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57880f68 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb87e0 space 0x558c57880f68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xd400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xdc00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xdc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xdc00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xe400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need {<0x0, 0x1000> : [0xe400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de408 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8870 space 0x558c578de408 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xe400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xec00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xec00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xf400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xf400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need {<0x1000, 0x1000> : [0xf400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xf400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 245 ==== MOSDPGRecoveryDelete(1.8s5 e40,39 1:10779340:::smithi13140796-32:4,29'24) v2 ==== 123+0+0 (crc 0 0 0) 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de408 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8900 space 0x558c578de408 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xf400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3e9a0 prio 3 cost 1000 latency 0.000017 epoch 40 MOSDPGRecoveryDelete(1.8s5 e40,39 1:10779340:::smithi13140796-32:4,29'24) v2 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xfc00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0xfc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:10779340:::smithi13140796-32:4,29'24) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0xfc00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need {<0x0, 0x1000> : [0x10400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.762+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:10779340:::smithi13140796-32:4,29'24) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.762+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:10779340:::smithi13140796-32:4,29'24) v2) prio 3 cost 1000 e40) pg 0x558c57380000 2019-12-07T19:34:17.762+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57f3e9a0 prio 3 cost 1000 latency 0.000064 MOSDPGRecoveryDelete(1.8s5 e40,39 1:10779340:::smithi13140796-32:4,29'24) v2 pg pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=7 mbc={} ps=[a~1]] 2019-12-07T19:34:17.762+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=7 mbc={} ps=[a~1]] handle_recovery_delete 0x558c57f3e9a0 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10400:400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de6a8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.762+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8990 space 0x558c578de6a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10400~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10c00~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x10c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=7 mbc={} ps=[a~1]] remove_missing_object 1:10779340:::smithi13140796-32:4 29'24 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.762+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x10c00~400 = 1024 2019-12-07T19:34:17.762+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11400~400 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.762+0000 7f8f04982700 20 snap_mapper.remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.762+0000 7f8f04982700 20 snap_mapper._remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need {<0x1000, 0x1000> : [0x11400:1400~400]} 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x11400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.762+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.762+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.762+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.762+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.762+0000 7f8f04982700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:4 got.empty() 2019-12-07T19:34:17.762+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.762+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57ff1b00 seq 6 2019-12-07T19:34:17.763+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10779340:::smithi13140796-32:4# key 0x858000000000000001107793'@!smithi13140796-32!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:17.763+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.763+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:10779340:::smithi13140796-32:4# 2019-12-07T19:34:17.763+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1b00 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.763+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1b00 onodes shared_blobs 2019-12-07T19:34:17.763+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.763+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 prepare 2019-12-07T19:34:17.763+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1b00 2019-12-07T19:34:17.763+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 io_done 2019-12-07T19:34:17.763+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57f3e9a0 finish 2019-12-07T19:34:17.763+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de6a8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8a20 space 0x558c578de6a8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11c00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x11c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x11c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need {<0x0, 0x1000> : [0x12400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12714f~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12714f 0x12714f~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf714f 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x14f 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x17d and padding block with 0xe83 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de4e8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8ab0 space 0x558c578de4e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12c00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x12c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x12c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x13400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x13400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de4e8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8b40 space 0x558c578de4e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12717d 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13c00~400 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x13c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x13c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000125371s (0.000000110s flush + 0.000125261s kv commit) 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14400~400 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff1b00> 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 kv_submitted 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1b00 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need {<0x0, 0x1000> : [0x14400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 finishing 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1b00 onodes 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1b00 done 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.763+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1b00 [] 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=7 mbc={} ps=[a~1]] on_local_recover: 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 snap_mapper.remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 snap_mapper._remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578deb08 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8bd0 space 0x558c578deb08 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14c00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x14c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:4 got.empty() 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x14c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 27'23 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=7 mbc={} ps=[a~1]] got missing 1:10779340:::smithi13140796-32:4 v 29'24 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need {<0x1000, 0x1000> : [0x15400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x15400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 27'23 active m=6 mbc={} ps=[a~1]] last_complete now 29'24 log.complete_to 29'25 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578deb08 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8c60 space 0x558c578deb08 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15c00~400 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x15c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c574fe300 seq 7 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x15c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.80EE9C20.4.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need {<0x0, 0x1000> : [0x16400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe300 cost 5406 (1 ios * 4000 + 1406 bytes) 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe300 onodes shared_blobs 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 prepare 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe300 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 io_done 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de948 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8cf0 space 0x558c578de948 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16c00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x16c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1182 < min_flush_size 524288 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 27'23 active m=6 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x16c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17400~400 2019-12-07T19:34:17.763+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need {<0x1000, 0x1000> : [0x17400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x17400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12717d~4b5 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12717d 0x12717d~4b5 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf717d 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x17d 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x632 and padding block with 0x9ce 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578de948 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8d80 space 0x558c578de948 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17c00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x17c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x17c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need {<0x0, 0x1000> : [0x18400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x127632 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.763+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000135469s (0.000000081s flush + 0.000135388s kv commit) 2019-12-07T19:34:17.763+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe300> 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 kv_submitted 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe300 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 finishing 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe300 onodes 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe300 done 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe300 [] 2019-12-07T19:34:17.763+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.763+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.763+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 27'23 active m=6 mbc={} ps=[a~1]] recovery_committed_to version 29'24 now ondisk 2019-12-07T19:34:17.763+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGRecoveryDeleteReply(1.8s0 e40,39 1:10779340:::smithi13140796-32:4,29'24) v2 -- 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.763+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.763+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578deef8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8e10 space 0x558c578deef8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18c00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x18c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x18c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x19400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578deef8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8ea0 space 0x558c578deef8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19c00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x19c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x19c00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1a400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a400:400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578defd8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8f30 space 0x558c578defd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1a400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ac00~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ac00~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1b400~400 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1b400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1b400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b400:1400~400]} 2019-12-07T19:34:17.763+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578defd8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.763+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb8fc0 space 0x558c578defd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.763+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1b400~400 = 1024 2019-12-07T19:34:17.763+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1bc00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1bc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1bc00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1c400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need {<0x0, 0x1000> : [0x1c400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578df0b8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9050 space 0x558c578df0b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1c400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1cc00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1cc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1cc00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1d400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1d400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1d400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578df0b8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb90e0 space 0x558c578df0b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1d400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1dc00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1dc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1dc00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1e400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need {<0x0, 0x1000> : [0x1e400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578df198 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9170 space 0x558c578df198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1e400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ec00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1ec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1ec00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1f400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1f400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1f400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578df198 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9200 space 0x558c578df198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1f400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1fc00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x1fc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x1fc00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need {<0x0, 0x1000> : [0x20400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c018 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9290 space 0x558c54e1c018 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20c00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x20c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x20c00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need {<0x1000, 0x1000> : [0x21400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x21400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c018 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9320 space 0x558c54e1c018 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21c00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x21c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x21c00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need {<0x0, 0x1000> : [0x22400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c0f8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb93b0 space 0x558c54e1c0f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22c00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x22c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x22c00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x23400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c0f8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9440 space 0x558c54e1c0f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23c00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x23c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x23c00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need {<0x0, 0x1000> : [0x24400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x24400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c948 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb94d0 space 0x558c54e1c948 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24c00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x24c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x24c00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need {<0x1000, 0x1000> : [0x25400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x25400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x25400:1400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c948 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9560 space 0x558c54e1c948 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25400~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25c00~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x25c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.764+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x25c00~400 = 1024 2019-12-07T19:34:17.764+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26400~400 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need {<0x0, 0x1000> : [0x26400:400~400]} 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.764+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x26400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c868 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb95f0 space 0x558c54e1c868 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26c00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x26c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x26c00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need {<0x1000, 0x1000> : [0x27400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x27400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x27400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c868 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9680 space 0x558c54e1c868 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27c00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x27c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x27c00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need {<0x0, 0x1000> : [0x28400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x28400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578f17b8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9710 space 0x558c578f17b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28c00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x28c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x28c00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x29400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x29400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x29400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578f17b8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb97a0 space 0x558c578f17b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29c00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x29c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x29c00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2a400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need {<0x0, 0x1000> : [0x2a400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2a400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578f1748 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9830 space 0x558c578f1748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2a400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ac00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ac00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2b400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2b400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need {<0x1000, 0x1000> : [0x2b400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2b400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2b400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578f1748 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb98c0 space 0x558c578f1748 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2b400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2bc00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2bc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2bc00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2c400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need {<0x0, 0x1000> : [0x2c400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2c400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c574e1cf8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9950 space 0x558c574e1cf8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2c400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2cc00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2cc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2cc00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2d400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2d400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need {<0x1000, 0x1000> : [0x2d400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2d400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2d400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c574e1cf8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb99e0 space 0x558c574e1cf8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2d400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2dc00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2dc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2dc00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2e400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need {<0x0, 0x1000> : [0x2e400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2e400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c574e0e18 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9a70 space 0x558c574e0e18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2e400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ec00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2ec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2ec00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2f400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2f400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need {<0x1000, 0x1000> : [0x2f400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2f400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2f400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c574e0e18 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9b00 space 0x558c574e0e18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2f400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2fc00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x2fc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x2fc00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need {<0x0, 0x1000> : [0x30400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x30400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578ded38 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9b90 space 0x558c578ded38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30c00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x30c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x30c00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need {<0x1000, 0x1000> : [0x31400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x31400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x31400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578ded38 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9c20 space 0x558c578ded38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31c00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x31c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x31c00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need {<0x0, 0x1000> : [0x32400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x32400:400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578dee18 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9cb0 space 0x558c578dee18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32c00~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x32c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x32c00~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33400~400 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need {<0x1000, 0x1000> : [0x33400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x33400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x33400:1400~400]} 2019-12-07T19:34:17.765+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578dee18 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.765+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9d40 space 0x558c578dee18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.765+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33400~400 = 1024 2019-12-07T19:34:17.765+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33c00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x33c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x33c00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need {<0x0, 0x1000> : [0x34400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x34400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x34400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c788 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9dd0 space 0x558c54e1c788 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34c00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x34c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x34c00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need {<0x1000, 0x1000> : [0x35400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x35400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x35400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c788 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9e60 space 0x558c54e1c788 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35c00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x35c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x35c00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need {<0x0, 0x1000> : [0x36400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x36400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x36400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c6a8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c57bb9ef0 space 0x558c54e1c6a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36c00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x36c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x36c00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need {<0x1000, 0x1000> : [0x37400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x37400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x37400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c6a8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e4000 space 0x558c54e1c6a8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37c00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x37c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x37c00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need {<0x0, 0x1000> : [0x38400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x38400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x38400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c5c8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e4090 space 0x558c54e1c5c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38c00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x38c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x38c00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need {<0x1000, 0x1000> : [0x39400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x39400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x39400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c5c8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e4120 space 0x558c54e1c5c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39c00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x39c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x39c00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3a400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need {<0x0, 0x1000> : [0x3a400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3a400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3a400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c4e8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e41b0 space 0x558c54e1c4e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3a400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ac00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ac00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3b400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3b400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3b400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3b400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3b400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1c4e8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e4240 space 0x558c54e1c4e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3b400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3bc00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3bc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3bc00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3c400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need {<0x0, 0x1000> : [0x3c400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3c400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3c400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1ca28 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e42d0 space 0x558c54e1ca28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3c400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3cc00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3cc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3cc00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3d400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3d400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need {<0x1000, 0x1000> : [0x3d400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3d400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3d400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1ca28 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e4360 space 0x558c54e1ca28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3d400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3dc00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3dc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3dc00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3e400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need {<0x0, 0x1000> : [0x3e400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3e400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3e400:400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1cb08 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e43f0 space 0x558c54e1cb08 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3e400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ec00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3ec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3ec00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3f400~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3f400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3f400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3f400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3f400:1400~400]} 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c54e1cb08 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.766+0000 7f8f04181700 20 _add level 1 near 0 on buffer(0x558c550e4480 space 0x558c54e1cb08 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3f400~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3fc00~400 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x3fc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.766+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x3fc00~400 = 1024 2019-12-07T19:34:17.766+0000 7f8f04181700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=1024 len=131072 2019-12-07T19:34:17.766+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:17.766+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:17.766+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:17.766+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key 2019-12-07T19:34:17.766+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key = 0 2019-12-07T19:34:17.766+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=192, attrs_read=0)) v2 -- 0x558c5793b900 con 0x558c54c64000 2019-12-07T19:34:17.766+0000 7f8f04181700 10 osd.7 40 dequeue_op 0x558c57f3fb80 finish 2019-12-07T19:34:17.766+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.770+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 246 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 263206+0+0 (crc 0 0 0) 0x558c575b9a00 con 0x558c54c64000 2019-12-07T19:34:17.770+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3ec60 prio 127 cost 263144 latency 0.000034 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.770+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) queued 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) pg 0x558c57380000 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 osd.7 40 dequeue_op 0x558c57f3ec60 prio 127 cost 263144 latency 0.000088 MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] get_temp_recovery_object -3:10779340:::temp_recovering_1.8s5_29'25_39_head:head 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] handle_recovery_push: Adding oid -3:10779340:::temp_recovering_1.8s5_29'25_39_head:head in the temp collection 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c54ddfb00 seq 8 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# key 0x857ffffffffffffffd107793'@!temp_recovering_1.8s5_29'25_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# key 0x857ffffffffffffffd107793'@!temp_recovering_1.8s5_29'25_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.770+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1197 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# = 0 2019-12-07T19:34:17.770+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 0x0~40000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 0x0~40000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~40000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ddfb00 32 blobs 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0x40000/1000,40000,0 2019-12-07T19:34:17.770+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x7c3000~40000/1000,40000,0 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x7c3000~40000] 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcb490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e631f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcb490 blob([0x7c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e631f0 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57dcb490) get_ref 0x0~2000 Blob(0x558c57dcb490 blob([0x7c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e631f0 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57dcb490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57dcb490 blob([0x7c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e631f0 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57e63208 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742b500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8690 blob([0x7c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b500 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c54df8690) get_ref 0x0~2000 Blob(0x558c54df8690 blob([0x7c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b500 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c54df8690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54df8690 blob([0x7c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b500 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5742b518 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3b810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f67d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3b810 blob([0x7c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67d50 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3b810) get_ref 0x0~2000 Blob(0x558c57f3b810 blob([0x7c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67d50 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f3b810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57f3b810 blob([0x7c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67d50 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f67d68 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f67dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f67e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f67dc0 blob([0x7c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67e30 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f67dc0) get_ref 0x0~2000 Blob(0x558c57f67dc0 blob([0x7c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67e30 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f67dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57f67dc0 blob([0x7c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67e30 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f67e48 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f67ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f67f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f67ea0 blob([0x7cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67f10 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f67ea0) get_ref 0x0~2000 Blob(0x558c57f67ea0 blob([0x7cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67f10 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f67ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57f67ea0 blob([0x7cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67f10 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f67f28 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f67f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f67f80 blob([0x7cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6d20 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f67f80) get_ref 0x0~2000 Blob(0x558c57f67f80 blob([0x7cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6d20 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57f67f80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57f67f80 blob([0x7cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6d20 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb6d38 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6d90 blob([0x7cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6e00 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb6d90) get_ref 0x0~2000 Blob(0x558c57bb6d90 blob([0x7cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6e00 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb6d90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57bb6d90 blob([0x7cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6e00 sbid 0x0)) 2019-12-07T19:34:17.770+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb6e18 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6e70 blob([0x7d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6ee0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb6e70) get_ref 0x0~2000 Blob(0x558c57bb6e70 blob([0x7d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6ee0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb6e70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57bb6e70 blob([0x7d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6ee0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb6ef8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6f50 blob([0x7d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6fc0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb6f50) get_ref 0x0~2000 Blob(0x558c57bb6f50 blob([0x7d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6fc0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb6f50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57bb6f50 blob([0x7d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6fc0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb6fd8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb70a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7030 blob([0x7d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb70a0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7030) get_ref 0x0~2000 Blob(0x558c57bb7030 blob([0x7d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb70a0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7030) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57bb7030 blob([0x7d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb70a0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb70b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7110 blob([0x7d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7180 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7110) get_ref 0x0~2000 Blob(0x558c57bb7110 blob([0x7d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7180 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7110) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57bb7110 blob([0x7d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7180 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7198 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb71f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb71f0 blob([0x7d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7260 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb71f0) get_ref 0x0~2000 Blob(0x558c57bb71f0 blob([0x7d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7260 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb71f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57bb71f0 blob([0x7d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7260 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7278 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb72d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb72d0 blob([0x7db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7340 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb72d0) get_ref 0x0~2000 Blob(0x558c57bb72d0 blob([0x7db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7340 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb72d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c57bb72d0 blob([0x7db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7340 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7358 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb73b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb73b0 blob([0x7dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7420 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb73b0) get_ref 0x0~2000 Blob(0x558c57bb73b0 blob([0x7dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7420 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb73b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57bb73b0 blob([0x7dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7420 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7438 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7490 blob([0x7df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7500 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7490) get_ref 0x0~2000 Blob(0x558c57bb7490 blob([0x7df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7500 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57bb7490 blob([0x7df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7500 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7518 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb75e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7570 blob([0x7e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb75e0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7570) get_ref 0x0~2000 Blob(0x558c57bb7570 blob([0x7e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb75e0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57bb7570 blob([0x7e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb75e0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb75f8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb76c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7650 blob([0x7e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb76c0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7650) get_ref 0x0~2000 Blob(0x558c57bb7650 blob([0x7e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb76c0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57bb7650 blob([0x7e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb76c0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb76d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb77a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7730 blob([0x7e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb77a0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7730) get_ref 0x0~2000 Blob(0x558c57bb7730 blob([0x7e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb77a0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7730) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57bb7730 blob([0x7e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb77a0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb77b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7810 blob([0x7e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7880 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7810) get_ref 0x0~2000 Blob(0x558c57bb7810 blob([0x7e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7880 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57bb7810 blob([0x7e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7880 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7898 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb78f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb78f0 blob([0x7e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7960 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb78f0) get_ref 0x0~2000 Blob(0x558c57bb78f0 blob([0x7e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7960 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb78f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57bb78f0 blob([0x7e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7960 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7978 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb79d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb79d0 blob([0x7eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7a40 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb79d0) get_ref 0x0~2000 Blob(0x558c57bb79d0 blob([0x7eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7a40 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb79d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57bb79d0 blob([0x7eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7a40 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7a58 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7ab0 blob([0x7ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7b20 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7ab0) get_ref 0x0~2000 Blob(0x558c57bb7ab0 blob([0x7ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7b20 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57bb7ab0 blob([0x7ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7b20 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7b38 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7b90 blob([0x7ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7c00 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7b90) get_ref 0x0~2000 Blob(0x558c57bb7b90 blob([0x7ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7c00 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7b90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57bb7b90 blob([0x7ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7c00 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7c18 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7c70 blob([0x7f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ce0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7c70) get_ref 0x0~2000 Blob(0x558c57bb7c70 blob([0x7f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ce0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7c70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57bb7c70 blob([0x7f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ce0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7cf8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7d50 blob([0x7f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7dc0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7d50) get_ref 0x0~2000 Blob(0x558c57bb7d50 blob([0x7f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7dc0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7d50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57bb7d50 blob([0x7f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7dc0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7dd8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7e30 blob([0x7f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ea0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7e30) get_ref 0x0~2000 Blob(0x558c57bb7e30 blob([0x7f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ea0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7e30) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c57bb7e30 blob([0x7f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ea0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7eb8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb7f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb7f10 blob([0x7f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7f80 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7f10) get_ref 0x0~2000 Blob(0x558c57bb7f10 blob([0x7f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7f80 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c57bb7f10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c57bb7f10 blob([0x7f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7f80 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57bb7f98 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840e000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840e070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840e000 blob([0x7f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e070 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e000) get_ref 0x0~2000 Blob(0x558c5840e000 blob([0x7f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e070 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c5840e000 blob([0x7f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e070 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840e088 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840e0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840e150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840e0e0 blob([0x7fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e150 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e0e0) get_ref 0x0~2000 Blob(0x558c5840e0e0 blob([0x7fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e150 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c5840e0e0 blob([0x7fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e150 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840e168 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840e1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840e230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840e1c0 blob([0x7fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e230 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e1c0) get_ref 0x0~2000 Blob(0x558c5840e1c0 blob([0x7fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e230 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c5840e1c0 blob([0x7fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e230 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840e248 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840e2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840e310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840e2a0 blob([0x7ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e310 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e2a0) get_ref 0x0~2000 Blob(0x558c5840e2a0 blob([0x7ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e310 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c5840e2a0 blob([0x7ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e310 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840e328 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840e380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840e3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840e380 blob([0x801000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e3f0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e380) get_ref 0x0~2000 Blob(0x558c5840e380 blob([0x801000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e3f0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.blob(0x558c5840e380) get_ref init 0x2000, 1000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c5840e380 blob([0x801000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e3f0 sbid 0x0)) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840e408 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x40000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.771+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 0x0~40000 = 0 2019-12-07T19:34:17.771+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 4 keys 2019-12-07T19:34:17.771+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 4 keys = 0 2019-12-07T19:34:17.771+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddfb00 cost 395060 (33 ios * 4000 + 263060 bytes) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddfb00 onodes 0x558c5840c000 shared_blobs 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update inline shard 611 bytes from 32 extents 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# is 1102 (485 bytes onode + 2 bytes spanning blobs + 615 bytes inline extents) 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddfb00 allocated 0x[7c3000~40000] released 0x[] 2019-12-07T19:34:17.771+0000 7f8f0898a700 10 freelist allocate 0x7c3000~40000 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 freelist _xor first_key 0x780000 last_key 0x800000 2019-12-07T19:34:17.771+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 prepare 2019-12-07T19:34:17.771+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ddfb00 2019-12-07T19:34:17.771+0000 7f8f0898a700 10 osd.7 40 dequeue_op 0x558c57f3ec60 finish 2019-12-07T19:34:17.771+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.771+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.771+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 aio_wait 2019-12-07T19:34:17.771+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddfb00 2019-12-07T19:34:17.771+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 io_done 2019-12-07T19:34:17.771+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.771+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.771+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.771+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1310 < min_flush_size 524288 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c000 had 1 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1333 < min_flush_size 524288 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1333 < min_flush_size 524288 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x127632~535 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x127632 0x127632~535 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf7632 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x632 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb67 and padding block with 0x499 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x127b67 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.772+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000158824s (0.000046111s flush + 0.000112713s kv commit) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.772+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddfb00> 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.772+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 kv_submitted 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddfb00 2019-12-07T19:34:17.772+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 finishing 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddfb00 onodes 0x558c5840c000 2019-12-07T19:34:17.772+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742b518 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad5f0 space 0x558c5742b518 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6d38 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad830 space 0x558c57bb6d38 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6e18 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4ab0 space 0x558c57bb6e18 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6ef8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4b40 space 0x558c57bb6ef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6fd8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4bd0 space 0x558c57bb6fd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb70b8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4c60 space 0x558c57bb70b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7198 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4cf0 space 0x558c57bb7198 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7278 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4d80 space 0x558c57bb7278 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7358 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4e10 space 0x558c57bb7358 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:10779340:::smithi13140796-32:head)]) v3 -- 0x558c54dfbe00 con 0x558c54c64000 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7438 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4ea0 space 0x558c57bb7438 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7518 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4f30 space 0x558c57bb7518 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb75f8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e4fc0 space 0x558c57bb75f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb76d8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5050 space 0x558c57bb76d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb77b8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e50e0 space 0x558c57bb77b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7898 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5170 space 0x558c57bb7898 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7978 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5200 space 0x558c57bb7978 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7a58 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5290 space 0x558c57bb7a58 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7b38 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5320 space 0x558c57bb7b38 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7c18 in 0x558c53a59960) _finish_write discard buffer(0x558c550e53b0 space 0x558c57bb7c18 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7cf8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5440 space 0x558c57bb7cf8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7dd8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e54d0 space 0x558c57bb7dd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7eb8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5560 space 0x558c57bb7eb8 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb7f98 in 0x558c53a59960) _finish_write discard buffer(0x558c550e55f0 space 0x558c57bb7f98 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e63208 in 0x558c53a59960) _finish_write discard buffer(0x558c57a08750 space 0x558c57e63208 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f67d68 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad680 space 0x558c57f67d68 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f67e48 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad710 space 0x558c57f67e48 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f67f28 in 0x558c53a59960) _finish_write discard buffer(0x558c578ad7a0 space 0x558c57f67f28 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840e088 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5680 space 0x558c5840e088 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840e168 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5710 space 0x558c5840e168 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840e248 in 0x558c53a59960) _finish_write discard buffer(0x558c550e57a0 space 0x558c5840e248 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840e328 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5830 space 0x558c5840e328 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840e408 in 0x558c53a59960) _finish_write discard buffer(0x558c550e58c0 space 0x558c5840e408 0x0~2000 writing nocache) 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddfb00 done 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.772+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddfb00 [] 2019-12-07T19:34:17.772+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.772+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.775+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 247 ==== MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=195, to_read={1:9b8adabe:::smithi13140796-6:head=262144,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 ==== 247+0+0 (crc 0 0 0) 0x558c5793b900 con 0x558c54c64000 2019-12-07T19:34:17.775+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3e6e0 prio 127 cost 0 latency 0.000019 epoch 40 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=195, to_read={1:9b8adabe:::smithi13140796-6:head=262144,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.775+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=195, to_read={1:9b8adabe:::smithi13140796-6:head=262144,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=195, to_read={1:9b8adabe:::smithi13140796-6:head=262144,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) queued 2019-12-07T19:34:17.775+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=195, to_read={1:9b8adabe:::smithi13140796-6:head=262144,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3) prio 127 cost 0 e40) pg 0x558c5500a000 2019-12-07T19:34:17.775+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c57f3e6e0 prio 127 cost 0 latency 0.000063 MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=195, to_read={1:9b8adabe:::smithi13140796-6:head=262144,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.775+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.9s4 40/39 ECSubRead(tid=195, to_read={1:9b8adabe:::smithi13140796-6:head=262144,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[2,2,6,2]}, attrs_to_read=)) v3 2019-12-07T19:34:17.775+0000 7f8f08189700 25 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:17.775+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x40400~400 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need {<0x0, 0x1000> : [0x40400:400~400]} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x40400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x40400:400~400]} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cbe8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.775+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca000 space 0x558c54e1cbe8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.775+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x40400~400 = 1024 2019-12-07T19:34:17.775+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x40c00~400 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x40c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.775+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x40c00~400 = 1024 2019-12-07T19:34:17.775+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x41400~400 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need {<0x1000, 0x1000> : [0x41400:1400~400]} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x41400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x41400:1400~400]} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cbe8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.775+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca090 space 0x558c54e1cbe8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.775+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x41400~400 = 1024 2019-12-07T19:34:17.775+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x41c00~400 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x41c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.775+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x41c00~400 = 1024 2019-12-07T19:34:17.775+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x42400~400 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need {<0x0, 0x1000> : [0x42400:400~400]} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x42400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.775+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 248 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 119384+0+0 (crc 0 0 0) 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.775+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57f3f8c0 prio 127 cost 119784 latency 0.000025 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.775+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e40) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x42400:400~400]} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1ccc8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.775+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca120 space 0x558c54e1ccc8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.775+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.775+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x42400~400 = 1024 2019-12-07T19:34:17.775+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x42c00~400 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x42c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.775+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x42c00~400 = 1024 2019-12-07T19:34:17.775+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x43400~400 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.775+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e40) queued 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need {<0x1000, 0x1000> : [0x43400:1400~400]} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x43400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.775+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.775+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.775+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e40) pg 0x558c57380000 2019-12-07T19:34:17.775+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 prio 127 cost 119784 latency 0.000081 MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] 2019-12-07T19:34:17.775+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:10779340:::smithi13140796-32:head, version: 29'25, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@29'25, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.775+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] get_temp_recovery_object -3:10779340:::temp_recovering_1.8s5_29'25_39_head:head 2019-12-07T19:34:17.775+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] handle_recovery_push: Removing oid -3:10779340:::temp_recovering_1.8s5_29'25_39_head:head from the temp collection 2019-12-07T19:34:17.776+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] on_local_recover: 1:10779340:::smithi13140796-32:head 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x43400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'24 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=6 mbc={} ps=[a~1]] got missing 1:10779340:::smithi13140796-32:head v 29'25 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1ccc8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca1b0 space 0x558c54e1ccc8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x43400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x43c00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x43c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'24 active m=5 mbc={} ps=[a~1]] last_complete now 29'25 log.complete_to 29'26 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x43c00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x44400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need {<0x0, 0x1000> : [0x44400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x44400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57ff0000 seq 9 2019-12-07T19:34:17.776+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 0x40000~1d000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 0x40000~1d000 - have 0x40000 (262144) bytes fadvise_flags 0x0 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x44400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cda8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca240 space 0x558c54e1cda8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x40000~1d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x44400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x44c00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x44c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x44c00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x45400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need {<0x1000, 0x1000> : [0x45400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ff0000 15 blobs 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x45400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.776+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1d000/1000,1d000,0 2019-12-07T19:34:17.776+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x803000~1d000/1000,1d000,0 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x803000~1d000] 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ec000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ec000 blob([0x803000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec070 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c580ec000) get_ref 0x0~2000 Blob(0x558c580ec000 blob([0x803000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec070 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c580ec000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c580ec000 blob([0x803000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec070 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ec088 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a06770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a06770 blob([0x805000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6540 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57a06770) get_ref 0x0~2000 Blob(0x558c57a06770 blob([0x805000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6540 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57a06770) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c57a06770 blob([0x805000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6540 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6558 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb65b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb65b0 blob([0x807000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6620 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb65b0) get_ref 0x0~2000 Blob(0x558c57bb65b0 blob([0x807000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6620 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb65b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c57bb65b0 blob([0x807000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6620 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6638 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6690 blob([0x809000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6700 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6690) get_ref 0x0~2000 Blob(0x558c57bb6690 blob([0x809000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6700 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x45400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c57bb6690 blob([0x809000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6700 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6718 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cda8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca2d0 space 0x558c54e1cda8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb67e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6770 blob([0x80b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb67e0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x45400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6770) get_ref 0x0~2000 Blob(0x558c57bb6770 blob([0x80b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb67e0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6770) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x45c00~400 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c57bb6770 blob([0x80b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb67e0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb67f8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x45c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb68c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6850 blob([0x80d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb68c0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6850) get_ref 0x0~2000 Blob(0x558c57bb6850 blob([0x80d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb68c0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6850) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x45c00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c57bb6850 blob([0x80d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb68c0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x46400~400 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb68d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb69a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6930 blob([0x80f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb69a0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need {<0x0, 0x1000> : [0x46400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x46400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6930) get_ref 0x0~2000 Blob(0x558c57bb6930 blob([0x80f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb69a0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6930) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c57bb6930 blob([0x80f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb69a0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb69b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6a10 blob([0x811000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6a80 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6a10) get_ref 0x0~2000 Blob(0x558c57bb6a10 blob([0x811000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6a80 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6a10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c57bb6a10 blob([0x811000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6a80 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6a98 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6af0 blob([0x813000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6b60 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6af0) get_ref 0x0~2000 Blob(0x558c57bb6af0 blob([0x813000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6b60 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6af0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c57bb6af0 blob([0x813000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6b60 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6b78 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6bd0 blob([0x815000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6c40 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6bd0) get_ref 0x0~2000 Blob(0x558c57bb6bd0 blob([0x815000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6c40 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c57bb6bd0 blob([0x815000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6c40 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6c58 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840fab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6cb0 blob([0x817000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fab0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6cb0) get_ref 0x0~2000 Blob(0x558c57bb6cb0 blob([0x817000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fab0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c57bb6cb0 blob([0x817000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fab0 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5840fac8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840fb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840fb20 blob([0x819000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb90 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fb20) get_ref 0x0~2000 Blob(0x558c5840fb20 blob([0x819000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb90 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fb20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c5840fb20 blob([0x819000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fb90 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5840fba8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840fc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840fc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840fc00 blob([0x81b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fc70 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fc00) get_ref 0x0~2000 Blob(0x558c5840fc00 blob([0x81b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fc70 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fc00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c5840fc00 blob([0x81b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fc70 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5840fc88 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840fce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840fd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840fce0 blob([0x81d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fd50 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fce0) get_ref 0x0~2000 Blob(0x558c5840fce0 blob([0x81d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fd50 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c5840fce0 blob([0x81d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fd50 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5840fd68 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840fdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840fe30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840fdc0 blob([0x81f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fe30 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fdc0) get_ref 0x0~1000 Blob(0x558c5840fdc0 blob([0x81f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fe30 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.blob(0x558c5840fdc0) get_ref init 0x1000, 1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~1000: 0x0~1000 Blob(0x558c5840fdc0 blob([0x81f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5840fe30 sbid 0x0)) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5840fe48 in 0x558c53a59960) _discard 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5d000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c108) dirty_range mark inline shard dirty 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# 0x40000~1d000 = 0 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10779340:::smithi13140796-32:head# key 0x858000000000000001107793'@!smithi13140796-32!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x46400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1ce88 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca360 space 0x558c54e1ce88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10779340:::smithi13140796-32:head# key 0x858000000000000001107793'@!smithi13140796-32!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x46400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x46c00~400 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x46c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# -> 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x46c00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x47400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_29'25_39_head:head# -> 5#1:10779340:::smithi13140796-32:head# = 0 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need {<0x1000, 0x1000> : [0x47400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x47400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.80EE9C20.head.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.head.smithi13140796-32.. 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.776+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0000 cost 184571 (16 ios * 4000 + 120571 bytes) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0000 onodes 0x558c5840c000 shared_blobs 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c108) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c108) update inline shard 893 bytes from 47 extents 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 1384 (485 bytes onode + 2 bytes spanning blobs + 897 bytes inline extents) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0000 allocated 0x[803000~1d000] released 0x[] 2019-12-07T19:34:17.776+0000 7f8f04982700 10 freelist allocate 0x803000~1d000 2019-12-07T19:34:17.776+0000 7f8f04982700 20 freelist _xor first_key 0x800000 last_key 0x800000 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 prepare 2019-12-07T19:34:17.776+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ff0000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x47400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1ce88 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca3f0 space 0x558c54e1ce88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x47400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x47c00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x47c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x47c00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x48400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need {<0x0, 0x1000> : [0x48400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x48400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x48400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cf68 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca480 space 0x558c54e1cf68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x48400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x48c00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x48c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x48c00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x49400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need {<0x1000, 0x1000> : [0x49400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x49400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 finish 2019-12-07T19:34:17.776+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.776+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'24 active m=5 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.776+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x49400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1cf68 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca510 space 0x558c54e1cf68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 aio_wait 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x49400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0000 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x49c00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x49c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.776+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 io_done 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x49c00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4a400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4a400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need {<0x0, 0x1000> : [0x4a400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4a400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1177 < min_flush_size 524288 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2790 < min_flush_size 524288 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c000 had 1 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bbb80 had 1 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2813 < min_flush_size 524288 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2813 < min_flush_size 524288 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x127b67~afd to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x127b67 0x127b67~afd to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf7b67 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb67 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4a400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x664 and padding block with 0x99c 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1d048 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca5a0 space 0x558c54e1d048 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4a400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4ac00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4ac00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4b400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4b400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need 0x1400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need {<0x1000, 0x1000> : [0x4b400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4b400: 0x1400 reading 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x128664 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.776+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000166734s (0.000049699s flush + 0.000117035s kv commit) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.776+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0000> 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 kv_submitted 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0000 2019-12-07T19:34:17.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 finishing 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0000 onodes 0x558c5840c000 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6558 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb83f0 space 0x558c57bb6558 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6638 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb8480 space 0x558c57bb6638 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6718 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb8510 space 0x558c57bb6718 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb67f8 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb85a0 space 0x558c57bb67f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb68d8 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb8630 space 0x558c57bb68d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb69b8 in 0x558c53a59960) _finish_write discard buffer(0x558c57bb86c0 space 0x558c57bb69b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6a98 in 0x558c53a59960) _finish_write discard buffer(0x558c583caab0 space 0x558c57bb6a98 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6b78 in 0x558c53a59960) _finish_write discard buffer(0x558c583cab40 space 0x558c57bb6b78 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6c58 in 0x558c53a59960) _finish_write discard buffer(0x558c583cabd0 space 0x558c57bb6c58 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec088 in 0x558c53a59960) _finish_write discard buffer(0x558c57e65200 space 0x558c580ec088 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840fac8 in 0x558c53a59960) _finish_write discard buffer(0x558c583cac60 space 0x558c5840fac8 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840fba8 in 0x558c53a59960) _finish_write discard buffer(0x558c583cacf0 space 0x558c5840fba8 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840fc88 in 0x558c53a59960) _finish_write discard buffer(0x558c583cad80 space 0x558c5840fc88 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840fd68 in 0x558c53a59960) _finish_write discard buffer(0x558c583cae10 space 0x558c5840fd68 0x0~2000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840fe48 in 0x558c53a59960) _finish_write discard buffer(0x558c583caea0 space 0x558c5840fe48 0x0~1000 writing nocache) 2019-12-07T19:34:17.776+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'24 active m=5 mbc={} ps=[a~1]] recovery_committed_to version 29'25 now ondisk 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0000 done 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.776+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0000 [] 2019-12-07T19:34:17.776+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.776+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4b400:1400~400]} 2019-12-07T19:34:17.776+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:10779340:::smithi13140796-32:head)]) v3 -- 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1d048 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca630 space 0x558c54e1d048 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4b400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4bc00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4bc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) need 0x1c00~400 cache has 0x[1c00~400] 2019-12-07T19:34:17.776+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4bc00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4c400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4c400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 sbid 0x0)) need 0x400~400 cache has 0x[] 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 sbid 0x0)) need {<0x0, 0x1000> : [0x4c400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4c400: 0x400 reading 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4c400:400~400]} 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54e1d128 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:17.776+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c583ca6c0 space 0x558c54e1d128 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4c400~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4cc00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4cc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 sbid 0x0)) need 0xc00~400 cache has 0x[c00~400] 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4cc00~400 = 1024 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4d400~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4d400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4d400~400 = 0 2019-12-07T19:34:17.776+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4dc00~400 2019-12-07T19:34:17.776+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4dc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.776+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4dc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4e400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4e400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4e400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4ec00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4ec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4ec00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4f400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4f400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4f400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4fc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x4fc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4fc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x50400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x50400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x50c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x50c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x50c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x51400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x51400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x51c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x51c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x51c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x52400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x52400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x52c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x52c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x52c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x53400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x53400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x53c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x53c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x53c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x54400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x54400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x54c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x54c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x54c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x55400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x55400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x55c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x55c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x55c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x56400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x56400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x56c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x56c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x56c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x57400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x57400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x57c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x57c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x57c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x58400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x58400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x58c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x58c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x58c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x59400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x59400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x59c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x59c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x59c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5a400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5a400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5a400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5ac00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5ac00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5b400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5b400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5b400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5bc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5bc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5bc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5c400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5c400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5c400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5cc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5cc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5cc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5d400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5d400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5d400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5dc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5dc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5dc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5e400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5e400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5e400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5ec00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5ec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5ec00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5f400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5f400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5f400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5fc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x5fc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x5fc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x60400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x60400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x60c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x60c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x60c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x61400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x61400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x61c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x61c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x61c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x62400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x62400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x62c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x62c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x62c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x63400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x63400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x63c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x63c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x63c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x64400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x64400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x64c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x64c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x64c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x65400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x65400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x65c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x65c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x65c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x66400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x66400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x66c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x66c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x66c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x67400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x67400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x67c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x67c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x67c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x68400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x68400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x68c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x68c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x68c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x69400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x69400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x69c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x69c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x69c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6a400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6a400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6a400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6ac00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6ac00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6b400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6b400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6b400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6bc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6bc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6bc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6c400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6c400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6c400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6cc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6cc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6cc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6d400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6d400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6d400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6dc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6dc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6dc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6e400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6e400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6e400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6ec00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6ec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6ec00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6f400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6f400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6f400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6fc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x6fc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x6fc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x70400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x70400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x70c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x70c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x70c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x71400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x71400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x71c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x71c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x71c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x72400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x72400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x72c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x72c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x72c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x73400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x73400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x73c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x73c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x73c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x74400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x74400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x74c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x74c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x74c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x75400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x75400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x75c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x75c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x75c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x76400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x76400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x76c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x76c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x76c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x77400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x77400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x77c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x77c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x77c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x78400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x78400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x78c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x78c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x78c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x79400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x79400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x79c00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x79c00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x79c00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7a400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7a400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7a400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7ac00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ac00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7ac00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7b400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7b400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7b400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7bc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7bc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7bc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7c400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7c400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7c400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7cc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7cc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7cc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7d400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7d400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7d400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7dc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7dc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7dc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7e400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7e400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7e400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7ec00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7ec00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7ec00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7f400~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7f400~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7f400~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7fc00~400 2019-12-07T19:34:17.777+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x7fc00~400 size 0x4d000 (315392) 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x7fc00~400 = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 20 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=0 len=26624 2019-12-07T19:34:17.777+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:17.777+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:17.777+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:17.777+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key 2019-12-07T19:34:17.777+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key = 0 2019-12-07T19:34:17.777+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 40/39 ECSubReadReply(tid=195, attrs_read=0)) v2 -- 0x558c58092c80 con 0x558c54c64000 2019-12-07T19:34:17.777+0000 7f8f08189700 10 osd.7 40 dequeue_op 0x558c57f3e6e0 finish 2019-12-07T19:34:17.777+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.777+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 249 ==== MOSDPGRecoveryDelete(1.8s5 e40,39 1:15219530:::smithi13140796-21:4,29'26) v2 ==== 123+0+0 (crc 0 0 0) 0x558c54dfbe00 con 0x558c54c64000 2019-12-07T19:34:17.777+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740c2c0 prio 3 cost 1000 latency 0.000017 epoch 40 MOSDPGRecoveryDelete(1.8s5 e40,39 1:15219530:::smithi13140796-21:4,29'26) v2 2019-12-07T19:34:17.777+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:15219530:::smithi13140796-21:4,29'26) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:15219530:::smithi13140796-21:4,29'26) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:15219530:::smithi13140796-21:4,29'26) v2) prio 3 cost 1000 e40) pg 0x558c57380000 2019-12-07T19:34:17.777+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740c2c0 prio 3 cost 1000 latency 0.000044 MOSDPGRecoveryDelete(1.8s5 e40,39 1:15219530:::smithi13140796-21:4,29'26) v2 pg pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=5 mbc={} ps=[a~1]] 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=5 mbc={} ps=[a~1]] handle_recovery_delete 0x558c5740c2c0 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=5 mbc={} ps=[a~1]] remove_missing_object 1:15219530:::smithi13140796-21:4 29'26 2019-12-07T19:34:17.777+0000 7f8f04982700 20 snap_mapper.remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:17.777+0000 7f8f04982700 20 snap_mapper._remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:17.777+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.777+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.777+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.777+0000 7f8f04982700 20 snap_mapper.get_snaps 1:15219530:::smithi13140796-21:4 got.empty() 2019-12-07T19:34:17.777+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.777+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5730a600 seq 10 2019-12-07T19:34:17.777+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:15219530:::smithi13140796-21:4# key 0x858000000000000001152195'0!smithi13140796-21!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:17.777+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.777+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:15219530:::smithi13140796-21:4# 2019-12-07T19:34:17.777+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a600 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.777+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a600 onodes shared_blobs 2019-12-07T19:34:17.777+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a600 allocated 0x[] released 0x[] 2019-12-07T19:34:17.777+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 prepare 2019-12-07T19:34:17.777+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a600 2019-12-07T19:34:17.777+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 io_done 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.777+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740c2c0 finish 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x128664~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x128664 0x128664~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf8664 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x664 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x692 and padding block with 0x96e 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x128692 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000111359s (0.000000103s flush + 0.000111256s kv commit) 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a600> 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.777+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 kv_submitted 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a600 2019-12-07T19:34:17.777+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 finishing 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a600 onodes 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a600 done 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.777+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a600 [] 2019-12-07T19:34:17.777+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.777+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=5 mbc={} ps=[a~1]] on_local_recover: 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 snap_mapper.remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 snap_mapper._remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:17.777+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:15219530:::smithi13140796-21:4 got.empty() 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'25 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=5 mbc={} ps=[a~1]] got missing 1:15219530:::smithi13140796-21:4 v 29'26 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'25 active m=4 mbc={} ps=[a~1]] last_complete now 29'27 log.complete_to 29'28 2019-12-07T19:34:17.777+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c577fc300 seq 11 2019-12-07T19:34:17.777+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.8A489AC0.4.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.777+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc300 cost 5406 (1 ios * 4000 + 1406 bytes) 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc300 onodes shared_blobs 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 prepare 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc300 2019-12-07T19:34:17.777+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 io_done 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.777+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1182 < min_flush_size 524288 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'25 active m=4 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.777+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x128692~4b5 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x128692 0x128692~4b5 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf8692 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x692 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.777+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb47 and padding block with 0x4b9 2019-12-07T19:34:17.778+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x128b47 2019-12-07T19:34:17.778+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.778+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.778+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.778+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.778+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107534s (0.000000064s flush + 0.000107470s kv commit) 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.778+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc300> 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.778+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 kv_submitted 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc300 2019-12-07T19:34:17.778+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 finishing 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc300 onodes 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc300 done 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.778+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc300 [] 2019-12-07T19:34:17.778+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.778+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.778+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.778+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'25 active m=4 mbc={} ps=[a~1]] recovery_committed_to version 29'27 now ondisk 2019-12-07T19:34:17.778+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGRecoveryDeleteReply(1.8s0 e40,39 1:15219530:::smithi13140796-21:4,29'26) v2 -- 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.778+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.778+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 250 ==== MOSDPGRecoveryDelete(1.8s5 e40,39 1:131421e8:::smithi13140796-7:4,29'28) v2 ==== 122+0+0 (crc 0 0 0) 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.778+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740d760 prio 3 cost 1000 latency 0.000010 epoch 40 MOSDPGRecoveryDelete(1.8s5 e40,39 1:131421e8:::smithi13140796-7:4,29'28) v2 2019-12-07T19:34:17.778+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:131421e8:::smithi13140796-7:4,29'28) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.778+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.778+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:131421e8:::smithi13140796-7:4,29'28) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.778+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.778+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e40,39 1:131421e8:::smithi13140796-7:4,29'28) v2) prio 3 cost 1000 e40) pg 0x558c57380000 2019-12-07T19:34:17.778+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740d760 prio 3 cost 1000 latency 0.000029 MOSDPGRecoveryDelete(1.8s5 e40,39 1:131421e8:::smithi13140796-7:4,29'28) v2 pg pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=4 mbc={} ps=[a~1]] 2019-12-07T19:34:17.778+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=4 mbc={} ps=[a~1]] handle_recovery_delete 0x558c5740d760 2019-12-07T19:34:17.778+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=4 mbc={} ps=[a~1]] remove_missing_object 1:131421e8:::smithi13140796-7:4 29'28 2019-12-07T19:34:17.778+0000 7f8f04982700 20 snap_mapper.remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:17.778+0000 7f8f04982700 20 snap_mapper._remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:17.778+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.778+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.778+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.778+0000 7f8f04982700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:4 got.empty() 2019-12-07T19:34:17.778+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.778+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5730b800 seq 12 2019-12-07T19:34:17.778+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:131421e8:::smithi13140796-7:4# key 0x858000000000000001131421e821736d'ithi13140796-7!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:17.779+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.779+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:131421e8:::smithi13140796-7:4# 2019-12-07T19:34:17.779+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b800 cost 4211 (1 ios * 4000 + 211 bytes) 2019-12-07T19:34:17.779+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b800 onodes shared_blobs 2019-12-07T19:34:17.779+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b800 allocated 0x[] released 0x[] 2019-12-07T19:34:17.779+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 prepare 2019-12-07T19:34:17.779+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b800 2019-12-07T19:34:17.779+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 io_done 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.779+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740d760 finish 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.779+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x128b47~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x128b47 0x128b47~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf8b47 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb47 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb75 and padding block with 0x48b 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x128b75 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000108821s (0.000000072s flush + 0.000108749s kv commit) 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b800> 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 kv_submitted 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b800 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 finishing 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b800 onodes 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 done 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b800 [] 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.779+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=4 mbc={} ps=[a~1]] on_local_recover: 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 snap_mapper.remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 snap_mapper._remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:17.779+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:4 got.empty() 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'27 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=4 mbc={} ps=[a~1]] got missing 1:131421e8:::smithi13140796-7:4 v 29'28 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'28 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'27 active m=3 mbc={} ps=[a~1]] last_complete now 29'28 log.complete_to 29'29 2019-12-07T19:34:17.779+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c577fcc00 seq 13 2019-12-07T19:34:17.779+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.8C824871.4.smithi13140796-7..' <- missing/0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.779+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcc00 cost 5405 (1 ios * 4000 + 1405 bytes) 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcc00 onodes shared_blobs 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcc00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 prepare 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcc00 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 io_done 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.779+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'28 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'27 active m=3 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1181 < min_flush_size 524288 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1204 < min_flush_size 524288 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1204 < min_flush_size 524288 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x128b75~4b4 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x128b75 0x128b75~4b4 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf8b75 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb75 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x29 and padding block with 0xfd7 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x129029 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.779+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000115989s (0.000000050s flush + 0.000115939s kv commit) 2019-12-07T19:34:17.779+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcc00> 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 kv_submitted 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcc00 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 finishing 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcc00 onodes 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcc00 done 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.779+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcc00 [] 2019-12-07T19:34:17.779+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.779+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.779+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'28 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'27 active m=3 mbc={} ps=[a~1]] recovery_committed_to version 29'28 now ondisk 2019-12-07T19:34:17.779+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGRecoveryDeleteReply(1.8s0 e40,39 1:131421e8:::smithi13140796-7:4,29'28) v2 -- 0x558c54dfbe00 con 0x558c54c64000 2019-12-07T19:34:17.779+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.780+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 251 ==== MRecoveryReserve(1.9s4 RELEASE e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c576dd4a0 con 0x558c54c64000 2019-12-07T19:34:17.780+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.9s4 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.780+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.780+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.780+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.780+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.780+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c5500a000 2019-12-07T19:34:17.780+0000 7f8f04181700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.780+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.781610+0000 1 0.000022 2019-12-07T19:34:17.780+0000 7f8f04181700 5 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.780+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:17.780+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:17.781+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 141 ==== MRecoveryReserve(1.fs5 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c581e8b40 con 0x558c54c5bc00 2019-12-07T19:34:17.781+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.fs5 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.781+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.781+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.781+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.781+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.781+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c57383000 2019-12-07T19:34:17.781+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.781+0000 7f8f05183700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.782763+0000 3 0.000224 2019-12-07T19:34:17.781+0000 7f8f05183700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.781+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:17.781+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.781+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.fs5 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.781+0000 7f8f0b990700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.781+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.781+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.781+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.781+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c57383000 2019-12-07T19:34:17.781+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.781+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.fs0 GRANT e40/40) v3 -- 0x558c57edfc20 con 0x558c54c5bc00 2019-12-07T19:34:17.781+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.782834+0000 1 0.000028 2019-12-07T19:34:17.781+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.781+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:17.781+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.784+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 142 ==== MOSDPGPush(1.fs5 40/39 [PushOp(1:f9bf9ba5:::smithi13140796-26:head, version: 21'4, data_included: [0~106496], data_size: 106496, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f9bf9ba5:::smithi13140796-26:head@21'4, size: 425984, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:425984, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 107558+0+0 (unknown 0 0 0) 0x558c5763cc00 con 0x558c54c5bc00 2019-12-07T19:34:17.784+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c57335760 prio 127 cost 107496 latency 0.000027 epoch 40 MOSDPGPush(1.fs5 40/39 [PushOp(1:f9bf9ba5:::smithi13140796-26:head, version: 21'4, data_included: [0~106496], data_size: 106496, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f9bf9ba5:::smithi13140796-26:head@21'4, size: 425984, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:425984, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.784+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f9bf9ba5:::smithi13140796-26:head, version: 21'4, data_included: [0~106496], data_size: 106496, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f9bf9ba5:::smithi13140796-26:head@21'4, size: 425984, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:425984, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 107496 e40) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.784+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f9bf9ba5:::smithi13140796-26:head, version: 21'4, data_included: [0~106496], data_size: 106496, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f9bf9ba5:::smithi13140796-26:head@21'4, size: 425984, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:425984, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 107496 e40) queued 2019-12-07T19:34:17.784+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.784+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f9bf9ba5:::smithi13140796-26:head, version: 21'4, data_included: [0~106496], data_size: 106496, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f9bf9ba5:::smithi13140796-26:head@21'4, size: 425984, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:425984, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 107496 e40) pg 0x558c57383000 2019-12-07T19:34:17.784+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c57335760 prio 127 cost 107496 latency 0.000078 MOSDPGPush(1.fs5 40/39 [PushOp(1:f9bf9ba5:::smithi13140796-26:head, version: 21'4, data_included: [0~106496], data_size: 106496, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f9bf9ba5:::smithi13140796-26:head@21'4, size: 425984, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:425984, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] 2019-12-07T19:34:17.784+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.fs5 40/39 [PushOp(1:f9bf9ba5:::smithi13140796-26:head, version: 21'4, data_included: [0~106496], data_size: 106496, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f9bf9ba5:::smithi13140796-26:head@21'4, size: 425984, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:425984, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.784+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] on_local_recover: 1:f9bf9ba5:::smithi13140796-26:head 2019-12-07T19:34:17.784+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'3 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=4 mbc={} ps=[a~1]] got missing 1:f9bf9ba5:::smithi13140796-26:head v 21'4 2019-12-07T19:34:17.784+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'5 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=3 mbc={} ps=[a~1]] last_complete now 21'5 log.complete_to 21'6 2019-12-07T19:34:17.784+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.784+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57ff0300 seq 4 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:f9bf9ba5:::smithi13140796-26:head# key 0x858000000000000001f9bf9ba521736d'ithi13140796-26!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.784+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:f9bf9ba5:::smithi13140796-26:head# key 0x858000000000000001f9bf9ba521736d'ithi13140796-26!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.784+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1198 2019-12-07T19:34:17.784+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# = 0 2019-12-07T19:34:17.784+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.784+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1a000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ff0300 13 blobs 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.784+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1a000/1000,1a000,0 2019-12-07T19:34:17.784+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x820000~1a000/1000,1a000,0 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x820000~1a000] 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c422a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c422a0 blob([0x820000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42af0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57c422a0) get_ref 0x0~2000 Blob(0x558c57c422a0 blob([0x820000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42af0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57c422a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57c422a0 blob([0x820000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42af0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57c42b08 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575667e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d260 blob([0x822000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575667e0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d260) get_ref 0x0~2000 Blob(0x558c54d3d260 blob([0x822000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575667e0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d260) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c54d3d260 blob([0x822000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575667e0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c575667f8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c552797a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c552797a0 blob([0x824000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c552797a0) get_ref 0x0~2000 Blob(0x558c552797a0 blob([0x824000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c552797a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c552797a0 blob([0x824000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279730 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c55279748 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897030 blob([0x826000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896fc0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897030) get_ref 0x0~2000 Blob(0x558c57897030 blob([0x826000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896fc0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897030) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57897030 blob([0x826000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896fc0 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57896fd8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896e70 blob([0x828000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896e00 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57896e70) get_ref 0x0~2000 Blob(0x558c57896e70 blob([0x828000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896e00 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57896e70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57896e70 blob([0x828000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896e00 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57896e18 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57896ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57896ee0 blob([0x82a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896f50 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57896ee0) get_ref 0x0~2000 Blob(0x558c57896ee0 blob([0x82a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896f50 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57896ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57896ee0 blob([0x82a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896f50 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57896f68 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897810 blob([0x82c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897880 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897810) get_ref 0x0~2000 Blob(0x558c57897810 blob([0x82c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897880 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57897810 blob([0x82c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897880 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57897898 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578978f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578978f0 blob([0x82e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897960 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c578978f0) get_ref 0x0~2000 Blob(0x558c578978f0 blob([0x82e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897960 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c578978f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c578978f0 blob([0x82e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897960 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57897978 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578979d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 252 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:131421e8:::smithi13140796-7:head, version: 29'29, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@29'29, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 128034+0+0 (crc 0 0 0) 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578979d0 blob([0x830000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897a40 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c578979d0) get_ref 0x0~2000 Blob(0x558c578979d0 blob([0x830000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897a40 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c578979d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740db80 prio 127 cost 127976 latency 0.000022 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:131421e8:::smithi13140796-7:head, version: 29'29, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@29'29, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c578979d0 blob([0x830000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897a40 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57897a58 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:131421e8:::smithi13140796-7:head, version: 29'29, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@29'29, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 127976 e40) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897ab0 blob([0x832000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897b20 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897ab0) get_ref 0x0~2000 Blob(0x558c57897ab0 blob([0x832000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897b20 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57897ab0 blob([0x832000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897b20 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57897b38 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.784+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897b90 blob([0x834000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c00 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:131421e8:::smithi13140796-7:head, version: 29'29, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@29'29, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 127976 e40) queued 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897b90) get_ref 0x0~2000 Blob(0x558c57897b90 blob([0x834000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c00 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore.blob(0x558c57897b90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.784+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57897b90 blob([0x834000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c00 sbid 0x0)) 2019-12-07T19:34:17.784+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57897c18 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:131421e8:::smithi13140796-7:head, version: 29'29, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@29'29, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 127976 e40) pg 0x558c57380000 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eae00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eae70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eae00 blob([0x836000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eae70 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.blob(0x558c583eae00) get_ref 0x0~2000 Blob(0x558c583eae00 blob([0x836000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eae70 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.blob(0x558c583eae00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c583eae00 blob([0x836000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eae70 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c583eae88 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740db80 prio 127 cost 127976 latency 0.000063 MOSDPGPush(1.8s5 40/39 [PushOp(1:131421e8:::smithi13140796-7:head, version: 29'29, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@29'29, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 29'28 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=3 mbc={} ps=[a~1]] 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eaee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eaf50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eaee0 blob([0x838000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaf50 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.blob(0x558c583eaee0) get_ref 0x0~2000 Blob(0x558c583eaee0 blob([0x838000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaf50 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.blob(0x558c583eaee0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'28 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=3 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:131421e8:::smithi13140796-7:head, version: 29'29, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@29'29, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c583eaee0 blob([0x838000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaf50 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c583eaf68 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1a000 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.extentmap(0x558c57e8f508) dirty_range mark inline shard dirty 2019-12-07T19:34:17.785+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'28 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=3 mbc={} ps=[a~1]] on_local_recover: 1:131421e8:::smithi13140796-7:head 2019-12-07T19:34:17.785+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 = 0 2019-12-07T19:34:17.785+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 29'28 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=3 mbc={} ps=[a~1]] got missing 1:131421e8:::smithi13140796-7:head v 29'29 2019-12-07T19:34:17.785+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 4 keys 2019-12-07T19:34:17.785+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'28 active m=2 mbc={} ps=[a~1]] last_complete now 31'31 log.complete_to 31'32 2019-12-07T19:34:17.785+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 4 keys = 0 2019-12-07T19:34:17.785+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.785+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049c'.missing/0000000000000001.F9DF9D5A.head.smithi13140796-26..' <- missing/0000000000000001.F9DF9D5A.head.smithi13140796-26.. 2019-12-07T19:34:17.785+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.785+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5730b200 seq 14 2019-12-07T19:34:17.785+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.785+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0300 cost 164733 (14 ios * 4000 + 108733 bytes) 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0300 onodes 0x558c57e8f400 shared_blobs 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.extentmap(0x558c57e8f508) update 5#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:131421e8:::smithi13140796-7:head# key 0x858000000000000001131421e821736d'ithi13140796-7!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore.extentmap(0x558c57e8f508) update inline shard 250 bytes from 13 extents 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:f9bf9ba5:::smithi13140796-26:head# is 741 (485 bytes onode + 2 bytes spanning blobs + 254 bytes inline extents) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:17.785+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0300 allocated 0x[820000~1a000] released 0x[] 2019-12-07T19:34:17.785+0000 7f8f05183700 10 freelist allocate 0x820000~1a000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:131421e8:::smithi13140796-7:head# key 0x858000000000000001131421e821736d'ithi13140796-7!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.785+0000 7f8f05183700 20 freelist _xor first_key 0x800000 last_key 0x800000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.785+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1199 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# = 0 2019-12-07T19:34:17.785+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~1f000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:131421e8:::smithi13140796-7:head# 0x0~1f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.785+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 prepare 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.785+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ff0300 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730b200 16 blobs 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.785+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1f000/1000,1f000,0 2019-12-07T19:34:17.785+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x83a000~1f000/1000,1f000,0 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x83a000~1f000] 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c388c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c388c0 blob([0x83a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2000 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c57c388c0) get_ref 0x0~2000 Blob(0x558c57c388c0 blob([0x83a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2000 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c57c388c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57c388c0 blob([0x83a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c577f2018 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ea540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ea690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ea540 blob([0x83c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea690 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea540) get_ref 0x0~2000 Blob(0x558c583ea540 blob([0x83c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea690 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea540) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c583ea540 blob([0x83c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ea6a8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ea700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ea770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ea700 blob([0x83e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea770 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea700) get_ref 0x0~2000 Blob(0x558c583ea700 blob([0x83e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea770 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea700) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c583ea700 blob([0x83e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ea788 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ea7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ea850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ea7e0 blob([0x840000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea850 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea7e0) get_ref 0x0~2000 Blob(0x558c583ea7e0 blob([0x840000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea850 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c583ea7e0 blob([0x840000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ea868 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ea8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ea930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ea8c0 blob([0x842000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea930 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea8c0) get_ref 0x0~2000 Blob(0x558c583ea8c0 blob([0x842000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea930 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c583ea8c0 blob([0x842000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ea948 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ea9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eaa10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ea9a0 blob([0x844000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaa10 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea9a0) get_ref 0x0~2000 Blob(0x558c583ea9a0 blob([0x844000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaa10 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ea9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c583ea9a0 blob([0x844000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583eaa28 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eaa80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eaaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eaa80 blob([0x846000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaaf0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eaa80) get_ref 0x0~2000 Blob(0x558c583eaa80 blob([0x846000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaaf0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eaa80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c583eaa80 blob([0x846000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583eab08 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eab60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eabd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eab60 blob([0x848000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eabd0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eab60) get_ref 0x0~2000 Blob(0x558c583eab60 blob([0x848000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eabd0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eab60) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c583eab60 blob([0x848000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583eabe8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eac40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eacb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eac40 blob([0x84a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eacb0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eac40) get_ref 0x0~2000 Blob(0x558c583eac40 blob([0x84a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eacb0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eac40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c583eac40 blob([0x84a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583eacc8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ead20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ead90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ead20 blob([0x84c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead90 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ead20) get_ref 0x0~2000 Blob(0x558c583ead20 blob([0x84c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead90 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ead20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c583ead20 blob([0x84c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583eada8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eb960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eb9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eb960 blob([0x84e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb9d0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eb960) get_ref 0x0~2000 Blob(0x558c583eb960 blob([0x84e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb9d0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eb960) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c583eb960 blob([0x84e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583eb9e8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ebab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eba40 blob([0x850000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebab0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eba40) get_ref 0x0~2000 Blob(0x558c583eba40 blob([0x850000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebab0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583eba40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c583eba40 blob([0x850000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ebac8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ebb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ebb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ebb20 blob([0x852000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebb90 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebb20) get_ref 0x0~2000 Blob(0x558c583ebb20 blob([0x852000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebb90 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebb20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c583ebb20 blob([0x852000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ebba8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ebc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ebc00 blob([0x854000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc70 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebc00) get_ref 0x0~2000 Blob(0x558c583ebc00 blob([0x854000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc70 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebc00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c583ebc00 blob([0x854000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ebc88 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ebce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ebd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ebce0 blob([0x856000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebd50 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebce0) get_ref 0x0~2000 Blob(0x558c583ebce0 blob([0x856000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebd50 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c583ebce0 blob([0x856000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ebd68 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ebdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ebe30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ebdc0 blob([0x858000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebe30 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebdc0) get_ref 0x0~1000 Blob(0x558c583ebdc0 blob([0x858000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebe30 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.blob(0x558c583ebdc0) get_ref init 0x1000, 1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~1000: 0x0~1000 Blob(0x558c583ebdc0 blob([0x858000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 sbid 0x0)) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ebe48 in 0x558c53a59960) _discard 0x0~1000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1f000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~1f000 = 0 2019-12-07T19:34:17.785+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 4 keys 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 4 keys = 0 2019-12-07T19:34:17.785+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.8C824871.head.smithi13140796-7..' <- missing/0000000000000001.8C824871.head.smithi13140796-7.. 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.785+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 197210 (17 ios * 4000 + 129210 bytes) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes 0x558c5840cc80 shared_blobs 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) update inline shard 304 bytes from 16 extents 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 794 (484 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:34:17.785+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[83a000~1f000] released 0x[] 2019-12-07T19:34:17.785+0000 7f8f04982700 10 freelist allocate 0x83a000~1f000 2019-12-07T19:34:17.785+0000 7f8f04982700 20 freelist _xor first_key 0x800000 last_key 0x800000 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:17.785+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730b200 2019-12-07T19:34:17.785+0000 7f8f05183700 10 osd.7 40 dequeue_op 0x558c57335760 finish 2019-12-07T19:34:17.785+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.785+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'5 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=3 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.785+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.785+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 aio_wait 2019-12-07T19:34:17.785+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0300 2019-12-07T19:34:17.785+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 io_done 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2064 < min_flush_size 524288 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f400 had 1 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2087 < min_flush_size 524288 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2087 < min_flush_size 524288 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x129029~827 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x129029 0x129029~827 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf9029 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x29 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x850 and padding block with 0x7b0 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x129850 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000188065s (0.000041132s flush + 0.000146933s kv commit) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0300> 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.785+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 kv_submitted 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0300 2019-12-07T19:34:17.785+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 finishing 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0300 onodes 0x558c57e8f400 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279748 in 0x558c54862000) _finish_write discard buffer(0x558c583cb320 space 0x558c55279748 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575667f8 in 0x558c54862000) _finish_write discard buffer(0x558c583cb290 space 0x558c575667f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896e18 in 0x558c54862000) _finish_write discard buffer(0x558c583cb440 space 0x558c57896e18 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896f68 in 0x558c54862000) _finish_write discard buffer(0x558c583cb4d0 space 0x558c57896f68 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896fd8 in 0x558c54862000) _finish_write discard buffer(0x558c583cb3b0 space 0x558c57896fd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897898 in 0x558c54862000) _finish_write discard buffer(0x558c583cb560 space 0x558c57897898 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897978 in 0x558c54862000) _finish_write discard buffer(0x558c583cb5f0 space 0x558c57897978 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897a58 in 0x558c54862000) _finish_write discard buffer(0x558c57a08ab0 space 0x558c57897a58 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897b38 in 0x558c54862000) _finish_write discard buffer(0x558c57f76870 space 0x558c57897b38 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897c18 in 0x558c54862000) _finish_write discard buffer(0x558c57f767e0 space 0x558c57897c18 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42b08 in 0x558c54862000) _finish_write discard buffer(0x558c576695f0 space 0x558c57c42b08 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eae88 in 0x558c54862000) _finish_write discard buffer(0x558c57f76750 space 0x558c583eae88 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eaf68 in 0x558c54862000) _finish_write discard buffer(0x558c57f766c0 space 0x558c583eaf68 0x0~2000 writing nocache) 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0300 done 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.785+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0300 [] 2019-12-07T19:34:17.785+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.785+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.785+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740db80 finish 2019-12-07T19:34:17.785+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'5 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 0'0 active m=3 mbc={} ps=[a~1]] recovery_committed_to version 21'5 now ondisk 2019-12-07T19:34:17.785+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPushReply(1.fs0 40/39 [PushReplyOp(1:f9bf9ba5:::smithi13140796-26:head)]) v3 -- 0x558c5763c800 con 0x558c54c5bc00 2019-12-07T19:34:17.785+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.785+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.785+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'28 active m=2 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.785+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.785+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 aio_wait 2019-12-07T19:34:17.785+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:17.785+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2115 < min_flush_size 524288 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840cc80 had 1 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2138 < min_flush_size 524288 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2138 < min_flush_size 524288 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x129850~85a to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x129850 0x129850~85a to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf9850 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x850 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.785+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xaa and padding block with 0xf56 2019-12-07T19:34:17.786+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12a0aa 2019-12-07T19:34:17.786+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.786+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.786+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.786+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.786+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000680231s (0.000031351s flush + 0.000648880s kv commit) 2019-12-07T19:34:17.786+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b200> 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.786+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:17.786+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 0x558c5840cc80 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f2018 in 0x558c53a59960) _finish_write discard buffer(0x558c583cafc0 space 0x558c577f2018 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ea6a8 in 0x558c53a59960) _finish_write discard buffer(0x558c583cb050 space 0x558c583ea6a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ea788 in 0x558c53a59960) _finish_write discard buffer(0x558c583cb0e0 space 0x558c583ea788 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ea868 in 0x558c53a59960) _finish_write discard buffer(0x558c583cb170 space 0x558c583ea868 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ea948 in 0x558c53a59960) _finish_write discard buffer(0x558c583cb200 space 0x558c583ea948 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eaa28 in 0x558c53a59960) _finish_write discard buffer(0x558c583cbe60 space 0x558c583eaa28 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eab08 in 0x558c53a59960) _finish_write discard buffer(0x558c583cbef0 space 0x558c583eab08 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eabe8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4000 space 0x558c583eabe8 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eacc8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4090 space 0x558c583eacc8 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eada8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4120 space 0x558c583eada8 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eb9e8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da41b0 space 0x558c583eb9e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ebac8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4240 space 0x558c583ebac8 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ebba8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da42d0 space 0x558c583ebba8 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ebc88 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4360 space 0x558c583ebc88 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ebd68 in 0x558c53a59960) _finish_write discard buffer(0x558c57da43f0 space 0x558c583ebd68 0x0~2000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ebe48 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4480 space 0x558c583ebe48 0x0~1000 writing nocache) 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.786+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [] 2019-12-07T19:34:17.786+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.786+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 29'28 active m=2 mbc={} ps=[a~1]] recovery_committed_to version 31'31 now ondisk 2019-12-07T19:34:17.786+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.786+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:131421e8:::smithi13140796-7:head)]) v3 -- 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.786+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.789+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 143 ==== MOSDPGPush(1.fs5 40/39 [PushOp(1:fc3a86a2:::smithi13140796-33:head, version: 21'6, data_included: [0~135168], data_size: 135168, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:fc3a86a2:::smithi13140796-33:head@21'6, size: 540672, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:540672, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 136230+0+0 (unknown 0 0 0) 0x558c5763c800 con 0x558c54c5bc00 2019-12-07T19:34:17.789+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c57f3e000 prio 127 cost 136168 latency 0.000037 epoch 40 MOSDPGPush(1.fs5 40/39 [PushOp(1:fc3a86a2:::smithi13140796-33:head, version: 21'6, data_included: [0~135168], data_size: 135168, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:fc3a86a2:::smithi13140796-33:head@21'6, size: 540672, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:540672, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.789+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:fc3a86a2:::smithi13140796-33:head, version: 21'6, data_included: [0~135168], data_size: 135168, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:fc3a86a2:::smithi13140796-33:head@21'6, size: 540672, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:540672, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 136168 e40) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:fc3a86a2:::smithi13140796-33:head, version: 21'6, data_included: [0~135168], data_size: 135168, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:fc3a86a2:::smithi13140796-33:head@21'6, size: 540672, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:540672, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 136168 e40) queued 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:fc3a86a2:::smithi13140796-33:head, version: 21'6, data_included: [0~135168], data_size: 135168, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:fc3a86a2:::smithi13140796-33:head@21'6, size: 540672, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:540672, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 136168 e40) pg 0x558c57383000 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3e000 prio 127 cost 136168 latency 0.000096 MOSDPGPush(1.fs5 40/39 [PushOp(1:fc3a86a2:::smithi13140796-33:head, version: 21'6, data_included: [0~135168], data_size: 135168, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:fc3a86a2:::smithi13140796-33:head@21'6, size: 540672, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:540672, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.fs5( v 29'12 lc 21'5 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=3 mbc={} ps=[a~1]] 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'5 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=3 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.fs5 40/39 [PushOp(1:fc3a86a2:::smithi13140796-33:head, version: 21'6, data_included: [0~135168], data_size: 135168, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:fc3a86a2:::smithi13140796-33:head@21'6, size: 540672, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:540672, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'5 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=3 mbc={} ps=[a~1]] on_local_recover: 1:fc3a86a2:::smithi13140796-33:head 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 21'5 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=3 mbc={} ps=[a~1]] got missing 1:fc3a86a2:::smithi13140796-33:head v 21'6 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 21'5 active m=2 mbc={} ps=[a~1]] last_complete now 27'10 log.complete_to 29'11 2019-12-07T19:34:17.789+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57b94900 seq 5 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:fc3a86a2:::smithi13140796-33:head# key 0x858000000000000001fc3a86a221736d'ithi13140796-33!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:fc3a86a2:::smithi13140796-33:head# key 0x858000000000000001fc3a86a221736d'ithi13140796-33!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.789+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1200 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# = 0 2019-12-07T19:34:17.789+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~21000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57b94900 17 blobs 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x21000/1000,21000,0 2019-12-07T19:34:17.789+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x859000~21000/1000,21000,0 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x859000~21000] 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c429a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42c40 blob([0x859000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c429a0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42c40) get_ref 0x0~2000 Blob(0x558c57c42c40 blob([0x859000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c429a0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42c40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57c42c40 blob([0x859000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c429a0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c429b8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42b60 blob([0x85b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43ea0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42b60) get_ref 0x0~2000 Blob(0x558c57c42b60 blob([0x85b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43ea0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42b60) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c57c42b60 blob([0x85b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43ea0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43eb8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897f10 blob([0x85d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897f80 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57897f10) get_ref 0x0~2000 Blob(0x558c57897f10 blob([0x85d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897f80 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57897f10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57897f10 blob([0x85d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897f80 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57897f98 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b977a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97730 blob([0x85f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b977a0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97730) get_ref 0x0~2000 Blob(0x558c57b97730 blob([0x85f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b977a0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97730) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57b97730 blob([0x85f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b977a0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b977b8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b978f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97880 blob([0x861000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b978f0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97880) get_ref 0x0~2000 Blob(0x558c57b97880 blob([0x861000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b978f0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97880) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c57b97880 blob([0x861000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b978f0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b97908 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b979d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97960 blob([0x863000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b979d0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97960) get_ref 0x0~2000 Blob(0x558c57b97960 blob([0x863000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b979d0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97960) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c57b97960 blob([0x863000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b979d0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b979e8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ce00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ce00 blob([0x865000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce70 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750ce00) get_ref 0x0~2000 Blob(0x558c5750ce00 blob([0x865000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce70 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750ce00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5750ce00 blob([0x865000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ce70 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750ce88 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750cee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750cf50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750cee0 blob([0x867000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cf50 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750cee0) get_ref 0x0~2000 Blob(0x558c5750cee0 blob([0x867000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cf50 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750cee0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5750cee0 blob([0x867000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cf50 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750cf68 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750cfc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750cfc0 blob([0x869000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d030 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750cfc0) get_ref 0x0~2000 Blob(0x558c5750cfc0 blob([0x869000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d030 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750cfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5750cfc0 blob([0x869000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d030 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d048 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d0a0 blob([0x86b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d110 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d0a0) get_ref 0x0~2000 Blob(0x558c5750d0a0 blob([0x86b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d110 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c5750d0a0 blob([0x86b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d110 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d128 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d180 blob([0x86d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d1f0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d180) get_ref 0x0~2000 Blob(0x558c5750d180 blob([0x86d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d1f0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d180) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c5750d180 blob([0x86d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d1f0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d208 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d260 blob([0x86f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d2d0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d260) get_ref 0x0~2000 Blob(0x558c5750d260 blob([0x86f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d2d0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d260) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c5750d260 blob([0x86f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d2d0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d2e8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d340 blob([0x871000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d3b0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d340) get_ref 0x0~2000 Blob(0x558c5750d340 blob([0x871000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d3b0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d340) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c5750d340 blob([0x871000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d3b0 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d3c8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d420 blob([0x873000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d490 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d420) get_ref 0x0~2000 Blob(0x558c5750d420 blob([0x873000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d490 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d420) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c5750d420 blob([0x873000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d490 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d4a8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d500 blob([0x875000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d570 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d500) get_ref 0x0~2000 Blob(0x558c5750d500 blob([0x875000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d570 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d500) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c5750d500 blob([0x875000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d570 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d588 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d5e0 blob([0x877000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d650 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d5e0) get_ref 0x0~2000 Blob(0x558c5750d5e0 blob([0x877000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d650 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c5750d5e0 blob([0x877000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d650 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d668 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d6c0 blob([0x879000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d730 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d6c0) get_ref 0x0~1000 Blob(0x558c5750d6c0 blob([0x879000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d730 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.blob(0x558c5750d6c0) get_ref init 0x1000, 1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~1000: 0x0~1000 Blob(0x558c5750d6c0 blob([0x879000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5750d730 sbid 0x0)) 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750d748 in 0x558c54862000) _discard 0x0~1000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x21000 2019-12-07T19:34:17.789+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dirty_range mark inline shard dirty 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 = 0 2019-12-07T19:34:17.790+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 4 keys 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 4 keys = 0 2019-12-07T19:34:17.790+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049c'.missing/0000000000000001.F3C51654.head.smithi13140796-33..' <- missing/0000000000000001.F3C51654.head.smithi13140796-33.. 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.790+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94900 cost 209405 (18 ios * 4000 + 137405 bytes) 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94900 onodes 0x558c57f35180 shared_blobs 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) update 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) update inline shard 323 bytes from 17 extents 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:head# is 814 (485 bytes onode + 2 bytes spanning blobs + 327 bytes inline extents) 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94900 allocated 0x[859000~21000] released 0x[] 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 freelist allocate 0x859000~21000 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 freelist _xor first_key 0x800000 last_key 0x800000 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 prepare 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57b94900 2019-12-07T19:34:17.790+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3e000 finish 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.790+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 21'5 active m=2 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.790+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.790+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 aio_wait 2019-12-07T19:34:17.790+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94900 2019-12-07T19:34:17.790+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 io_done 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2137 < min_flush_size 524288 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35180 had 1 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2160 < min_flush_size 524288 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2160 < min_flush_size 524288 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12a0aa~870 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12a0aa 0x12a0aa~870 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xfa0aa 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xaa 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x91a and padding block with 0x6e6 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12a91a 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.790+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000141075s (0.000036605s flush + 0.000104470s kv commit) 2019-12-07T19:34:17.790+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94900> 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.790+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 kv_submitted 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94900 2019-12-07T19:34:17.790+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 finishing 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94900 onodes 0x558c57f35180 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ce88 in 0x558c54862000) _finish_write discard buffer(0x558c57ccdb90 space 0x558c5750ce88 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750cf68 in 0x558c54862000) _finish_write discard buffer(0x558c57da4900 space 0x558c5750cf68 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d048 in 0x558c54862000) _finish_write discard buffer(0x558c57da4990 space 0x558c5750d048 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d128 in 0x558c54862000) _finish_write discard buffer(0x558c57da4a20 space 0x558c5750d128 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d208 in 0x558c54862000) _finish_write discard buffer(0x558c57da4ab0 space 0x558c5750d208 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d2e8 in 0x558c54862000) _finish_write discard buffer(0x558c57da4b40 space 0x558c5750d2e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d3c8 in 0x558c54862000) _finish_write discard buffer(0x558c57da4bd0 space 0x558c5750d3c8 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d4a8 in 0x558c54862000) _finish_write discard buffer(0x558c57da4c60 space 0x558c5750d4a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d588 in 0x558c54862000) _finish_write discard buffer(0x558c57da4cf0 space 0x558c5750d588 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d668 in 0x558c54862000) _finish_write discard buffer(0x558c57da4d80 space 0x558c5750d668 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d748 in 0x558c54862000) _finish_write discard buffer(0x558c57da4e10 space 0x558c5750d748 0x0~1000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897f98 in 0x558c54862000) _finish_write discard buffer(0x558c57ccd950 space 0x558c57897f98 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b977b8 in 0x558c54862000) _finish_write discard buffer(0x558c57ccd9e0 space 0x558c57b977b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97908 in 0x558c54862000) _finish_write discard buffer(0x558c57ccda70 space 0x558c57b97908 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b979e8 in 0x558c54862000) _finish_write discard buffer(0x558c57ccdb00 space 0x558c57b979e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c429b8 in 0x558c54862000) _finish_write discard buffer(0x558c57ccd830 space 0x558c57c429b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43eb8 in 0x558c54862000) _finish_write discard buffer(0x558c57ccd8c0 space 0x558c57c43eb8 0x0~2000 writing nocache) 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94900 done 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.790+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94900 [] 2019-12-07T19:34:17.790+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.790+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.790+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.790+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 21'5 active m=2 mbc={} ps=[a~1]] recovery_committed_to version 27'10 now ondisk 2019-12-07T19:34:17.790+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPushReply(1.fs0 40/39 [PushReplyOp(1:fc3a86a2:::smithi13140796-33:head)]) v3 -- 0x558c54c1ee00 con 0x558c54c5bc00 2019-12-07T19:34:17.790+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.791+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 144 ==== MOSDPGRecoveryDelete(1.fs5 e40,39 1:f7919c75:::smithi13140796-20:4,29'11) v2 ==== 123+0+0 (unknown 0 0 0) 0x558c54c1ee00 con 0x558c54c5bc00 2019-12-07T19:34:17.791+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c57f3e9a0 prio 3 cost 1000 latency 0.000019 epoch 40 MOSDPGRecoveryDelete(1.fs5 e40,39 1:f7919c75:::smithi13140796-20:4,29'11) v2 2019-12-07T19:34:17.791+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGRecoveryDelete(1.fs5 e40,39 1:f7919c75:::smithi13140796-20:4,29'11) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGRecoveryDelete(1.fs5 e40,39 1:f7919c75:::smithi13140796-20:4,29'11) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGRecoveryDelete(1.fs5 e40,39 1:f7919c75:::smithi13140796-20:4,29'11) v2) prio 3 cost 1000 e40) pg 0x558c57383000 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3e9a0 prio 3 cost 1000 latency 0.000051 MOSDPGRecoveryDelete(1.fs5 e40,39 1:f7919c75:::smithi13140796-20:4,29'11) v2 pg pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=2 mbc={} ps=[a~1]] handle_recovery_delete 0x558c57f3e9a0 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=2 mbc={} ps=[a~1]] remove_missing_object 1:f7919c75:::smithi13140796-20:4 29'11 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 snap_mapper.remove_oid 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 snap_mapper._remove_oid 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:17.791+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:f7919c75:::smithi13140796-20:4 got.empty() 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57b94c00 seq 6 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:f7919c75:::smithi13140796-20:4# key 0x858000000000000001f7919c'u!smithi13140796-20!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:f7919c75:::smithi13140796-20:4# 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94c00 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94c00 onodes shared_blobs 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94c00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 prepare 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94c00 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 io_done 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.791+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3e9a0 finish 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12a91a~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12a91a 0x12a91a~2e to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xfa91a 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x91a 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x948 and padding block with 0x6b8 2019-12-07T19:34:17.791+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12a948 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.791+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124738s (0.000000120s flush + 0.000124618s kv commit) 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.791+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94c00> 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.791+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 kv_submitted 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94c00 2019-12-07T19:34:17.791+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 finishing 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94c00 onodes 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94c00 done 2019-12-07T19:34:17.791+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.791+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94c00 [] 2019-12-07T19:34:17.791+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.791+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.791+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=2 mbc={} ps=[a~1]] on_local_recover: 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:17.791+0000 7f8f05183700 20 snap_mapper.remove_oid 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:17.791+0000 7f8f05183700 20 snap_mapper._remove_oid 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:17.791+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.791+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.791+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.791+0000 7f8f05183700 20 snap_mapper.get_snaps 1:f7919c75:::smithi13140796-20:4 got.empty() 2019-12-07T19:34:17.791+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 27'10 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=2 mbc={} ps=[a~1]] got missing 1:f7919c75:::smithi13140796-20:4 v 29'11 2019-12-07T19:34:17.791+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 27'10 active m=1 mbc={} ps=[a~1]] last_complete now 29'11 log.complete_to 29'12 2019-12-07T19:34:17.791+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.792+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57ff0600 seq 7 2019-12-07T19:34:17.792+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049c'.missing/0000000000000001.FE9893EA.4.smithi13140796-20..' <- missing/0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:17.792+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.792+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.792+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.792+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0600 cost 5406 (1 ios * 4000 + 1406 bytes) 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0600 onodes shared_blobs 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0600 allocated 0x[] released 0x[] 2019-12-07T19:34:17.792+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 prepare 2019-12-07T19:34:17.792+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0600 2019-12-07T19:34:17.792+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 io_done 2019-12-07T19:34:17.792+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.792+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 27'10 active m=1 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.792+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1182 < min_flush_size 524288 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12a948~4b5 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12a948 0x12a948~4b5 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xfa948 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x948 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdfd and padding block with 0x203 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12adfd 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.792+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119879s (0.000000099s flush + 0.000119780s kv commit) 2019-12-07T19:34:17.792+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0600> 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.792+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 kv_submitted 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0600 2019-12-07T19:34:17.792+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 finishing 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0600 onodes 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0600 done 2019-12-07T19:34:17.792+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.792+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0600 [] 2019-12-07T19:34:17.792+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.792+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.792+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 27'10 active m=1 mbc={} ps=[a~1]] recovery_committed_to version 29'11 now ondisk 2019-12-07T19:34:17.792+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGRecoveryDeleteReply(1.fs0 e40,39 1:f7919c75:::smithi13140796-20:4,29'11) v2 -- 0x558c5763c800 con 0x558c54c5bc00 2019-12-07T19:34:17.792+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.797+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 145 ==== MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 263206+0+0 (unknown 0 0 0) 0x558c5763c800 con 0x558c54c5bc00 2019-12-07T19:34:17.797+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c57f3f8c0 prio 127 cost 263144 latency 0.000027 epoch 40 MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.797+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) queued 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) pg 0x558c57383000 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 prio 127 cost 263144 latency 0.000069 MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] get_temp_recovery_object -3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] handle_recovery_push: Adding oid -3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head in the temp collection 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57b94f00 seq 8 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# key 0x857ffffffffffffffdf7919c'u!temp_recovering_1.fs5_29'12_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# key 0x857ffffffffffffffdf7919c'u!temp_recovering_1.fs5_29'12_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.797+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1201 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# = 0 2019-12-07T19:34:17.797+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 0x0~40000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 0x0~40000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~40000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57b94f00 32 blobs 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x40000/1000,40000,0 2019-12-07T19:34:17.797+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x87a000~40000/1000,40000,0 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x87a000~40000] 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800d570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800d570 blob([0x87a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d5e0 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c5800d570) get_ref 0x0~2000 Blob(0x558c5800d570 blob([0x87a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d5e0 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c5800d570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c5800d570 blob([0x87a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d5e0 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5800d5f8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800d650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800d650 blob([0x87c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42310 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c5800d650) get_ref 0x0~2000 Blob(0x558c5800d650 blob([0x87c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42310 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c5800d650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c5800d650 blob([0x87c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42310 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42328 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750df10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2690 blob([0x87e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750df10 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c574d2690) get_ref 0x0~2000 Blob(0x558c574d2690 blob([0x87e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750df10 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c574d2690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c574d2690 blob([0x87e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750df10 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5750df28 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750df80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750df80 blob([0x880000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060b60 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c5750df80) get_ref 0x0~2000 Blob(0x558c5750df80 blob([0x880000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060b60 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c5750df80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5750df80 blob([0x880000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060b60 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55060b78 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060bd0 blob([0x882000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060c40 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c55060bd0) get_ref 0x0~2000 Blob(0x558c55060bd0 blob([0x882000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060c40 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.blob(0x558c55060bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c55060bd0 blob([0x882000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060c40 sbid 0x0)) 2019-12-07T19:34:17.797+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55060c58 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060cb0 blob([0x884000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060d20 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060cb0) get_ref 0x0~2000 Blob(0x558c55060cb0 blob([0x884000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060d20 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c55060cb0 blob([0x884000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060d20 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55060d38 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060d90 blob([0x886000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060e00 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060d90) get_ref 0x0~2000 Blob(0x558c55060d90 blob([0x886000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060e00 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060d90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c55060d90 blob([0x886000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060e00 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55060e18 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060e70 blob([0x888000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060ee0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060e70) get_ref 0x0~2000 Blob(0x558c55060e70 blob([0x888000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060ee0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060e70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c55060e70 blob([0x888000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060ee0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55060ef8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060f50 blob([0x88a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060fc0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060f50) get_ref 0x0~2000 Blob(0x558c55060f50 blob([0x88a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060fc0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55060f50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c55060f50 blob([0x88a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060fc0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55060fd8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550610a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061030 blob([0x88c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550610a0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061030) get_ref 0x0~2000 Blob(0x558c55061030 blob([0x88c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550610a0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061030) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c55061030 blob([0x88c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550610a0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550610b8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061110 blob([0x88e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061180 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061110) get_ref 0x0~2000 Blob(0x558c55061110 blob([0x88e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061180 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061110) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c55061110 blob([0x88e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061180 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061198 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550611f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550611f0 blob([0x890000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061260 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550611f0) get_ref 0x0~2000 Blob(0x558c550611f0 blob([0x890000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061260 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550611f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c550611f0 blob([0x890000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061260 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061278 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550612d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550612d0 blob([0x892000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061340 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550612d0) get_ref 0x0~2000 Blob(0x558c550612d0 blob([0x892000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061340 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550612d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c550612d0 blob([0x892000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061340 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061358 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550613b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550613b0 blob([0x894000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061420 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550613b0) get_ref 0x0~2000 Blob(0x558c550613b0 blob([0x894000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061420 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550613b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c550613b0 blob([0x894000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061420 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061438 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061490 blob([0x896000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061500 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061490) get_ref 0x0~2000 Blob(0x558c55061490 blob([0x896000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061500 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c55061490 blob([0x896000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061500 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061518 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550615e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061570 blob([0x898000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550615e0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061570) get_ref 0x0~2000 Blob(0x558c55061570 blob([0x898000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550615e0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c55061570 blob([0x898000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550615e0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550615f8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550616c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061650 blob([0x89a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550616c0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061650) get_ref 0x0~2000 Blob(0x558c55061650 blob([0x89a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550616c0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c55061650 blob([0x89a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550616c0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550616d8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550617a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061730 blob([0x89c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550617a0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061730) get_ref 0x0~2000 Blob(0x558c55061730 blob([0x89c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550617a0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061730) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c55061730 blob([0x89c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550617a0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550617b8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061810 blob([0x89e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061880 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061810) get_ref 0x0~2000 Blob(0x558c55061810 blob([0x89e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061880 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c55061810 blob([0x89e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061880 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061898 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550618f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550618f0 blob([0x8a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061960 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550618f0) get_ref 0x0~2000 Blob(0x558c550618f0 blob([0x8a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061960 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550618f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c550618f0 blob([0x8a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061960 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061978 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550619d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550619d0 blob([0x8a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061a40 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550619d0) get_ref 0x0~2000 Blob(0x558c550619d0 blob([0x8a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061a40 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550619d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c550619d0 blob([0x8a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061a40 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061a58 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061ab0 blob([0x8a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061b20 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061ab0) get_ref 0x0~2000 Blob(0x558c55061ab0 blob([0x8a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061b20 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c55061ab0 blob([0x8a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061b20 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061b38 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061b90 blob([0x8a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061c00 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061b90) get_ref 0x0~2000 Blob(0x558c55061b90 blob([0x8a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061c00 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061b90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c55061b90 blob([0x8a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061c00 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061c18 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061c70 blob([0x8a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061ce0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061c70) get_ref 0x0~2000 Blob(0x558c55061c70 blob([0x8a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061ce0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061c70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c55061c70 blob([0x8a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061ce0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061cf8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061d50 blob([0x8aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061dc0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061d50) get_ref 0x0~2000 Blob(0x558c55061d50 blob([0x8aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061dc0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061d50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c55061d50 blob([0x8aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061dc0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061dd8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061e30 blob([0x8ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061ea0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061e30) get_ref 0x0~2000 Blob(0x558c55061e30 blob([0x8ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061ea0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061e30) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c55061e30 blob([0x8ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061ea0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061eb8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55061f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55061f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55061f10 blob([0x8ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061f80 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061f10) get_ref 0x0~2000 Blob(0x558c55061f10 blob([0x8ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55061f80 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55061f10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c55061f10 blob([0x8ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55061f80 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55061f98 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55062000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55062070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55062000 blob([0x8b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062070 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55062000) get_ref 0x0~2000 Blob(0x558c55062000 blob([0x8b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062070 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55062000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c55062000 blob([0x8b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55062070 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55062088 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550620e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55062150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550620e0 blob([0x8b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062150 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550620e0) get_ref 0x0~2000 Blob(0x558c550620e0 blob([0x8b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062150 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550620e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c550620e0 blob([0x8b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55062150 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55062168 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550621c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55062230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550621c0 blob([0x8b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062230 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550621c0) get_ref 0x0~2000 Blob(0x558c550621c0 blob([0x8b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062230 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550621c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c550621c0 blob([0x8b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55062230 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55062248 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550622a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55062310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550622a0 blob([0x8b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062310 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550622a0) get_ref 0x0~2000 Blob(0x558c550622a0 blob([0x8b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55062310 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c550622a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c550622a0 blob([0x8b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55062310 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55062328 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55062380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550623f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55062380 blob([0x8b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550623f0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55062380) get_ref 0x0~2000 Blob(0x558c55062380 blob([0x8b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550623f0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.blob(0x558c55062380) get_ref init 0x2000, 1000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c55062380 blob([0x8b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550623f0 sbid 0x0)) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55062408 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x40000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54ce3c88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.798+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 0x0~40000 = 0 2019-12-07T19:34:17.798+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 4 keys 2019-12-07T19:34:17.798+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 4 keys = 0 2019-12-07T19:34:17.798+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94f00 cost 395060 (33 ios * 4000 + 263060 bytes) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94f00 onodes 0x558c54ce3b80 shared_blobs 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54ce3c88) update 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54ce3c88) update inline shard 611 bytes from 32 extents 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# is 1102 (485 bytes onode + 2 bytes spanning blobs + 615 bytes inline extents) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94f00 allocated 0x[87a000~40000] released 0x[] 2019-12-07T19:34:17.798+0000 7f8f0117b700 10 freelist allocate 0x87a000~40000 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 freelist _xor first_key 0x800000 last_key 0x880000 2019-12-07T19:34:17.798+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 prepare 2019-12-07T19:34:17.798+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57b94f00 2019-12-07T19:34:17.798+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3f8c0 finish 2019-12-07T19:34:17.798+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.798+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 aio_wait 2019-12-07T19:34:17.798+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94f00 2019-12-07T19:34:17.798+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 io_done 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1310 < min_flush_size 524288 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3b80 had 1 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1333 < min_flush_size 524288 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1333 < min_flush_size 524288 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12adfd~535 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12adfd 0x12adfd~535 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xfadfd 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdfd 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x332 and padding block with 0xcce 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12b332 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.798+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119052s (0.000036342s flush + 0.000082710s kv commit) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.798+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94f00> 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.798+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 kv_submitted 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94f00 2019-12-07T19:34:17.798+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 finishing 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94f00 onodes 0x558c54ce3b80 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060b78 in 0x558c54862000) _finish_write discard buffer(0x558c57da50e0 space 0x558c55060b78 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060c58 in 0x558c54862000) _finish_write discard buffer(0x558c57f76750 space 0x558c55060c58 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060d38 in 0x558c54862000) _finish_write discard buffer(0x558c57da5170 space 0x558c55060d38 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060e18 in 0x558c54862000) _finish_write discard buffer(0x558c57da5200 space 0x558c55060e18 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060ef8 in 0x558c54862000) _finish_write discard buffer(0x558c57da5290 space 0x558c55060ef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060fd8 in 0x558c54862000) _finish_write discard buffer(0x558c57da5320 space 0x558c55060fd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550610b8 in 0x558c54862000) _finish_write discard buffer(0x558c57da53b0 space 0x558c550610b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061198 in 0x558c54862000) _finish_write discard buffer(0x558c57da5440 space 0x558c55061198 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061278 in 0x558c54862000) _finish_write discard buffer(0x558c57da54d0 space 0x558c55061278 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061358 in 0x558c54862000) _finish_write discard buffer(0x558c57da5560 space 0x558c55061358 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061438 in 0x558c54862000) _finish_write discard buffer(0x558c57da55f0 space 0x558c55061438 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061518 in 0x558c54862000) _finish_write discard buffer(0x558c57da5680 space 0x558c55061518 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550615f8 in 0x558c54862000) _finish_write discard buffer(0x558c57da5710 space 0x558c550615f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPushReply(1.fs0 40/39 [PushReplyOp(1:f7919c75:::smithi13140796-20:head)]) v3 -- 0x558c54c1ee00 con 0x558c54c5bc00 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550616d8 in 0x558c54862000) _finish_write discard buffer(0x558c57da57a0 space 0x558c550616d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550617b8 in 0x558c54862000) _finish_write discard buffer(0x558c57da5830 space 0x558c550617b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061898 in 0x558c54862000) _finish_write discard buffer(0x558c57da58c0 space 0x558c55061898 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061978 in 0x558c54862000) _finish_write discard buffer(0x558c57da5950 space 0x558c55061978 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061a58 in 0x558c54862000) _finish_write discard buffer(0x558c57da59e0 space 0x558c55061a58 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061b38 in 0x558c54862000) _finish_write discard buffer(0x558c57da5a70 space 0x558c55061b38 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061c18 in 0x558c54862000) _finish_write discard buffer(0x558c57da5b00 space 0x558c55061c18 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061cf8 in 0x558c54862000) _finish_write discard buffer(0x558c57da5b90 space 0x558c55061cf8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061dd8 in 0x558c54862000) _finish_write discard buffer(0x558c57da5c20 space 0x558c55061dd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061eb8 in 0x558c54862000) _finish_write discard buffer(0x558c57da5cb0 space 0x558c55061eb8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55061f98 in 0x558c54862000) _finish_write discard buffer(0x558c57da5d40 space 0x558c55061f98 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55062088 in 0x558c54862000) _finish_write discard buffer(0x558c57da5dd0 space 0x558c55062088 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55062168 in 0x558c54862000) _finish_write discard buffer(0x558c57da5e60 space 0x558c55062168 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55062248 in 0x558c54862000) _finish_write discard buffer(0x558c57da5ef0 space 0x558c55062248 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55062328 in 0x558c54862000) _finish_write discard buffer(0x558c57ece000 space 0x558c55062328 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55062408 in 0x558c54862000) _finish_write discard buffer(0x558c57ece090 space 0x558c55062408 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750df28 in 0x558c54862000) _finish_write discard buffer(0x558c57da5050 space 0x558c5750df28 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42328 in 0x558c54862000) _finish_write discard buffer(0x558c57da4fc0 space 0x558c57c42328 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d5f8 in 0x558c54862000) _finish_write discard buffer(0x558c57ccd7a0 space 0x558c5800d5f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94f00 done 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.798+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94f00 [] 2019-12-07T19:34:17.798+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.798+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.800+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 253 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 263206+0+0 (crc 0 0 0) 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.800+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c5740d600 prio 127 cost 263144 latency 0.000025 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.800+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.800+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) queued 2019-12-07T19:34:17.800+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.800+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e40) pg 0x558c57380000 2019-12-07T19:34:17.800+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740d600 prio 127 cost 263144 latency 0.000075 MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.800+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.800+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] get_temp_recovery_object -3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head 2019-12-07T19:34:17.800+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] handle_recovery_push: Adding oid -3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head in the temp collection 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5730af00 seq 15 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# key 0x857ffffffffffffffd14fc029d217465'mp_recovering_1.8s5_31'32_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# key 0x857ffffffffffffffd14fc029d217465'mp_recovering_1.8s5_31'32_39_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.800+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1202 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# = 0 2019-12-07T19:34:17.800+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 0x0~40000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 0x0~40000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~40000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730af00 32 blobs 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate 0x40000/1000,40000,0 2019-12-07T19:34:17.800+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x8ba000~40000/1000,40000,0 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x8ba000~40000] 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897dc0 blob([0x8ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec460 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57897dc0) get_ref 0x0~2000 Blob(0x558c57897dc0 blob([0x8ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec460 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57897dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57897dc0 blob([0x8ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec460 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ec478 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ec4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ea620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ec4d0 blob([0x8bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea620 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c580ec4d0) get_ref 0x0~2000 Blob(0x558c580ec4d0 blob([0x8bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea620 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c580ec4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c580ec4d0 blob([0x8bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea620 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c583ea638 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757f0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750c5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757f0a0 blob([0x8be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c5b0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5757f0a0) get_ref 0x0~2000 Blob(0x558c5757f0a0 blob([0x8be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c5b0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5757f0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c5757f0a0 blob([0x8be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750c5b0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750c5c8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750c620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750c690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750c620 blob([0x8c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c690 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c620) get_ref 0x0~2000 Blob(0x558c5750c620 blob([0x8c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c690 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c620) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5750c620 blob([0x8c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750c690 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750c6a8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750c700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750c770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750c700 blob([0x8c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c770 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c700) get_ref 0x0~2000 Blob(0x558c5750c700 blob([0x8c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c770 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c700) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5750c700 blob([0x8c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750c770 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750c788 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750c7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750c850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750c7e0 blob([0x8c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c850 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c7e0) get_ref 0x0~2000 Blob(0x558c5750c7e0 blob([0x8c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c850 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5750c7e0 blob([0x8c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750c850 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750c868 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750c8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750c930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750c8c0 blob([0x8c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c930 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c8c0) get_ref 0x0~2000 Blob(0x558c5750c8c0 blob([0x8c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750c930 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5750c8c0 blob([0x8c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750c930 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750c948 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750c9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ca10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750c9a0 blob([0x8c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ca10 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c9a0) get_ref 0x0~2000 Blob(0x558c5750c9a0 blob([0x8c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ca10 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750c9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5750c9a0 blob([0x8c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ca10 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750ca28 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750ca80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750caf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750ca80 blob([0x8ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750caf0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750ca80) get_ref 0x0~2000 Blob(0x558c5750ca80 blob([0x8ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750caf0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750ca80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5750ca80 blob([0x8ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750caf0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750cb08 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750cb60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750cbd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750cb60 blob([0x8cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cbd0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750cb60) get_ref 0x0~2000 Blob(0x558c5750cb60 blob([0x8cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cbd0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750cb60) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c5750cb60 blob([0x8cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cbd0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750cbe8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750cc40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ccb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750cc40 blob([0x8ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ccb0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750cc40) get_ref 0x0~2000 Blob(0x558c5750cc40 blob([0x8ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ccb0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750cc40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c5750cc40 blob([0x8ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ccb0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750ccc8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750cd20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750cd90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750cd20 blob([0x8d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cd90 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750cd20) get_ref 0x0~2000 Blob(0x558c5750cd20 blob([0x8d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cd90 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c5750cd20) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c5750cd20 blob([0x8d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cd90 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c5750cda8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063c00 blob([0x8d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063c70 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063c00) get_ref 0x0~2000 Blob(0x558c55063c00 blob([0x8d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063c70 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063c00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c55063c00 blob([0x8d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063c70 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c55063c88 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063ce0 blob([0x8d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063d50 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063ce0) get_ref 0x0~2000 Blob(0x558c55063ce0 blob([0x8d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063d50 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c55063ce0 blob([0x8d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063d50 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c55063d68 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063dc0 blob([0x8d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063e30 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063dc0) get_ref 0x0~2000 Blob(0x558c55063dc0 blob([0x8d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063e30 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c55063dc0 blob([0x8d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063e30 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c55063e48 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063ea0 blob([0x8d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063f10 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063ea0) get_ref 0x0~2000 Blob(0x558c55063ea0 blob([0x8d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063f10 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c55063ea0 blob([0x8d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063f10 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c55063f28 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063f80 blob([0x8da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0000 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063f80) get_ref 0x0~2000 Blob(0x558c55063f80 blob([0x8da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0000 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c55063f80) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c55063f80 blob([0x8da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0000 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0018 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed00e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0070 blob([0x8dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed00e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0070) get_ref 0x0~2000 Blob(0x558c57ed0070 blob([0x8dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed00e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0070) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57ed0070 blob([0x8dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed00e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed00f8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed01c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0150 blob([0x8de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed01c0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0150) get_ref 0x0~2000 Blob(0x558c57ed0150 blob([0x8de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed01c0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0150) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57ed0150 blob([0x8de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed01c0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed01d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed02a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0230 blob([0x8e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed02a0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0230) get_ref 0x0~2000 Blob(0x558c57ed0230 blob([0x8e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed02a0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0230) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57ed0230 blob([0x8e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed02a0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed02b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0310 blob([0x8e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0380 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0310) get_ref 0x0~2000 Blob(0x558c57ed0310 blob([0x8e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0380 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0310) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57ed0310 blob([0x8e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0380 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0398 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed03f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed03f0 blob([0x8e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0460 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed03f0) get_ref 0x0~2000 Blob(0x558c57ed03f0 blob([0x8e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0460 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed03f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57ed03f0 blob([0x8e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0460 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0478 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed04d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed04d0 blob([0x8e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0540 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed04d0) get_ref 0x0~2000 Blob(0x558c57ed04d0 blob([0x8e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0540 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed04d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57ed04d0 blob([0x8e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0540 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0558 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed05b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed05b0 blob([0x8e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0620 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed05b0) get_ref 0x0~2000 Blob(0x558c57ed05b0 blob([0x8e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0620 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed05b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57ed05b0 blob([0x8e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0620 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0638 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0690 blob([0x8ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0700 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0690) get_ref 0x0~2000 Blob(0x558c57ed0690 blob([0x8ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0700 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0690) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57ed0690 blob([0x8ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0700 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0718 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed07e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0770 blob([0x8ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed07e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0770) get_ref 0x0~2000 Blob(0x558c57ed0770 blob([0x8ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed07e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0770) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c57ed0770 blob([0x8ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed07e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed07f8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed08c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0850 blob([0x8ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed08c0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0850) get_ref 0x0~2000 Blob(0x558c57ed0850 blob([0x8ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed08c0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0850) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c57ed0850 blob([0x8ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed08c0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed08d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed09a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0930 blob([0x8f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed09a0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0930) get_ref 0x0~2000 Blob(0x558c57ed0930 blob([0x8f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed09a0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0930) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c57ed0930 blob([0x8f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed09a0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed09b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0a10 blob([0x8f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0a80 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0a10) get_ref 0x0~2000 Blob(0x558c57ed0a10 blob([0x8f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0a80 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0a10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c57ed0a10 blob([0x8f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0a80 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0a98 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0af0 blob([0x8f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0b60 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0af0) get_ref 0x0~2000 Blob(0x558c57ed0af0 blob([0x8f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0b60 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0af0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c57ed0af0 blob([0x8f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0b60 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0b78 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0bd0 blob([0x8f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0c40 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0bd0) get_ref 0x0~2000 Blob(0x558c57ed0bd0 blob([0x8f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0c40 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c57ed0bd0 blob([0x8f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0c40 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0c58 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed0cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed0cb0 blob([0x8f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0d20 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0cb0) get_ref 0x0~2000 Blob(0x558c57ed0cb0 blob([0x8f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed0d20 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.blob(0x558c57ed0cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c57ed0cb0 blob([0x8f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed0d20 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed0d38 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x40000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d008) dirty_range mark inline shard dirty 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 0x0~40000 = 0 2019-12-07T19:34:17.800+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 146 ==== MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [262144~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1343488, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 74328+0+0 (unknown 0 0 0) 0x558c54c1ee00 con 0x558c54c5bc00 2019-12-07T19:34:17.800+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 4 keys 2019-12-07T19:34:17.800+0000 7f8f23c56700 15 osd.7 40 enqueue_op 0x558c57f3ef20 prio 127 cost 74728 latency 0.000021 epoch 40 MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [262144~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1343488, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 4 keys = 0 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730af00 cost 395060 (33 ios * 4000 + 263060 bytes) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730af00 onodes 0x558c5840cf00 shared_blobs 2019-12-07T19:34:17.800+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [262144~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1343488, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e40) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d008) update 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d008) update inline shard 611 bytes from 32 extents 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# is 1102 (485 bytes onode + 2 bytes spanning blobs + 615 bytes inline extents) 2019-12-07T19:34:17.800+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730af00 allocated 0x[8ba000~40000] released 0x[] 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.800+0000 7f8f04982700 10 freelist allocate 0x8ba000~40000 2019-12-07T19:34:17.800+0000 7f8f04982700 20 freelist _xor first_key 0x880000 last_key 0x880000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [262144~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1343488, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e40) queued 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 prepare 2019-12-07T19:34:17.800+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730af00 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [262144~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1343488, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e40) pg 0x558c57383000 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3ef20 prio 127 cost 74728 latency 0.000061 MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [262144~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1343488, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.fs5 40/39 [PushOp(1:f7919c75:::smithi13140796-20:head, version: 29'12, data_included: [262144~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:f7919c75:::smithi13140796-20:head@29'12, size: 1343488, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1343488, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] get_temp_recovery_object -3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] handle_recovery_push: Removing oid -3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head from the temp collection 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] on_local_recover: 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 lc 29'11 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active m=1 mbc={} ps=[a~1]] got missing 1:f7919c75:::smithi13140796-20:head v 29'12 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 29'11 active mbc={} ps=[a~1]] last_complete now 29'12 log.complete_to at end 2019-12-07T19:34:17.800+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57b95200 seq 9 2019-12-07T19:34:17.800+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 0x40000~12000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 0x40000~12000 - have 0x40000 (262144) bytes fadvise_flags 0x0 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x40000~12000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57b95200 9 blobs 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x12000/1000,12000,0 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x8fa000~12000/1000,12000,0 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x8fa000~12000] 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550600e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060070 blob([0x8fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550600e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55060070) get_ref 0x0~2000 Blob(0x558c55060070 blob([0x8fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550600e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55060070) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c55060070 blob([0x8fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550600e0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550600f8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800d500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550632d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800d500 blob([0x8fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550632d0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c5800d500) get_ref 0x0~2000 Blob(0x558c5800d500 blob([0x8fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550632d0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c5800d500) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c5800d500 blob([0x8fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550632d0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550632e8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550633b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063340 blob([0x8fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550633b0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063340) get_ref 0x0~2000 Blob(0x558c55063340 blob([0x8fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550633b0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063340) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c55063340 blob([0x8fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550633b0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550633c8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063420 blob([0x900000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063490 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063420) get_ref 0x0~2000 Blob(0x558c55063420 blob([0x900000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063490 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063420) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c55063420 blob([0x900000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063490 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550634a8 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063500 blob([0x902000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063570 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063500) get_ref 0x0~2000 Blob(0x558c55063500 blob([0x902000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063570 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063500) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c55063500 blob([0x902000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063570 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55063588 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550635e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550635e0 blob([0x904000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063650 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c550635e0) get_ref 0x0~2000 Blob(0x558c550635e0 blob([0x904000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063650 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c550635e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c550635e0 blob([0x904000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063650 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55063668 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550636c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550636c0 blob([0x906000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063730 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c550636c0) get_ref 0x0~2000 Blob(0x558c550636c0 blob([0x906000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063730 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c550636c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c550636c0 blob([0x906000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063730 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55063748 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550637a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550637a0 blob([0x908000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063810 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c550637a0) get_ref 0x0~2000 Blob(0x558c550637a0 blob([0x908000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063810 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c550637a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c550637a0 blob([0x908000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063810 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55063828 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55063880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550638f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55063880 blob([0x90a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550638f0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063880) get_ref 0x0~2000 Blob(0x558c55063880 blob([0x90a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550638f0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.blob(0x558c55063880) get_ref init 0x2000, 1000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c55063880 blob([0x90a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550638f0 sbid 0x0)) 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c55063908 in 0x558c54862000) _discard 0x0~2000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x52000 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54ce3c88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.800+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# 0x40000~12000 = 0 2019-12-07T19:34:17.800+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:f7919c75:::smithi13140796-20:head# key 0x858000000000000001f7919c'u!smithi13140796-20!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:f7919c75:::smithi13140796-20:head# key 0x858000000000000001f7919c'u!smithi13140796-20!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:17.801+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# -> 5#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.fs5_head 5#-3:f7919c75:::temp_recovering_1.fs5_29'12_39_head:head# -> 5#1:f7919c75:::smithi13140796-20:head# = 0 2019-12-07T19:34:17.801+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049c'.missing/0000000000000001.FE9893EA.head.smithi13140796-20..' <- missing/0000000000000001.FE9893EA.head.smithi13140796-20.. 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.801+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95200 cost 115515 (10 ios * 4000 + 75515 bytes) 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95200 onodes 0x558c54ce3b80 shared_blobs 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54ce3c88) update 5#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54ce3c88) update inline shard 782 bytes from 41 extents 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:f7919c75:::smithi13140796-20:head# is 1273 (485 bytes onode + 2 bytes spanning blobs + 786 bytes inline extents) 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95200 allocated 0x[8fa000~12000] released 0x[] 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 freelist allocate 0x8fa000~12000 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 freelist _xor first_key 0x880000 last_key 0x900000 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 prepare 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57b95200 2019-12-07T19:34:17.801+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c5740d600 finish 2019-12-07T19:34:17.801+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.801+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.801+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 aio_wait 2019-12-07T19:34:17.801+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730af00 2019-12-07T19:34:17.801+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 io_done 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1281 < min_flush_size 524288 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840cf00 had 1 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1304 < min_flush_size 524288 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1304 < min_flush_size 524288 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12b332~518 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12b332 0x12b332~518 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xfb332 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x332 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x84a and padding block with 0x7b6 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12b84a 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.801+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.801+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.801+0000 7f8f0117b700 10 osd.7 40 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:17.801+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.801+0000 7f8f05183700 20 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 29'11 active mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.801+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002616918s (0.000047855s flush + 0.002569063s kv commit) 2019-12-07T19:34:17.803+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 aio_wait 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.803+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95200 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730af00> 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.803+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 kv_submitted 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730af00 2019-12-07T19:34:17.803+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 io_done 2019-12-07T19:34:17.803+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 finishing 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730af00 onodes 0x558c5840cf00 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063c88 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecee10 space 0x558c55063c88 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063d68 in 0x558c53a59960) _finish_write discard buffer(0x558c57eceea0 space 0x558c55063d68 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063e48 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecef30 space 0x558c55063e48 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063f28 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecefc0 space 0x558c55063f28 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750c5c8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da46c0 space 0x558c5750c5c8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750c6a8 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4750 space 0x558c5750c6a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750c788 in 0x558c53a59960) _finish_write discard buffer(0x558c57da47e0 space 0x558c5750c788 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750c868 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4870 space 0x558c5750c868 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750c948 in 0x558c53a59960) _finish_write discard buffer(0x558c57eceab0 space 0x558c5750c948 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ca28 in 0x558c53a59960) _finish_write discard buffer(0x558c57eceb40 space 0x558c5750ca28 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750cb08 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecebd0 space 0x558c5750cb08 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750cbe8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecec60 space 0x558c5750cbe8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ccc8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ececf0 space 0x558c5750ccc8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750cda8 in 0x558c53a59960) _finish_write discard buffer(0x558c57eced80 space 0x558c5750cda8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:14fc029d:::smithi13140796-41:head)]) v3 -- 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0018 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf050 space 0x558c57ed0018 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed00f8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf0e0 space 0x558c57ed00f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed01d8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf170 space 0x558c57ed01d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed02b8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf200 space 0x558c57ed02b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0398 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf290 space 0x558c57ed0398 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0478 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf320 space 0x558c57ed0478 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0558 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf3b0 space 0x558c57ed0558 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0638 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf440 space 0x558c57ed0638 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0718 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf4d0 space 0x558c57ed0718 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed07f8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf560 space 0x558c57ed07f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed08d8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf5f0 space 0x558c57ed08d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed09b8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf680 space 0x558c57ed09b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0a98 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf710 space 0x558c57ed0a98 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0b78 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf7a0 space 0x558c57ed0b78 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0c58 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf830 space 0x558c57ed0c58 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed0d38 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecf8c0 space 0x558c57ed0d38 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec478 in 0x558c53a59960) _finish_write discard buffer(0x558c583caf30 space 0x558c580ec478 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ea638 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4630 space 0x558c583ea638 0x0~2000 writing nocache) 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730af00 done 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.803+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730af00 [] 2019-12-07T19:34:17.803+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.803+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.803+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2697 < min_flush_size 524288 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3b80 had 1 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:17.803+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840d680 had 1 2019-12-07T19:34:17.803+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2720 < min_flush_size 524288 2019-12-07T19:34:17.803+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2720 < min_flush_size 524288 2019-12-07T19:34:17.804+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.804+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12b84a~aa0 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.804+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12b84a 0x12b84a~aa0 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xfb84a 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x84a 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2ea and padding block with 0xd16 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12c2ea 2019-12-07T19:34:17.804+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.804+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.804+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000154614s (0.000056724s flush + 0.000097890s kv commit) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.804+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95200> 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.804+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 kv_submitted 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95200 2019-12-07T19:34:17.804+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 finishing 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95200 onodes 0x558c54ce3b80 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550600f8 in 0x558c54862000) _finish_write discard buffer(0x558c57ece240 space 0x558c550600f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550632e8 in 0x558c54862000) _finish_write discard buffer(0x558c57ece2d0 space 0x558c550632e8 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550633c8 in 0x558c54862000) _finish_write discard buffer(0x558c57ece360 space 0x558c550633c8 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550634a8 in 0x558c54862000) _finish_write discard buffer(0x558c57ece3f0 space 0x558c550634a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063588 in 0x558c54862000) _finish_write discard buffer(0x558c57ece480 space 0x558c55063588 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063668 in 0x558c54862000) _finish_write discard buffer(0x558c57ece510 space 0x558c55063668 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063748 in 0x558c54862000) _finish_write discard buffer(0x558c57ece5a0 space 0x558c55063748 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063828 in 0x558c54862000) _finish_write discard buffer(0x558c57ece630 space 0x558c55063828 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063908 in 0x558c54862000) _finish_write discard buffer(0x558c57ece6c0 space 0x558c55063908 0x0~2000 writing nocache) 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95200 done 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:17.804+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 lcod 29'11 active mbc={} ps=[a~1]] recovery_committed_to version 29'12 now ondisk 2019-12-07T19:34:17.804+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95200 [] 2019-12-07T19:34:17.804+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.804+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.804+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_trim(1.fs0 29'12 e40/40) v2 -- 0x558c575cc1e0 con 0x558c54c5bc00 2019-12-07T19:34:17.804+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDPGPushReply(1.fs0 40/39 [PushReplyOp(1:f7919c75:::smithi13140796-20:head)]) v3 -- 0x558c5763c800 con 0x558c54c5bc00 2019-12-07T19:34:17.804+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.804+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 918103 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.805+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 147 ==== MRecoveryReserve(1.fs5 RELEASE e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c581e8b40 con 0x558c54c5bc00 2019-12-07T19:34:17.805+0000 7f8f23c56700 15 osd.7 40 enqueue_peering_evt 1.fs5 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.805+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.805+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.805+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.805+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.805+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c57383000 2019-12-07T19:34:17.805+0000 7f8f0117b700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.805+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.806563+0000 1 0.000034 2019-12-07T19:34:17.805+0000 7f8f0117b700 5 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.805+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:17.805+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:17.807+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 254 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [262144~28672], data_size: 28672, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1163264, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 29272+0+0 (crc 0 0 0) 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:17.807+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c57eff340 prio 127 cost 29672 latency 0.000030 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [262144~28672], data_size: 28672, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1163264, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.807+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [262144~28672], data_size: 28672, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1163264, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 29672 e40) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.807+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [262144~28672], data_size: 28672, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1163264, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 29672 e40) queued 2019-12-07T19:34:17.807+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.807+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [262144~28672], data_size: 28672, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1163264, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 29672 e40) pg 0x558c57380000 2019-12-07T19:34:17.807+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 165 ==== MRecoveryReserve(1.2s0 REQUEST prio: 190 e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c576dd4a0 con 0x558c54c5b400 2019-12-07T19:34:17.807+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.2s0 epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.807+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) 2019-12-07T19:34:17.807+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57eff340 prio 127 cost 29672 latency 0.000071 MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [262144~28672], data_size: 28672, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1163264, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.807+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.807+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.807+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.807+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.807+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:14fc029d:::smithi13140796-41:head, version: 31'32, data_included: [262144~28672], data_size: 28672, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:14fc029d:::smithi13140796-41:head@31'32, size: 1163264, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1163264, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.807+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RequestRecoveryPrio: priority 190 2019-12-07T19:34:17.807+0000 7f8f04982700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] get_temp_recovery_object -3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head 2019-12-07T19:34:17.807+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] handle_recovery_push: Removing oid -3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head from the temp collection 2019-12-07T19:34:17.807+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.808362+0000 4 0.000258 2019-12-07T19:34:17.807+0000 7f8f07988700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.807+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] on_local_recover: 1:14fc029d:::smithi13140796-41:head 2019-12-07T19:34:17.807+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:17.807+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.807+0000 7f8f0b990700 15 osd.7 40 enqueue_peering_evt 1.2s0 epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.807+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) 2019-12-07T19:34:17.807+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'31 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=2 mbc={} ps=[a~1]] got missing 1:14fc029d:::smithi13140796-41:head v 31'32 2019-12-07T19:34:17.807+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'32 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 31'31 active m=1 mbc={} ps=[a~1]] last_complete now 31'32 log.complete_to 36'33 2019-12-07T19:34:17.807+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.807+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.807+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.807+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.807+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.807+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RemoteRecoveryReserved 2019-12-07T19:34:17.807+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.2s1 GRANT e40/40) v3 -- 0x558c54d9e5a0 con 0x558c54c5b400 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57ef9200 seq 16 2019-12-07T19:34:17.807+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 0x40000~7000 2019-12-07T19:34:17.807+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:17.808444+0000 1 0.000035 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 0x40000~7000 - have 0x40000 (262144) bytes fadvise_flags 0x0 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x40000~7000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef9200 4 blobs 2019-12-07T19:34:17.807+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.807+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:17.807+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.807+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate 0x7000/1000,7000,0 2019-12-07T19:34:17.807+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x90c000~7000/1000,7000,0 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x90c000~7000] 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ec620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ec620 blob([0x90c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec690 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c580ec620) get_ref 0x0~2000 Blob(0x558c580ec620 blob([0x90c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec690 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c580ec620) get_ref init 0x2000, 1000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c580ec620 blob([0x90c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec690 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ec6a8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ec3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db4000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ec3f0 blob([0x90e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4000 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c580ec3f0) get_ref 0x0~2000 Blob(0x558c580ec3f0 blob([0x90e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4000 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c580ec3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c580ec3f0 blob([0x90e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db4000 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4018 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db4070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db40e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db4070 blob([0x910000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db40e0 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c57db4070) get_ref 0x0~2000 Blob(0x558c57db4070 blob([0x910000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db40e0 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c57db4070) get_ref init 0x2000, 1000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c57db4070 blob([0x910000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db40e0 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db40f8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db4150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db41c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db4150 blob([0x912000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db41c0 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c57db4150) get_ref 0x0~1000 Blob(0x558c57db4150 blob([0x912000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db41c0 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.blob(0x558c57db4150) get_ref init 0x1000, 1000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~1000: 0x0~1000 Blob(0x558c57db4150 blob([0x912000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57db41c0 sbid 0x0)) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db41d8 in 0x558c53a59960) _discard 0x0~1000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x47000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d008) dirty_range mark inline shard dirty 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# 0x40000~7000 = 0 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:14fc029d:::smithi13140796-41:head# key 0x85800000000000000114fc029d21736d'ithi13140796-41!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:14fc029d:::smithi13140796-41:head# key 0x85800000000000000114fc029d21736d'ithi13140796-41!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.807+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# -> 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#-3:14fc029d:::temp_recovering_1.8s5_31'32_39_head:head# -> 5#1:14fc029d:::smithi13140796-41:head# = 0 2019-12-07T19:34:17.807+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.82F3049B.head.smithi13140796-41..' <- missing/0000000000000001.82F3049B.head.smithi13140796-41.. 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.807+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9200 cost 50459 (5 ios * 4000 + 30459 bytes) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9200 onodes 0x558c5840cf00 shared_blobs 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d008) update 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d008) update inline shard 684 bytes from 36 extents 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:14fc029d:::smithi13140796-41:head# is 1175 (485 bytes onode + 2 bytes spanning blobs + 688 bytes inline extents) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9200 allocated 0x[90c000~7000] released 0x[] 2019-12-07T19:34:17.807+0000 7f8f04982700 10 freelist allocate 0x90c000~7000 2019-12-07T19:34:17.807+0000 7f8f04982700 20 freelist _xor first_key 0x900000 last_key 0x900000 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 prepare 2019-12-07T19:34:17.807+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef9200 2019-12-07T19:34:17.807+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c57eff340 finish 2019-12-07T19:34:17.807+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.807+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'32 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 31'31 active m=1 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.807+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.807+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 aio_wait 2019-12-07T19:34:17.807+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9200 2019-12-07T19:34:17.807+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 io_done 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2570 < min_flush_size 524288 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840cf00 had 1 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840d180 had 1 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2593 < min_flush_size 524288 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2593 < min_flush_size 524288 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12c2ea~a21 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12c2ea 0x12c2ea~a21 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xfc2ea 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2ea 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd0b and padding block with 0x2f5 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x12cd0b 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.807+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000181980s (0.000042693s flush + 0.000139287s kv commit) 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.807+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9200> 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.807+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 kv_submitted 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9200 2019-12-07T19:34:17.807+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 finishing 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9200 onodes 0x558c5840cf00 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4018 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecfb00 space 0x558c57db4018 0x0~2000 writing nocache) 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db40f8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecfb90 space 0x558c57db40f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db41d8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecfc20 space 0x558c57db41d8 0x0~1000 writing nocache) 2019-12-07T19:34:17.807+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec6a8 in 0x558c53a59960) _finish_write discard buffer(0x558c57ecfa70 space 0x558c580ec6a8 0x0~2000 writing nocache) 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9200 done 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.807+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9200 [] 2019-12-07T19:34:17.807+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.807+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.807+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'32 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 31'31 active m=1 mbc={} ps=[a~1]] recovery_committed_to version 31'32 now ondisk 2019-12-07T19:34:17.807+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:14fc029d:::smithi13140796-41:head)]) v3 -- 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:17.807+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.808+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 166 ==== MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=69, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c58092c80 con 0x558c54c5b400 2019-12-07T19:34:17.808+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c578d3a20 prio 3 cost 0 latency 0.000021 epoch 40 MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=69, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.808+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=69, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.808+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=69, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.808+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=69, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5731a000 2019-12-07T19:34:17.808+0000 7f8f07988700 10 osd.7 40 dequeue_op 0x558c578d3a20 prio 3 cost 0 latency 0.000064 MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=69, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.808+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=69, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.808+0000 7f8f07988700 25 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.808+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 0x0~40000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x2c000 (180224) 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578182a0 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578181c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578180e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818070 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818310 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578183f0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578184d0 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818540 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578185b0 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818690 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818770 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578187e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818850 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578188c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818930 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578189a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818a10 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818a80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818af0 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818b60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818bd0 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818cb0 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818d20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818d90 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818e00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818e70 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818f50 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819030 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578190a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578191f0 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819260 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578192d0 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819340 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818070 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818000 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578181c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578180e0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578182a0 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818230 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818310 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818380 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578183f0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818460 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578184d0 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818540 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578185b0 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818620 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818690 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818700 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818770 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578187e0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818850 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578188c0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818930 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578189a0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818a10 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818a80 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818af0 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818b60 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818bd0 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818cb0 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818d20 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818d90 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818e00 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818e70 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818f50 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818fc0 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57819030 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578190a0 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57819110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819180 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578191f0 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819260 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578192d0 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819340 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.808+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.810+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 255 ==== MOSDPGPush(1.8s5 40/39 [PushOp(1:15219530:::smithi13140796-21:head, version: 36'33, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 23, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@36'33, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 77131+0+0 (crc 0 0 0) 0x558c54dfbe00 con 0x558c54c64000 2019-12-07T19:34:17.810+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c581cc000 prio 127 cost 1000 latency 0.000028 epoch 40 MOSDPGPush(1.8s5 40/39 [PushOp(1:15219530:::smithi13140796-21:head, version: 36'33, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 23, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@36'33, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.810+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:15219530:::smithi13140796-21:head, version: 36'33, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 23, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@36'33, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 1000 e40) 2019-12-07T19:34:17.810+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.810+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:15219530:::smithi13140796-21:head, version: 36'33, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 23, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@36'33, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 1000 e40) queued 2019-12-07T19:34:17.810+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.810+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 40/39 [PushOp(1:15219530:::smithi13140796-21:head, version: 36'33, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 23, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@36'33, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 1000 e40) pg 0x558c57380000 2019-12-07T19:34:17.810+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c581cc000 prio 127 cost 1000 latency 0.000066 MOSDPGPush(1.8s5 40/39 [PushOp(1:15219530:::smithi13140796-21:head, version: 36'33, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 23, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@36'33, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 36'33 lc 31'32 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.810+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'32 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=1 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.8s5 40/39 [PushOp(1:15219530:::smithi13140796-21:head, version: 36'33, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 23, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@36'33, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.810+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'32 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=1 mbc={} ps=[a~1]] on_local_recover: 1:15219530:::smithi13140796-21:head 2019-12-07T19:34:17.810+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 lc 31'32 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active m=1 mbc={} ps=[a~1]] got missing 1:15219530:::smithi13140796-21:head v 36'33 2019-12-07T19:34:17.810+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 31'32 active mbc={} ps=[a~1]] last_complete now 36'33 log.complete_to at end 2019-12-07T19:34:17.810+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818070 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818018 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c578ee090 space 0x558c57818018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57ef8000 seq 17 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578181c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578180e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578180f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c580cd710 space 0x558c578180f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578182a0 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57ece630 space 0x558c57818248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:15219530:::smithi13140796-21:head# key 0x858000000000000001152195'0!smithi13140796-21!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818310 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818398 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57ecfe60 space 0x558c57818398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578183f0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818478 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57ecfef0 space 0x558c57818478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578184d0 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818540 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818558 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa000 space 0x558c57818558 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578185b0 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818638 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa090 space 0x558c57818638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:15219530:::smithi13140796-21:head# key 0x858000000000000001152195'0!smithi13140796-21!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818690 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818718 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa120 space 0x558c57818718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818770 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578187e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578187f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa1b0 space 0x558c578187f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818850 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578188c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1203 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578188d8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa240 space 0x558c578188d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# = 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818930 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578189a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578189b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa2d0 space 0x558c578189b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818a10 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818a80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818a98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa360 space 0x558c57818a98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818af0 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818b60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818b78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa3f0 space 0x558c57818b78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818bd0 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818c58 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa480 space 0x558c57818c58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818cb0 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818d20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818d38 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa510 space 0x558c57818d38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818d90 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818e00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818e18 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa5a0 space 0x558c57818e18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818e70 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818ef8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa630 space 0x558c57818ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818f50 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 23 keys 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818fd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa6c0 space 0x558c57818fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57819030 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578190a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578190b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa750 space 0x558c578190b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57819110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819198 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa7e0 space 0x558c57819198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578191f0 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819260 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819278 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa870 space 0x558c57819278 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578192d0 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819340 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:17.810+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819358 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.810+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa900 space 0x558c57819358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.810+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 0x0~40000 = 180224 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 23 keys = 0 2019-12-07T19:34:17.810+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=180224 len=180224 2019-12-07T19:34:17.810+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.810+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x000000000000049f'.missing/0000000000000001.8A489AC0.head.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.head.smithi13140796-21.. 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.810+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:17.810+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:17.810+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:17.810+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 180224 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.810+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.810+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8000 cost 82246 (1 ios * 4000 + 78246 bytes) 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8000 onodes 0x558c5840d400 shared_blobs 2019-12-07T19:34:17.810+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d508) update 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840d508) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:17.810+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.2s1 40/39 ECSubReadReply(tid=69, attrs_read=0)) v2 -- 0x558c57e8e780 con 0x558c54c5b400 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:15219530:::smithi13140796-21:head# is 76576 (76568 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:17.810+0000 7f8f07988700 10 osd.7 40 dequeue_op 0x558c578d3a20 finish 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8000 allocated 0x[] released 0x[] 2019-12-07T19:34:17.810+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 prepare 2019-12-07T19:34:17.810+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8000 2019-12-07T19:34:17.810+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 io_done 2019-12-07T19:34:17.810+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.810+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.810+0000 7f8f04982700 10 osd.7 40 dequeue_op 0x558c581cc000 finish 2019-12-07T19:34:17.810+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13045 < min_flush_size 524288 2019-12-07T19:34:17.810+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 45813 < min_flush_size 524288 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 77840 < min_flush_size 524288 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840d400 had 1 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 77863 < min_flush_size 524288 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 77863 < min_flush_size 524288 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.811+0000 7f8f0898a700 20 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 31'32 active mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x12cd0b~13027 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.811+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x12cd0b 0x12cd0b~13027 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:17.735158+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _allocate len 0xfd32 from 0 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _allocate len 0xfd32 from 1 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd260000 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd260000~10000/10000,10000,acd260000 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x140000 to include allocated 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 41 (was clean) 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x140000 mtime 2019-12-07T19:34:17.812271+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~110000 x_off 0xfcd0b 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd0b 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd32 and padding block with 0x2ce 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x13fd32 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (41) on file(ino 16 size 0x140000 mtime 2019-12-07T19:34:17.812271+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]), flushing log 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x140000 mtime 2019-12-07T19:34:17.812271+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 41 len 0x49 crc 0x89cdc6e7) 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x28000~1000 to file(ino 1 size 0x28000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x28000 0x28000~1000 to file(ino 1 size 0x28000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x29000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x28000 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x29000 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:17.811+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000019 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 41 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x140000 mtime 2019-12-07T19:34:17.812271+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000440570s (0.000000105s flush + 0.000440465s kv commit) 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.811+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8000> 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.811+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 kv_submitted 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8000 2019-12-07T19:34:17.811+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 finishing 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8000 onodes 0x558c5840d400 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8000 done 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:17.811+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.811+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8000 [] 2019-12-07T19:34:17.811+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.811+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.811+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 lcod 31'32 active mbc={} ps=[a~1]] recovery_committed_to version 36'33 now ondisk 2019-12-07T19:34:17.811+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_trim(1.8s0 36'33 e40/40) v2 -- 0x558c57f10780 con 0x558c54c64000 2019-12-07T19:34:17.811+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 40/39 [PushReplyOp(1:15219530:::smithi13140796-21:head)]) v3 -- 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:17.811+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.812+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 256 ==== MRecoveryReserve(1.8s5 RELEASE e40/40) v3 ==== 30+0+0 (crc 0 0 0) 0x558c57f10780 con 0x558c54c64000 2019-12-07T19:34:17.812+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.8s5 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.812+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.812+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.812+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.812+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.812+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c57380000 2019-12-07T19:34:17.812+0000 7f8f04982700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.812+0000 7f8f04982700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.813304+0000 1 0.000028 2019-12-07T19:34:17.812+0000 7f8f04982700 5 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.812+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:17.812+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:17.813+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 167 ==== MOSDPGPush(1.2s0 40/39 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 181286+0+0 (unknown 0 0 0) 0x558c5763de00 con 0x558c54c5b400 2019-12-07T19:34:17.813+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c581cc160 prio 127 cost 181224 latency 0.000024 epoch 40 MOSDPGPush(1.2s0 40/39 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.813+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPush(1.2s0 40/39 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 181224 e40) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.813+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPush(1.2s0 40/39 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 181224 e40) queued 2019-12-07T19:34:17.813+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.813+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPush(1.2s0 40/39 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 181224 e40) pg 0x558c57e88000 2019-12-07T19:34:17.813+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc160 prio 127 cost 181224 latency 0.000064 MOSDPGPush(1.2s0 40/39 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] 2019-12-07T19:34:17.813+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.2s0 40/39 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.813+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] on_local_recover: 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:17.813+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'3 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=5 mbc={} ps=[a~1]] got missing 1:4986b721:::smithi13140796-43:head v 21'4 2019-12-07T19:34:17.813+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'5 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=4 mbc={} ps=[a~1]] last_complete now 21'5 log.complete_to 21'6 2019-12-07T19:34:17.813+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57ef9800 seq 5 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:4986b721:::smithi13140796-43:head# key 0x8080000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:4986b721:::smithi13140796-43:head# key 0x8080000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:17.813+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1204 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# = 0 2019-12-07T19:34:17.813+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef9800 22 blobs 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.813+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2c000/1000,2c000,0 2019-12-07T19:34:17.813+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x913000~2c000/1000,2c000,0 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x913000~2c000] 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794db90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42bd0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5794db90) get_ref 0x0~2000 Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42bd0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5794db90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42bd0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57c42be8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d25b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881730 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c574d25b0) get_ref 0x0~2000 Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881730 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c574d25b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881730 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57881748 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c700 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578df500) get_ref 0x0~2000 Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c700 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578df500) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c700 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c718 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800c770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c7e0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800c770) get_ref 0x0~2000 Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c7e0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800c770) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c7e0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c7f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800c850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c8c0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800c850) get_ref 0x0~2000 Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c8c0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800c850) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c8c0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c8d8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800c930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800c9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c9a0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800c930) get_ref 0x0~2000 Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c9a0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800c930) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c9a0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800c9b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800ca10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800ca80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800ca80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800ca10) get_ref 0x0~2000 Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800ca80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800ca10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ca80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800ca98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800caf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800cb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cb60 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800caf0) get_ref 0x0~2000 Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800cb60 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800caf0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cb60 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5800cb78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5800cbd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ecc40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecc40 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800cbd0) get_ref 0x0~2000 Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecc40 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5800cbd0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecc40 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c580ecc58 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ecbd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ecb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecb60 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c580ecbd0) get_ref 0x0~2000 Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ecb60 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c580ecbd0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecb60 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c580ecb78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ecaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580eca80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580eca80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c580ecaf0) get_ref 0x0~2000 Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580eca80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c580ecaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580eca80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c580eca98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580eca10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c770 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c580eca10) get_ref 0x0~2000 Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c770 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c580eca10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c770 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5794c788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794c700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c070 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5794c700) get_ref 0x0~2000 Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c070 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5794c700) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c070 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5794c088 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578dea80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dea80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5809b0a0) get_ref 0x0~2000 Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dea80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c5809b0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea80 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578dea98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578dec40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de9a0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578dec40) get_ref 0x0~2000 Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de9a0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578dec40) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de9a0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de9b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de7e0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578de8c0) get_ref 0x0~2000 Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de7e0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578de8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de7e0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578de7f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0770 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578de700) get_ref 0x0~2000 Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0770 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c578de700) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0770 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574e0788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e0bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57396ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396ee0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c574e0bd0) get_ref 0x0~2000 Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396ee0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c574e0bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396ee0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57396ef8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5c00 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c574fa3f0) get_ref 0x0~2000 Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5c00 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c574fa3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5c00 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57db5c18 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5ce0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c57db5c70) get_ref 0x0~2000 Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5ce0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c57db5c70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ce0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57db5cf8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5dc0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c57db5d50) get_ref 0x0~2000 Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5dc0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c57db5d50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5dc0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57db5dd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5ea0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c57db5e30) get_ref 0x0~2000 Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5ea0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.blob(0x558c57db5e30) get_ref init 0x2000, 1000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ea0 sbid 0x0)) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c57db5eb8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2c000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13508) dirty_range mark inline shard dirty 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 = 0 2019-12-07T19:34:17.813+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 4 keys 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 4 keys = 0 2019-12-07T19:34:17.813+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000493'.missing/0000000000000001.2916DE48.head.smithi13140796-43..' <- missing/0000000000000001.2916DE48.head.smithi13140796-43.. 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.813+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9800 cost 274461 (23 ios * 4000 + 182461 bytes) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9800 onodes 0x558c54e13400 shared_blobs 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13508) update 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13508) update inline shard 421 bytes from 22 extents 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:4986b721:::smithi13140796-43:head# is 912 (485 bytes onode + 2 bytes spanning blobs + 425 bytes inline extents) 2019-12-07T19:34:17.813+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9800 allocated 0x[913000~2c000] released 0x[] 2019-12-07T19:34:17.813+0000 7f8f03980700 10 freelist allocate 0x913000~2c000 2019-12-07T19:34:17.813+0000 7f8f03980700 20 freelist _xor first_key 0x900000 last_key 0x900000 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 prepare 2019-12-07T19:34:17.813+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef9800 2019-12-07T19:34:17.813+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc160 finish 2019-12-07T19:34:17.813+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.813+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'5 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=4 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.813+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.814+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 aio_wait 2019-12-07T19:34:17.814+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9800 2019-12-07T19:34:17.814+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 io_done 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 718 < min_flush_size 524288 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2246 < min_flush_size 524288 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13400 had 1 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2269 < min_flush_size 524288 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2269 < min_flush_size 524288 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x140000 mtime 2019-12-07T19:34:17.812271+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x13fd32~8dd to file(ino 16 size 0x140000 mtime 2019-12-07T19:34:17.812271+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x13fd32 0x13fd32~8dd to file(ino 16 size 0x140000 mtime 2019-12-07T19:34:17.812271+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _allocate len 0x60f from 0 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _allocate len 0x60f from 1 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd270000 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd270000~10000/10000,10000,acd270000 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x150000 to include allocated 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 42 (was clean) 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x10fd32 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd32 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x60f and padding block with 0x9f1 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x150000 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14060f 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.814+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.814+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (42) on file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]), flushing log 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.816+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 42 len 0x49 crc 0xe0a78483) 2019-12-07T19:34:17.816+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x29000~1000 to file(ino 1 size 0x29000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.816+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x29000 0x29000~1000 to file(ino 1 size 0x29000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x2a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x29000 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x2a000 2019-12-07T19:34:17.816+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:17.816+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:17.816+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000039 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 42 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002794795s (0.000038115s flush + 0.002756680s kv commit) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.816+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9800> 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.816+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 kv_submitted 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9800 2019-12-07T19:34:17.816+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 finishing 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9800 onodes 0x558c54e13400 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396ef8 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb3b0 space 0x558c57396ef8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0788 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb320 space 0x558c574e0788 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881748 in 0x558c53a595e0) _finish_write discard buffer(0x558c57669b00 space 0x558c57881748 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de7f8 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb290 space 0x558c578de7f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de9b8 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb200 space 0x558c578de9b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dea98 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb170 space 0x558c578dea98 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c088 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb0e0 space 0x558c5794c088 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c788 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb050 space 0x558c5794c788 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42be8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57669a70 space 0x558c57c42be8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5c18 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb440 space 0x558c57db5c18 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5cf8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57ecfcb0 space 0x558c57db5cf8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5dd8 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb4d0 space 0x558c57db5dd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5eb8 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fb560 space 0x558c57db5eb8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c718 in 0x558c53a595e0) _finish_write discard buffer(0x558c57669b90 space 0x558c5800c718 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c7f8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57669c20 space 0x558c5800c7f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c8d8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57669cb0 space 0x558c5800c8d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'5 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 0'0 active+remapped m=4 mbc={} ps=[a~1]] recovery_committed_to version 21'5 now ondisk 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800c9b8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57669d40 space 0x558c5800c9b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800ca98 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fad80 space 0x558c5800ca98 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800cb78 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fae10 space 0x558c5800cb78 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580eca98 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fafc0 space 0x558c580eca98 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ecb78 in 0x558c53a595e0) _finish_write discard buffer(0x558c579faf30 space 0x558c580ecb78 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ecc58 in 0x558c53a595e0) _finish_write discard buffer(0x558c579faea0 space 0x558c580ecc58 0x0~2000 writing nocache) 2019-12-07T19:34:17.816+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.2s1 40/39 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 -- 0x558c57612400 con 0x558c54c5b400 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9800 done 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.816+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9800 [] 2019-12-07T19:34:17.816+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.816+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.816+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.817+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 168 ==== MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=70, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57e8e780 con 0x558c54c5b400 2019-12-07T19:34:17.817+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c581cc420 prio 3 cost 0 latency 0.000018 epoch 40 MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=70, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.818+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=70, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) 2019-12-07T19:34:17.818+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=70, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) queued 2019-12-07T19:34:17.818+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGOpItem(op=MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=70, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e40) pg 0x558c5731a000 2019-12-07T19:34:17.818+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc420 prio 3 cost 0 latency 0.000072 MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=70, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] 2019-12-07T19:34:17.818+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] _handle_message: MOSDECSubOpRead(1.2s4 40/39 ECSubRead(tid=70, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:17.818+0000 7f8f03980700 25 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:17.818+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 0x0~40000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~40000 size 0x2e000 (188416) 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa4d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8b60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54cf9730 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df5e0 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df6c0 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df7a0 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df880 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df8f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df960 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df9d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dfa40 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dfab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0000 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f00e0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f01c0 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f02a0 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0310 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0380 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f03f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0460 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f04d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0540 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f05b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0620 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0700 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0770 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f07e0 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0850 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f08c0 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0930 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f09a0 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0a10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0a80 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0b60 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0bd0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f0c40 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0cb0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54cf9730 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd650 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fa4d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8b60 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df5e0 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df6c0 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df730 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df7a0 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df810 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df880 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df8f0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df960 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df9d0 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578dfa40 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dfab0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0000 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0070 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f00e0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f01c0 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0230 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f02a0 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0310 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0380 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f03f0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0460 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f04d0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0540 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f05b0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0620 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0690 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0700 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0770 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f07e0 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0850 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f08c0 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0930 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f09a0 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0a10 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0a80 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0af0 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0b60 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0bd0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f0c40 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0cb0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54cf9730 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575bd668 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c54c9a120 space 0x558c575bd668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fa4d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8b60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c54cf8b78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fb710 space 0x558c54cf8b78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df5e0 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df668 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fb7a0 space 0x558c578df668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df6c0 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df748 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fb830 space 0x558c578df748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df7a0 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df828 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fb8c0 space 0x558c578df828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df880 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df8f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df908 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fb950 space 0x558c578df908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df960 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df9d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578df9e8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fb9e0 space 0x558c578df9e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578dfa40 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dfab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578dfac8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fba70 space 0x558c578dfac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0000 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0088 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbb00 space 0x558c578f0088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f00e0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0168 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbb90 space 0x558c578f0168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f01c0 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbc20 space 0x558c578f0248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f02a0 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0310 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0328 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbcb0 space 0x558c578f0328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0380 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f03f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0408 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbd40 space 0x558c578f0408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0460 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f04d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f04e8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbdd0 space 0x558c578f04e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0540 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f05b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f05c8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbe60 space 0x558c578f05c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0620 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f06a8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c579fbef0 space 0x558c578f06a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0700 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0770 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c57f4e000 space 0x558c578f0788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f07e0 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0850 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0868 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c57f4e090 space 0x558c578f0868 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f08c0 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0930 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0948 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c57f4e120 space 0x558c578f0948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f09a0 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0a10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0a28 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c57f4e1b0 space 0x558c578f0a28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0a80 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0b08 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c57f4e240 space 0x558c578f0b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0b60 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0bd0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0be8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c57f4e2d0 space 0x558c578f0be8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f0c40 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0cb0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:17.818+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0cc8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.818+0000 7f8f03980700 20 _add level 1 near 0 on buffer(0x558c57f4e360 space 0x558c578f0cc8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:17.818+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 0x0~40000 = 188416 2019-12-07T19:34:17.818+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_sub_read read request=262144 r=188416 len=188416 2019-12-07T19:34:17.818+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:17.818+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:17.818+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:17.818+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] get_hash_info: found on disk, size 188416 2019-12-07T19:34:17.818+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:34:17.818+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:34:17.818+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_sub_read: Checking hash of 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:17.818+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.2s1 40/39 ECSubReadReply(tid=70, attrs_read=0)) v2 -- 0x558c57d9a000 con 0x558c54c5b400 2019-12-07T19:34:17.818+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc420 finish 2019-12-07T19:34:17.818+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.820+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 169 ==== MOSDPGPush(1.2s0 40/39 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 189478+0+0 (unknown 0 0 0) 0x558c57612400 con 0x558c54c5b400 2019-12-07T19:34:17.820+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c581cc580 prio 127 cost 189416 latency 0.000031 epoch 40 MOSDPGPush(1.2s0 40/39 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.820+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPush(1.2s0 40/39 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 189416 e40) 2019-12-07T19:34:17.820+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.820+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPush(1.2s0 40/39 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 189416 e40) queued 2019-12-07T19:34:17.821+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.821+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPush(1.2s0 40/39 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 189416 e40) pg 0x558c57e88000 2019-12-07T19:34:17.821+0000 7f8f07988700 10 osd.7 40 dequeue_op 0x558c581cc580 prio 127 cost 189416 latency 0.000082 MOSDPGPush(1.2s0 40/39 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.2s0( v 29'15 lc 21'5 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=4 mbc={} ps=[a~1]] 2019-12-07T19:34:17.821+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'5 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=4 mbc={} ps=[a~1]] _handle_message: MOSDPGPush(1.2s0 40/39 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:17.821+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'5 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=4 mbc={} ps=[a~1]] on_local_recover: 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:17.821+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'5 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=4 mbc={} ps=[a~1]] got missing 1:41fe8a11:::smithi13140796-45:head v 21'6 2019-12-07T19:34:17.821+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 21'5 active+remapped m=3 mbc={} ps=[a~1]] last_complete now 21'9 log.complete_to 23'10 2019-12-07T19:34:17.821+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c5730a000 seq 6 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:41fe8a11:::smithi13140796-45:head# key 0x80800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:41fe8a11:::smithi13140796-45:head# key 0x80800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:17.821+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1205 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# = 0 2019-12-07T19:34:17.821+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2e000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5730a000 23 blobs 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:17.821+0000 7f8f07988700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2e000/1000,2e000,0 2019-12-07T19:34:17.821+0000 7f8f07988700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x93f000~2e000/1000,2e000,0 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x93f000~2e000] 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fce00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fce00 blob([0x93f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fce70 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fce00) get_ref 0x0~2000 Blob(0x558c579fce00 blob([0x93f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fce70 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fce00) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c579fce00 blob([0x93f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fce70 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fce88 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fcee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fcfc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fcee0 blob([0x941000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcfc0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fcee0) get_ref 0x0~2000 Blob(0x558c579fcee0 blob([0x941000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcfc0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fcee0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c579fcee0 blob([0x941000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fcfc0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fcfd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd030 blob([0x943000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd0a0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd030) get_ref 0x0~2000 Blob(0x558c579fd030 blob([0x943000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd0a0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd030) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c579fd030 blob([0x943000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd0a0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd0b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd110 blob([0x945000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd180 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd110) get_ref 0x0~2000 Blob(0x558c579fd110 blob([0x945000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd180 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd110) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c579fd110 blob([0x945000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd180 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd198 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd1f0 blob([0x947000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd260 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd1f0) get_ref 0x0~2000 Blob(0x558c579fd1f0 blob([0x947000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd260 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c579fd1f0 blob([0x947000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd260 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd278 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd2d0 blob([0x949000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd340 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd2d0) get_ref 0x0~2000 Blob(0x558c579fd2d0 blob([0x949000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd340 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c579fd2d0 blob([0x949000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd340 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd358 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd3b0 blob([0x94b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd420 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd3b0) get_ref 0x0~2000 Blob(0x558c579fd3b0 blob([0x94b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd420 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c579fd3b0 blob([0x94b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd420 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd438 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd490 blob([0x94d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd500 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd490) get_ref 0x0~2000 Blob(0x558c579fd490 blob([0x94d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd500 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd490) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c579fd490 blob([0x94d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd500 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd518 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd570 blob([0x94f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd5e0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd570) get_ref 0x0~2000 Blob(0x558c579fd570 blob([0x94f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd5e0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd570) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c579fd570 blob([0x94f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd5e0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd5f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd650 blob([0x951000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd6c0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd650) get_ref 0x0~2000 Blob(0x558c579fd650 blob([0x951000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd6c0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd650) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c579fd650 blob([0x951000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd6c0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd6d8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd730 blob([0x953000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd7a0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd730) get_ref 0x0~2000 Blob(0x558c579fd730 blob([0x953000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd7a0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd730) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c579fd730 blob([0x953000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd7a0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd7b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd810 blob([0x955000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd880 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd810) get_ref 0x0~2000 Blob(0x558c579fd810 blob([0x955000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd880 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd810) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c579fd810 blob([0x955000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd880 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd898 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fd960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd8f0 blob([0x957000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd960 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd8f0) get_ref 0x0~2000 Blob(0x558c579fd8f0 blob([0x957000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fd960 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c579fd8f0 blob([0x957000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fd960 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fd978 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fd9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fda40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fd9d0 blob([0x959000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fda40 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd9d0) get_ref 0x0~2000 Blob(0x558c579fd9d0 blob([0x959000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fda40 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fd9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c579fd9d0 blob([0x959000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fda40 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fda58 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fdab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fdb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fdab0 blob([0x95b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdb20 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdab0) get_ref 0x0~2000 Blob(0x558c579fdab0 blob([0x95b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdb20 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdab0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c579fdab0 blob([0x95b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fdb20 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fdb38 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fdb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fdc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fdb90 blob([0x95d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdc00 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdb90) get_ref 0x0~2000 Blob(0x558c579fdb90 blob([0x95d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdc00 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdb90) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c579fdb90 blob([0x95d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fdc00 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fdc18 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fdc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fdce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fdc70 blob([0x95f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdce0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdc70) get_ref 0x0~2000 Blob(0x558c579fdc70 blob([0x95f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdce0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdc70) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c579fdc70 blob([0x95f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fdce0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fdcf8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fdd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fddc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fdd50 blob([0x961000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fddc0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdd50) get_ref 0x0~2000 Blob(0x558c579fdd50 blob([0x961000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fddc0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdd50) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c579fdd50 blob([0x961000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fddc0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fddd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fde30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fdea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fde30 blob([0x963000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdea0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fde30) get_ref 0x0~2000 Blob(0x558c579fde30 blob([0x963000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdea0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fde30) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c579fde30 blob([0x963000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fdea0 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fdeb8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fdf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fdf80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fdf10 blob([0x965000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdf80 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdf10) get_ref 0x0~2000 Blob(0x558c579fdf10 blob([0x965000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fdf80 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c579fdf10) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c579fdf10 blob([0x965000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fdf80 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c579fdf98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f50000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f50070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f50000 blob([0x967000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50070 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c57f50000) get_ref 0x0~2000 Blob(0x558c57f50000 blob([0x967000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50070 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c57f50000) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57f50000 blob([0x967000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f50070 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57f50088 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f500e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f50150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f500e0 blob([0x969000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50150 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c57f500e0) get_ref 0x0~2000 Blob(0x558c57f500e0 blob([0x969000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50150 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c57f500e0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57f500e0 blob([0x969000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f50150 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57f50168 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f501c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f50230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f501c0 blob([0x96b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50230 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c57f501c0) get_ref 0x0~2000 Blob(0x558c57f501c0 blob([0x96b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50230 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.blob(0x558c57f501c0) get_ref init 0x2000, 1000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57f501c0 blob([0x96b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f50230 sbid 0x0)) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57f50248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2e000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.extentmap(0x558c58092d88) dirty_range mark inline shard dirty 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 = 0 2019-12-07T19:34:17.821+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 4 keys 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 4 keys = 0 2019-12-07T19:34:17.821+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000493'.missing/0000000000000001.28F71588.head.smithi13140796-45..' <- missing/0000000000000001.28F71588.head.smithi13140796-45.. 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.821+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a000 cost 286653 (24 ios * 4000 + 190653 bytes) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a000 onodes 0x558c58092c80 shared_blobs 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.extentmap(0x558c58092d88) update 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore.extentmap(0x558c58092d88) update inline shard 440 bytes from 23 extents 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:41fe8a11:::smithi13140796-45:head# is 931 (485 bytes onode + 2 bytes spanning blobs + 444 bytes inline extents) 2019-12-07T19:34:17.821+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a000 allocated 0x[93f000~2e000] released 0x[] 2019-12-07T19:34:17.821+0000 7f8f07988700 10 freelist allocate 0x93f000~2e000 2019-12-07T19:34:17.821+0000 7f8f07988700 20 freelist _xor first_key 0x900000 last_key 0x900000 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 prepare 2019-12-07T19:34:17.821+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5730a000 2019-12-07T19:34:17.821+0000 7f8f07988700 10 osd.7 40 dequeue_op 0x558c581cc580 finish 2019-12-07T19:34:17.821+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.821+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 21'5 active+remapped m=3 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.821+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.821+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 aio_wait 2019-12-07T19:34:17.821+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a000 2019-12-07T19:34:17.821+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 io_done 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:17.821+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2254 < min_flush_size 524288 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092c80 had 1 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:17.821+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2277 < min_flush_size 524288 2019-12-07T19:34:17.821+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2277 < min_flush_size 524288 2019-12-07T19:34:17.821+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.821+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14060f~8e5 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.821+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14060f 0x14060f~8e5 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11060f 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x60f 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.821+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xef4 and padding block with 0x10c 2019-12-07T19:34:17.822+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x140ef4 2019-12-07T19:34:17.822+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.822+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.822+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.822+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.822+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127957s (0.000037110s flush + 0.000090847s kv commit) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.822+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a000> 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.822+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 kv_submitted 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a000 2019-12-07T19:34:17.822+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 finishing 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a000 onodes 0x558c58092c80 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fce88 in 0x558c53a595e0) _finish_write discard buffer(0x558c579faa20 space 0x558c579fce88 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fcfd8 in 0x558c53a595e0) _finish_write discard buffer(0x558c579faab0 space 0x558c579fcfd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd0b8 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fab40 space 0x558c579fd0b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd198 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fabd0 space 0x558c579fd198 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd278 in 0x558c53a595e0) _finish_write discard buffer(0x558c579fac60 space 0x558c579fd278 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd358 in 0x558c53a595e0) _finish_write discard buffer(0x558c579facf0 space 0x558c579fd358 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd438 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4e990 space 0x558c579fd438 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd518 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4ea20 space 0x558c579fd518 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd5f8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4eab0 space 0x558c579fd5f8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd6d8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4eb40 space 0x558c579fd6d8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd7b8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4ebd0 space 0x558c579fd7b8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd898 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4ec60 space 0x558c579fd898 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fd978 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4ecf0 space 0x558c579fd978 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fda58 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4ed80 space 0x558c579fda58 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fdb38 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4ee10 space 0x558c579fdb38 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fdc18 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4eea0 space 0x558c579fdc18 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 21'5 active+remapped m=3 mbc={} ps=[a~1]] recovery_committed_to version 21'9 now ondisk 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fdcf8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4ef30 space 0x558c579fdcf8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fddd8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4efc0 space 0x558c579fddd8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fdeb8 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4f050 space 0x558c579fdeb8 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fdf98 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4f0e0 space 0x558c579fdf98 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f50088 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4f170 space 0x558c57f50088 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f50168 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4f200 space 0x558c57f50168 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f50248 in 0x558c53a595e0) _finish_write discard buffer(0x558c57f4f290 space 0x558c57f50248 0x0~2000 writing nocache) 2019-12-07T19:34:17.822+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.2s1 40/39 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 -- 0x558c5501d000 con 0x558c54c5b400 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 done 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.822+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.822+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a000 [] 2019-12-07T19:34:17.822+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.822+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.823+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 170 ==== MOSDPGRecoveryDelete(1.2s0 e40,39 1:4c9e444c:::smithi13140796-48:head,23'10) v2 ==== 123+0+0 (unknown 0 0 0) 0x558c5501d000 con 0x558c54c5b400 2019-12-07T19:34:17.823+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c581cc6e0 prio 3 cost 1000 latency 0.000013 epoch 40 MOSDPGRecoveryDelete(1.2s0 e40,39 1:4c9e444c:::smithi13140796-48:head,23'10) v2 2019-12-07T19:34:17.823+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 1000 e40) pg 0x558c57e88000 2019-12-07T19:34:17.823+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc6e0 prio 3 cost 1000 latency 0.000041 MOSDPGRecoveryDelete(1.2s0 e40,39 1:4c9e444c:::smithi13140796-48:head,23'10) v2 pg pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=3 mbc={} ps=[a~1]] 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=3 mbc={} ps=[a~1]] handle_recovery_delete 0x558c581cc6e0 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=3 mbc={} ps=[a~1]] remove_missing_object 1:4c9e444c:::smithi13140796-48:head 23'10 2019-12-07T19:34:17.823+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.823+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c5730a600 seq 7 2019-12-07T19:34:17.823+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.823+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:17.823+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:17.823+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a600 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.823+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a600 onodes shared_blobs 2019-12-07T19:34:17.823+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a600 allocated 0x[] released 0x[] 2019-12-07T19:34:17.823+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 prepare 2019-12-07T19:34:17.823+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a600 2019-12-07T19:34:17.823+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 io_done 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.823+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc6e0 finish 2019-12-07T19:34:17.823+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x140ef4~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x140ef4 0x140ef4~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x110ef4 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xef4 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf22 and padding block with 0xde 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x140f22 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000103645s (0.000000110s flush + 0.000103535s kv commit) 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a600> 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 kv_submitted 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a600 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 finishing 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a600 onodes 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a600 done 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a600 [] 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.823+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=3 mbc={} ps=[a~1]] on_local_recover: 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:17.823+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 21'9 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=3 mbc={} ps=[a~1]] got missing 1:4c9e444c:::smithi13140796-48:head v 23'10 2019-12-07T19:34:17.823+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 21'9 active+remapped m=2 mbc={} ps=[a~1]] last_complete now 29'13 log.complete_to 29'14 2019-12-07T19:34:17.823+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.823+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c5740bb00 seq 8 2019-12-07T19:34:17.823+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000493'.missing/0000000000000001.23972223.head.smithi13140796-48..' <- missing/0000000000000001.23972223.head.smithi13140796-48.. 2019-12-07T19:34:17.823+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.823+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.823+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.823+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:17.823+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:17.823+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:17.823+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:17.823+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 21'9 active+remapped m=2 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.823+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x140f22~4b8 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x140f22 0x140f22~4b8 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x110f22 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf22 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3da and padding block with 0xc26 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1413da 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.823+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112278s (0.000000068s flush + 0.000112210s kv commit) 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.823+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740bb00> 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:34:17.823+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.823+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.823+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.823+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 21'9 active+remapped m=2 mbc={} ps=[a~1]] recovery_committed_to version 29'13 now ondisk 2019-12-07T19:34:17.823+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGRecoveryDeleteReply(1.2s1 e40,39 1:4c9e444c:::smithi13140796-48:head,23'10) v2 -- 0x558c5763de00 con 0x558c54c5b400 2019-12-07T19:34:17.823+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.824+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 171 ==== MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:4,29'14) v2 ==== 123+0+0 (unknown 0 0 0) 0x558c5763de00 con 0x558c54c5b400 2019-12-07T19:34:17.824+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c581cc840 prio 3 cost 1000 latency 0.000012 epoch 40 MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:4,29'14) v2 2019-12-07T19:34:17.824+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:4,29'14) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:4,29'14) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:4,29'14) v2) prio 3 cost 1000 e40) pg 0x558c57e88000 2019-12-07T19:34:17.824+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc840 prio 3 cost 1000 latency 0.000035 MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:4,29'14) v2 pg pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=2 mbc={} ps=[a~1]] 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=2 mbc={} ps=[a~1]] handle_recovery_delete 0x558c581cc840 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=2 mbc={} ps=[a~1]] remove_missing_object 1:41df217a:::smithi13140796-39:4 29'14 2019-12-07T19:34:17.824+0000 7f8f03980700 20 snap_mapper.remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:17.824+0000 7f8f03980700 20 snap_mapper._remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:17.824+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.824+0000 7f8f03980700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.824+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.824+0000 7f8f03980700 20 snap_mapper.get_snaps 1:41df217a:::smithi13140796-39:4 got.empty() 2019-12-07T19:34:17.824+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.824+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c574fe300 seq 9 2019-12-07T19:34:17.824+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:41df217a:::smithi13140796-39:4# key 0x80800000000000000141df21'z!smithi13140796-39!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:17.824+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:17.824+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:17.824+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe300 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.824+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe300 onodes shared_blobs 2019-12-07T19:34:17.824+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe300 allocated 0x[] released 0x[] 2019-12-07T19:34:17.824+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 prepare 2019-12-07T19:34:17.824+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe300 2019-12-07T19:34:17.824+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 io_done 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.824+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.824+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc840 finish 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1413da~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1413da 0x1413da~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x1113da 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3da 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x408 and padding block with 0xbf8 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x141408 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000109518s (0.000000071s flush + 0.000109447s kv commit) 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe300> 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 kv_submitted 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe300 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 finishing 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe300 onodes 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe300 done 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe300 [] 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.824+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=2 mbc={} ps=[a~1]] on_local_recover: 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:17.824+0000 7f8f07988700 20 snap_mapper.remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:17.824+0000 7f8f07988700 20 snap_mapper._remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:17.824+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:17.824+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:17.824+0000 7f8f07988700 20 snap_mapper.get_snaps 1:41df217a:::smithi13140796-39:4 got.empty() 2019-12-07T19:34:17.824+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'13 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=2 mbc={} ps=[a~1]] got missing 1:41df217a:::smithi13140796-39:4 v 29'14 2019-12-07T19:34:17.824+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 29'13 active+remapped m=1 mbc={} ps=[a~1]] last_complete now 29'14 log.complete_to 29'15 2019-12-07T19:34:17.824+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.824+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c5740b500 seq 10 2019-12-07T19:34:17.824+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000493'.missing/0000000000000001.28BF48E5.4.smithi13140796-39..' <- missing/0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:17.824+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.824+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.824+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.824+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 5406 (1 ios * 4000 + 1406 bytes) 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes shared_blobs 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[] released 0x[] 2019-12-07T19:34:17.824+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:17.824+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:17.824+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1182 < min_flush_size 524288 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.824+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 29'13 active+remapped m=1 mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1205 < min_flush_size 524288 2019-12-07T19:34:17.824+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x141408~4b5 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x141408 0x141408~4b5 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x111408 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x408 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8bd and padding block with 0x743 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1418bd 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.824+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000102657s (0.000000040s flush + 0.000102617s kv commit) 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.824+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:17.824+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.824+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.824+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.824+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 29'13 active+remapped m=1 mbc={} ps=[a~1]] recovery_committed_to version 29'14 now ondisk 2019-12-07T19:34:17.825+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGRecoveryDeleteReply(1.2s1 e40,39 1:41df217a:::smithi13140796-39:4,29'14) v2 -- 0x558c5501d000 con 0x558c54c5b400 2019-12-07T19:34:17.825+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.825+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 172 ==== MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:head,29'15) v2 ==== 123+0+0 (unknown 0 0 0) 0x558c5501d000 con 0x558c54c5b400 2019-12-07T19:34:17.825+0000 7f8f24457700 15 osd.7 40 enqueue_op 0x558c581cc9a0 prio 3 cost 1000 latency 0.000011 epoch 40 MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:head,29'15) v2 2019-12-07T19:34:17.825+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:head,29'15) v2) prio 3 cost 1000 e40) 2019-12-07T19:34:17.825+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.825+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:head,29'15) v2) prio 3 cost 1000 e40) queued 2019-12-07T19:34:17.825+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.825+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:head,29'15) v2) prio 3 cost 1000 e40) pg 0x558c57e88000 2019-12-07T19:34:17.825+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc9a0 prio 3 cost 1000 latency 0.000033 MOSDPGRecoveryDelete(1.2s0 e40,39 1:41df217a:::smithi13140796-39:head,29'15) v2 pg pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=1 mbc={} ps=[a~1]] 2019-12-07T19:34:17.825+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=1 mbc={} ps=[a~1]] handle_recovery_delete 0x558c581cc9a0 2019-12-07T19:34:17.825+0000 7f8f03980700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=1 mbc={} ps=[a~1]] remove_missing_object 1:41df217a:::smithi13140796-39:head 29'15 2019-12-07T19:34:17.825+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.825+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c574ff800 seq 11 2019-12-07T19:34:17.825+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:41df217a:::smithi13140796-39:head# key 0x80800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:17.826+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:17.826+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_add_transaction op 13 got ENOENT on 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:17.826+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff800 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:17.826+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff800 onodes shared_blobs 2019-12-07T19:34:17.826+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff800 allocated 0x[] released 0x[] 2019-12-07T19:34:17.826+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 prepare 2019-12-07T19:34:17.826+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff800 2019-12-07T19:34:17.826+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 io_done 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.826+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.826+0000 7f8f03980700 10 osd.7 40 dequeue_op 0x558c581cc9a0 finish 2019-12-07T19:34:17.826+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1418bd~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1418bd 0x1418bd~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x1118bd 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8bd 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8eb and padding block with 0x715 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1418eb 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000110192s (0.000000099s flush + 0.000110093s kv commit) 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff800> 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff800 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 finishing 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff800 onodes 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 done 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff800 [] 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.826+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.826+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=1 mbc={} ps=[a~1]] on_local_recover: 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:17.826+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 lc 29'14 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped m=1 mbc={} ps=[a~1]] got missing 1:41df217a:::smithi13140796-39:head v 29'15 2019-12-07T19:34:17.826+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 29'14 active+remapped mbc={} ps=[a~1]] last_complete now 29'15 log.complete_to at end 2019-12-07T19:34:17.826+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:17.826+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c54d59200 seq 12 2019-12-07T19:34:17.826+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x0000000000000493'.missing/0000000000000001.28BF48E5.head.smithi13140796-39..' <- missing/0000000000000001.28BF48E5.head.smithi13140796-39.. 2019-12-07T19:34:17.826+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.826+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:17.826+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:17.826+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:17.826+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:17.826+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:17.826+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:17.826+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:17.826+0000 7f8f07988700 20 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 29'14 active+remapped mbc={} ps=[a~1]] _applied_recovered_object_replica 2019-12-07T19:34:17.826+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1418eb~4b8 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1418eb 0x1418eb~4b8 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x1118eb 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8eb 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xda3 and padding block with 0x25d 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x141da3 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:17.826+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000114254s (0.000000086s flush + 0.000114168s kv commit) 2019-12-07T19:34:17.826+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:17.826+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.826+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:17.826+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:17.826+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 lcod 29'14 active+remapped mbc={} ps=[a~1]] recovery_committed_to version 29'15 now ondisk 2019-12-07T19:34:17.826+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_trim(1.2s1 29'15 e40/40) v2 -- 0x558c575cdc20 con 0x558c54c5b400 2019-12-07T19:34:17.826+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGRecoveryDeleteReply(1.2s1 e40,39 1:41df217a:::smithi13140796-39:head,29'15) v2 -- 0x558c5763de00 con 0x558c54c5b400 2019-12-07T19:34:17.826+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.827+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 173 ==== MRecoveryReserve(1.2s0 RELEASE e40/40) v3 ==== 30+0+0 (unknown 0 0 0) 0x558c575cdc20 con 0x558c54c5b400 2019-12-07T19:34:17.827+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.2s0 epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.827+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) 2019-12-07T19:34:17.827+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:17.827+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) queued 2019-12-07T19:34:17.827+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:17.827+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 40 epoch_requested: 40 RecoveryDone) prio 255 cost 10 e40) pg 0x558c57e88000 2019-12-07T19:34:17.827+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] do_peering_event: epoch_sent: 40 epoch_requested: 40 RecoveryDone 2019-12-07T19:34:17.827+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:17.828405+0000 1 0.000035 2019-12-07T19:34:17.827+0000 7f8f03980700 5 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:17.827+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:17.827+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:17.854+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.904+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:17.954+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.005+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.054+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.104+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.155+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.205+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.255+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.305+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.355+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.398+0000 7f8f21451700 10 osd.7 40 tick 2019-12-07T19:34:18.398+0000 7f8f21451700 10 osd.7 40 do_waiters -- start 2019-12-07T19:34:18.398+0000 7f8f21451700 10 osd.7 40 do_waiters -- finish 2019-12-07T19:34:18.398+0000 7f8f21451700 20 osd.7 40 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:18.405+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.455+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.461+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 8 ==== osd_ping(ping e40 up_from 13 ping_stamp 2019-12-07T19:34:18.462075+0000/41.911301999s send_stamp 41.911301999s delta_ub 0.014031587s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c54c65400 2019-12-07T19:34:18.461+0000 7f8f24c58700 20 osd.7 40 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013574587s,0.014031587s]) 2019-12-07T19:34:18.461+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e40 up_from 13 ping_stamp 2019-12-07T19:34:18.462075+0000/41.911301999s send_stamp 41.897727412s delta_ub 18446744073.695977029s) v5 -- 0x558c53a5e800 con 0x558c54c65400 2019-12-07T19:34:18.461+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 8 ==== osd_ping(ping e40 up_from 13 ping_stamp 2019-12-07T19:34:18.462075+0000/41.911301999s send_stamp 41.911301999s delta_ub 0.014031587s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57519a00 con 0x558c54c65000 2019-12-07T19:34:18.461+0000 7f8f24c58700 20 osd.7 40 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013502867s,0.014031587s]) 2019-12-07T19:34:18.461+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e40 up_from 13 ping_stamp 2019-12-07T19:34:18.462075+0000/41.911301999s send_stamp 41.897799132s delta_ub 18446744073.696048749s) v5 -- 0x558c53a5e800 con 0x558c54c65000 2019-12-07T19:34:18.505+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.555+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1026046 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 25 ==== osd_map(41..41 src has 1..41) v4 ==== 974+0+0 (crc 0 0 0) 0x558c575d4b40 con 0x558c5476d000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 osd.7 40 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 osd.7 40 do_waiters -- start 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 osd.7 40 do_waiters -- finish 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 osd.7 40 _dispatch 0x558c575d4b40 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.604+0000 7f8f164bb700 3 osd.7 40 handle_osd_map epochs [41,41], i have 40, src has [1,41] 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 osd.7 40 handle_osd_map got inc map for epoch 41 2019-12-07T19:34:18.604+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3508e826:::osdmap.40:0# 0x0~0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1681 size 0x1681 (5761) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5523a930 blob([0x5f3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c577f3dc0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efc4d0 blob([!~1000,0x5f4000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,681]) SharedBlob(0x558c57efc540 sbid 0x0)) need 0x1000~681 cache has 0x[1000~681] 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3508e826:::osdmap.40:0# 0x0~1681 = 5761 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 osd.7 40 add_map_bl 40 5761 bytes 2019-12-07T19:34:18.604+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 72 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c19c00 2019-12-07T19:34:18.604+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.604+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 osd.7 40 got_full_map 41, nothing requested 2019-12-07T19:34:18.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.604+0000 7f8f06986700 20 osd.7 op_wq _add_slot_waiter 1.4s0 peering, item epoch is 41, will wait on OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 osd.7 40 handle_osd_map pg_num_history pg_num_history(e41 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {41={}} 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 osd.7 40 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e41 [1,41] lci=[13,41]) 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54d58f00 seq 32 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c6d7972f:::inc_osdmap.41:0# key 0x7f7fffffffffffffffc6d797'/!inc_osdmap.41!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c6d7972f:::inc_osdmap.41:0# 0x0~3a2 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1206 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c6d7972f:::inc_osdmap.41:0# 0x0~3a2 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~3a2 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc5e on front/back, now 0x0~1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58f00 1 blobs 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x96d000~1000/1000,1000,0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x96d000~1000] 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df83f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efc690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df83f0 blob([0x96d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc690 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c54df83f0) get_ref 0x0~3a2 Blob(0x558c54df83f0 blob([0x96d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc690 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c54df83f0) get_ref init 0x1000, 1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~3a2: 0x0~3a2 Blob(0x558c54df83f0 blob([0x96d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3a2) SharedBlob(0x558c57efc690 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efc6a8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3a2 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efab08) dirty_range mark inline shard dirty 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c6d7972f:::inc_osdmap.41:0# 0x0~3a2 = 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3e08e826:::osdmap.41:0# key 0x7f7fffffffffffffff3e08e8'&!osdmap.41!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3e08e826:::osdmap.41:0# 0x0~1464 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1207 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3e08e826:::osdmap.41:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58f00 2 blobs 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x96e000~2000/1000,2000,0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x96e000~2000] 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efc770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efc7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efc770 blob([0x96e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc7e0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc770) get_ref 0x0~1000 Blob(0x558c57efc770 blob([0x96e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc7e0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc770) get_ref init 0x1000, 1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57efc770 blob([0x96e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57efc7e0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efc7f8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efc850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efc8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efc850 blob([!~1000,0x96f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc8c0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc850) get_ref 0x1000~464 Blob(0x558c57efc850 blob([!~1000,0x96f000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57efc8c0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c57efc850) get_ref init 0x2000, 1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c57efc850 blob([!~1000,0x96f000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57efc8c0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efc8d8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c608) dirty_range mark inline shard dirty 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3e08e826:::osdmap.41:0# 0x0~1464 = 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:18.604+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 73 ==== osd_map(41..41 src has 1..41) v4 ==== 974+0+0 (unknown 0 0 0) 0x558c579458c0 con 0x558c54c19c00 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:18.604+0000 7f8f1199c700 20 osd.7 40 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58f00 0 blobs 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58f00 0 blobs 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 23631 (4 ios * 4000 + 7631 bytes) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes 0x558c53a91900,0x558c55276c80,0x558c5738c500,0x558c57efaa00 shared_blobs 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c608) update #-1:3e08e826:::osdmap.41:0# 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c608) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3e08e826:::osdmap.41:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efab08) update #-1:c6d7972f:::inc_osdmap.41:0# 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57efab08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c6d7972f:::inc_osdmap.41:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[96d000~3000] released 0x[] 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 freelist allocate 0x96d000~3000 2019-12-07T19:34:18.604+0000 7f8f164bb700 20 freelist _xor first_key 0x900000 last_key 0x900000 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:34:18.604+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58f00 2019-12-07T19:34:18.605+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 74 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c19c00 2019-12-07T19:34:18.605+0000 7f8f24457700 15 osd.7 40 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.605+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 aio_wait 2019-12-07T19:34:18.605+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:34:18.605+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:34:18.605+0000 7f8f1199c700 10 osd.7 40 do_waiters -- start 2019-12-07T19:34:18.605+0000 7f8f1199c700 10 osd.7 40 do_waiters -- finish 2019-12-07T19:34:18.605+0000 7f8f1199c700 20 osd.7 40 _dispatch 0x558c579458c0 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.605+0000 7f8f1199c700 3 osd.7 40 handle_osd_map epochs [41,41], i have 41, src has [1,41] 2019-12-07T19:34:18.605+0000 7f8f1199c700 10 osd.7 40 no new maps here, dropping 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738c500 had 1 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57efaa00 had 1 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x141da3~226c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x141da3 0x141da3~226c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x111da3 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xda3 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf and padding block with 0xff1 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14400f 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000177967s (0.000047291s flush + 0.000130676s kv commit) 2019-12-07T19:34:18.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58f00> 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54d58f00 osr 0x558c54ea4b40 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) _discard drop 29 0x1b000~1000 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7 40 _committed_osd_maps 41..41 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7 40 advance to epoch 41 (<= last 41 <= newest_map 41) 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) prepare_write seq 30 0x1b000~1000 crc 41a1cd18 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) _discard drop 29 0x2000~1000 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57ee1cc0) prepare_write seq 30 0x2000~1000 crc 2fe41629 2019-12-07T19:34:18.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.605+0000 7f8f141a1700 7 osd.7 41 consume_map version 41 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.8s5 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.0s3 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.1s1 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.9s4 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.605+0000 7f8f07988700 20 osd.7 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 41, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.as5 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.2s0 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.2s4 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.bs3 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.bs1 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.3s4 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.cs0 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.4s0 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.5s4 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.ds2 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.es4 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.6s2 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.fs5 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 identify_splits_and_merges 1.7s4 e40 to e41 pg_nums {18=16} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 41 prune_pg_created 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:0.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:1.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:2.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 pending_peering first epoch 41 <= 41, requeueing 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {41=0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)>} 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:3.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:4.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 pending_peering first epoch 41 <= 41, requeueing 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:4._wake_pg_slot _wake_pg_slot 1.4s0 to_process <> waiting <> waiting_peering {41=0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)>} 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:5.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:6.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7:7.consume_map 41 (was 40) 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 40 -> 41 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 40 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 40 -> 41 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 40 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,3,1,5,2,4] 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] state: Active advmap interval change, fast return 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,3,1,5,2,4] 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [7,4,1,5,2,3] 2019-12-07T19:34:18.605+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] exit Started/Primary/Active/Recovered 2019-12-07T19:34:18.606858+0000 1 0.000320 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.605+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] exit Started/Primary/Active 2019-12-07T19:34:18.606866+0000 0 0.000000 2019-12-07T19:34:18.605+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.606869+0000 1 0.000026 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] agent_stop 2019-12-07T19:34:18.605+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.606877+0000 0 0.000000 2019-12-07T19:34:18.605+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.606887+0000 0 0.000000 2019-12-07T19:34:18.605+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.605+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} trimq=[d~1] ps=[a~1]] exit Started/Primary 2019-12-07T19:34:18.606886+0000 0 0.000000 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] agent_stop 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.605+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] exit Started 2019-12-07T19:34:18.606919+0000 0 0.000000 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [7,4,1,5,2,3] 2019-12-07T19:34:18.605+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] enter Reset 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.605+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.605+0000 7f8f03980700 15 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 mlcod 0'0 active+remapped mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=39 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [7,4,1,5,2,3](7) acting [2147483647,4,1,5,2147483647,3](4)) : acting set is too small 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.605+0000 7f8f141a1700 10 osd.7 41 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.8s5 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.0s3 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.1s1 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.9s4 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] new interval newup [7,3,1,5,2,4] newacting [7,3,1,5,2,4] 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.as5 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.2s4 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]/[7,2147483647,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.bs3 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 41 clear_ready_to_merge 1.4s0 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.bs1 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.3s4 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f0297e700 15 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,39)/1 crt=37'4 lcod 21'2 mlcod 0'0 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] clear_stats 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.cs0 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.5s4 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.ds2 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.es4 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [7,3,1,5,2,4](7) acting [7,2147483647,1,5,2,4](7)) up_thru 39 up_from 13 last_epoch_clean 19 interval(39-40 up [7,3,1,5,2,4](7) acting [7,2147483647,1,5,2,4](7) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.6s2 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.fs5 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 15 osd.7 41 enqueue_peering_evt 1.7s4 epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.605+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) 2019-12-07T19:34:18.605+0000 7f8f141a1700 7 osd.7 41 activate_map version 41 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] noting past ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0))) 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c5736a000 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] init_hb_stamps now [0x558c54ea3ce0,0x558c5523a230,0x558c54677f10,0x558c5523a150,0x558c5523a3f0] 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active pruub 57.054076579s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] on_new_interval prior_readable_until_ub 57.054076579s (mnow 42.042574774s + 15.011501805s) 2019-12-07T19:34:18.605+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.605+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 40 -> 41 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active pruub 57.054076579s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.605+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active pruub 57.054076579s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.605+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.605+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.605+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.605+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c57522000 2019-12-07T19:34:18.605+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] new interval newup [3,7,2,4,5,1] newacting [3,7,2,4,5,1] 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active pruub 57.054076579s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:34:56.763457+0000, must = 0 2019-12-07T19:34:18.605+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.605+0000 7f8f0297e700 1 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active pruub 57.054076579s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] start_peering_interval up [7,3,1,5,2,4] -> [7,3,1,5,2,4], acting [7,2147483647,1,5,2,4] -> [7,3,1,5,2,4], acting_primary 7(0) -> 7, up_primary 7(0) -> 7, role 0 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.605+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.607185+0000 1 0.000026 2019-12-07T19:34:18.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) async=[3(1)] r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.605+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 40 -> 41 2019-12-07T19:34:18.606+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.607200+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c57e8b000 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:18.606+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.607211+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:18.606+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] new interval newup [2,1,3,4,7,5] newacting [2,1,3,4,7,5] 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 40 -> 41 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.607247+0000 1 0.000031 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] new interval newup [3,7,2,4,5,1] newacting [3,7,2,4,5,1] 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.607258+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] new interval newup [1,4,2,5,3,7] newacting [1,4,2,5,3,7] 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.607269+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]/[3,2147483647,2,4,5,1]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029282 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.607279+0000 1 0.000022 2019-12-07T19:34:18.606+0000 7f8f08189700 15 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.606+0000 7f8f0297e700 15 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] publish_stats_to_osd 40:76 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.607289+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.607297+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [3,7,2,4,5,1](3) acting [3,2147483647,2,4,5,1](3)) up_thru 39 up_from 13 last_epoch_clean 29 interval(39-40 up [3,7,2,4,5,1](3) acting [3,2147483647,2,4,5,1](3) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([39,40] acting 1(5),2(2),3(0),4(3),5(4))) 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] new interval newup [2,1,3,4,7,5] newacting [2,1,3,4,7,5] 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]/[2,1,3,4,2147483647,5]p2(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c54ea3ce0] 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] [7,2147483647,1,5,2,4] -> [7,3,1,5,2,4], replicas changed 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c57380000 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.042782623s + 0s) 2019-12-07T19:34:18.606+0000 7f8f06185700 15 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]p2(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] new interval newup [1,4,2,5,3,7] newacting [1,4,2,5,3,7] 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.606+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]/[1,4,2,2147483647,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 40 -> 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f08189700 1 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] start_peering_interval up [3,7,2,4,5,1] -> [3,7,2,4,5,1], acting [3,2147483647,2,4,5,1] -> [3,7,2,4,5,1], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role -1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [2,1,3,4,7,5](2) acting [2,1,3,4,2147483647,5](2)) up_thru 39 up_from 13 last_epoch_clean 29 interval(39-40 up [2,1,3,4,7,5](2) acting [2,1,3,4,2147483647,5](2) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.606+0000 7f8f07988700 15 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([39,40] acting 1(1),2(0),3(2),4(3),5(5))) 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [1,4,2,5,3,7](1) acting [1,4,2,2147483647,3,2147483647](1)) : acting set is too small 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] release_backoffs [1:80000000::::0,1:90000000::::head) 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] new interval newup [1,4,5,2,3,7] newacting [1,4,5,2,3,7] 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4))) 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a150] 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.606+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.607405+0000 1 0.000526 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.042848057s + 0s) 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:18.606+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.042865731s + 0s) 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.606+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f06185700 1 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] start_peering_interval up [2,1,3,4,7,5] -> [2,1,3,4,7,5], acting [2,1,3,4,2147483647,5] -> [2,1,3,4,7,5], acting_primary 2(0) -> 2, up_primary 2(0) -> 2, role -1 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] state: transitioning to Primary 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.607442+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07988700 1 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] start_peering_interval up [1,4,2,5,3,7] -> [1,4,2,5,3,7], acting [1,4,2,2147483647,3,2147483647] -> [1,4,2,5,3,7], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role -1 -> 5, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] release_backoffs [1:a0000000::::0,1:b0000000::::head) 2019-12-07T19:34:18.606+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] enter Started/Primary 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown pruub 57.054076579s@ mbc={} ps=[a~1]] enter Started/Primary/Peering 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] release_backoffs [1:50000000::::0,1:60000000::::head) 2019-12-07T19:34:18.606+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.606+0000 7f8f0297e700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@ mbc={} ps=[a~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.607404+0000 1 0.000017 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@ mbc={} ps=[a~1]] build_prior all_probe 0(1),1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@ mbc={} ps=[a~1]] build_prior maybe_rw interval:39, acting: 1(2),2(4),4(5),5(3),7(0) 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c54d9b000 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.607494+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@ mbc={} ps=[a~1]] build_prior final: probe 1(2),2(4),3(1),4(5),5(3),7(0) down 0 blocked_by {} 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.607506+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@ mbc={} ps=[a~1]] up_thru 40 < same_since 41, must notify monitor 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs3 40 -> 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@0 mbc={} ps=[a~1]] state: querying info from osd.1(2) 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:18.606+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@0 mbc={} ps=[a~1]] state: querying info from osd.2(4) 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@0 mbc={} ps=[a~1]] state: querying info from osd.3(1) 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,7,2,3,1,5] 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@0 mbc={} ps=[a~1]] state: querying info from osd.4(5) 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@0 mbc={} ps=[a~1]] state: querying info from osd.5(3) 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] new interval newup [1,4,5,2,3,7] newacting [1,4,5,2,3,7] 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.607560+0000 2 0.000041 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@0 mbc={} ps=[a~1]] check_prior_readable_down_osds prior_readable_down_osds osd.0 is dead as of epoch 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.607570+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering pruub 57.054076579s@ mbc={} ps=[a~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.607577+0000 1 0.000313 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.607580+0000 1 0.000284 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.607579+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f0898a700 15 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.606+0000 7f8f0297e700 15 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] publish_stats_to_osd 40:77 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f06185700 1 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.606+0000 7f8f07988700 1 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [1,4,5,2,3,7](1) acting [1,4,5,2,3,2147483647](1)) up_thru 39 up_from 13 last_epoch_clean 29 interval(39-40 up [1,4,5,2,3,7](1) acting [1,4,5,2,3,2147483647](1) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.607607+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.607607+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.606+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([39,40] acting 1(0),2(3),3(4),4(1),5(2))) 2019-12-07T19:34:18.606+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,7,2,3,1,5] 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.011501805s) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 2019-12-07T19:34:18.606+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:15.243818+0000, must = 0 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.043083291s + 0s) 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:18.606+0000 7f8f06185700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:18.606+0000 7f8f07187700 15 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c6d7972f:::inc_osdmap.41:0# 0x0~0 2019-12-07T19:34:18.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~3a2 size 0x3a2 (930) 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:18.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f0898a700 1 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] start_peering_interval up [1,4,5,2,3,7] -> [1,4,5,2,3,7], acting [1,4,5,2,3,2147483647] -> [1,4,5,2,3,7], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role -1 -> 5, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [4,7,2,3,1,5](4) acting [4,2147483647,2,2147483647,1,5](4)) : acting set is too small 2019-12-07T19:34:18.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c57383000 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54df83f0 blob([0x96d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3a2) SharedBlob(0x558c57efc690 sbid 0x0)) need 0x0~3a2 cache has 0x[0~3a2] 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c6d7972f:::inc_osdmap.41:0# 0x0~3a2 = 930 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 41 add_map_inc_bl 41 930 bytes 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c57676000 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] release_backoffs [1:10000000::::0,1:20000000::::head) 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(41..41 src has 1..41) v4 -- 0x558c5781a480 con 0x558c54c64000 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f07988700 10 osd.7 41 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 40 -> 41 (shared) 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_advance_map: 41 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 40 -> 41 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] init_hb_stamps now [] 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 40 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.043149806s + 0s) 2019-12-07T19:34:18.606+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:41 sent:41 1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4)))) e41/41) v1 -- 0x558c576dfc00 con 0x558c54c64000 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1]] state: Active advmap 2019-12-07T19:34:18.606+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54d59b00 seq 5 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.606+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] on_active_advmap new removed_snaps [10~1], snap_trimq now [10~1] 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:18.606+0000 7f8f07187700 1 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] start_peering_interval up [4,7,2,3,1,5] -> [4,7,2,3,1,5], acting [4,2147483647,2,2147483647,1,5] -> [4,7,2,3,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5458 (1 ios * 4000 + 1458 bytes) 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:18.606+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:18.606+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] handle_activate_map: 41 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] activate_map 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] state: Active: handling ActMap 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:18.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] Active: kicking snap trim 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.606+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14400f~52d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14400f 0x14400f~52d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.607808+0000 1 0.000387 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11400f 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.606+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x53c and padding block with 0xac4 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f0898a700 1 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.606+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming/WaitReservation 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.607836+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f0898a700 5 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.606+0000 7f8f06986700 15 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 39:122 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(41..41 src has 1..41) v4 -- 0x558c54d2efc0 con 0x558c54c64000 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.4s2 1.4s2 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c574a0780 con 0x558c54c64000 2019-12-07T19:34:18.606+0000 7f8f06986700 7 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] state: handle ActMap primary 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(41..41 src has 1..41) v4 -- 0x558c57570fc0 con 0x558c54c19800 2019-12-07T19:34:18.606+0000 7f8f06986700 15 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 40:123 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.607877+0000 1 0.000299 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 40 -> 41 (shared) 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 41 2019-12-07T19:34:18.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.606+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.8s0 1.8s0 (query:41 sent:41 1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([39,40] acting 1(0),2(3),3(4),4(1),5(2)))) e41/41) v1 -- 0x558c548a1c00 con 0x558c54c64000 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.4s4 1.4s4 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c5738ca00 con 0x558c54c19800 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c577fdb00 seq 18 2019-12-07T19:34:18.606+0000 7f8f07187700 1 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(41..41 src has 1..41) v4 -- 0x558c57570d80 con 0x558c54c5bc00 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.607906+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 40 -> 41 (shared) 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.4s1 1.4s1 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c58092780 con 0x558c54c5bc00 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:18.606+0000 7f8f0b990700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fdb00 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fdb00 onodes shared_blobs 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fdb00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.606+0000 7f8f0b990700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: NotTrimming: trimming 10 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(41..41 src has 1..41) v4 -- 0x558c54dd1200 con 0x558c54c5b400 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 40 -> 41 (shared) 2019-12-07T19:34:18.606+0000 7f8f0b990700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] exit Trimming/WaitReservation 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 prepare 2019-12-07T19:34:18.606+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fdb00 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 io_done 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.4s5 1.4s5 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c58092500 con 0x558c54c5b400 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:18.606+0000 7f8f0b990700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(41..41 src has 1..41) v4 -- 0x558c57e03b00 con 0x558c54c19c00 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f0b990700 15 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 41:124 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 40 -> 41 (shared) 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c5500d000 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.4s3 1.4s3 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c58093b80 con 0x558c54c19c00 2019-12-07T19:34:18.606+0000 7f8f0b990700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:18.606+0000 7f8f0918b700 20 osd.7 41 agent_entry empty queue 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c5740af00 seq 24 2019-12-07T19:34:18.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4))) 2019-12-07T19:34:18.606+0000 7f8f0b990700 10 osd.7 41 queueing pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[10~1] ps=[a~1,d~1]] for snaptrim 2019-12-07T19:34:18.606+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=41) prio 5 cost 1048576 e41) 2019-12-07T19:34:18.606+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:18.606+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:41 sent:41 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 3->0 ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e41/41) v1 -- 0x558c57391c00 con 0x558c54c5b400 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_advance_map: 41 2019-12-07T19:34:18.606+0000 7f8f0b990700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 40 -> 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:18.606+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:18.606+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c57ff1800 seq 23 2019-12-07T19:34:18.606+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:18.606+0000 7f8f0b990700 15 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 41:125 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 40 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 2019-12-07T19:34:18.606+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.606+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._biginfo' <- _biginfo 2019-12-07T19:34:18.606+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.606+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:18.606+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:18.606+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:18.606+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1800 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:18.606+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1800 onodes shared_blobs 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a3f0,0x558c5523a230,0x558c54677f10,0x558c5523a150,0x558c54ea3ce0] 2019-12-07T19:34:18.606+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1800 allocated 0x[] released 0x[] 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 queue_want_up_thru now 41 (was 39), currently 40 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 send_alive up_thru currently 40 want 41 2019-12-07T19:34:18.606+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 41 send_alive want 41 2019-12-07T19:34:18.606+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 prepare 2019-12-07T19:34:18.606+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1800 2019-12-07T19:34:18.606+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 io_done 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.043435133s + 0s) 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_alive(want up_thru 41 have 41) v1 -- 0x558c581e83c0 con 0x558c5476d000 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c575ef000 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=41) prio 5 cost 1048576 e41) queued 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=41) prio 5 cost 1048576 e41) pg 0x558c57676000 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] handle_activate_map: 41 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:05.196880+0000, must = 0 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] activate_map 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] snap_trimmer posting 2019-12-07T19:34:18.606+0000 7f8f0898a700 20 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 38 while current is 41 2019-12-07T19:34:18.606+0000 7f8f03980700 1 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 mlcod 0'0 active mbc={} ps=[a~1]] start_peering_interval up [7,4,1,5,2,3] -> [7,4,1,5,2,3], acting [2147483647,4,1,5,2147483647,3] -> [7,4,1,5,2,3], acting_primary 4(1) -> 7, up_primary 7(0) -> 7, role -1 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 40 -> 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 10 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 40 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,7,2,3,1,5] 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.608071+0000 1 0.000024 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630380) upper_bound after SNA_1_0000000000000010_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.0_0000000000000001.C' 2019-12-07T19:34:18.606+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.608080+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630380) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.608089+0000 0 0.000000 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000010_.0_0000000000000001.C) returns 0 SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239760 in raw 0x558c55239760 len 69 nref 1) ) 2019-12-07T19:34:18.606+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1]] SnapTrimmer state: adding snap 10 to purged_snaps 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~1]] SnapTrimmer state: purged_snaps now [a~1,d~1,10~1], snap_trimq now [] 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:18.606+0000 7f8f03980700 15 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] publish_stats_to_osd 0:0 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c5740a600 seq 35 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] new interval newup [4,7,2,3,1,5] newacting [4,7,2,3,1,5] 2019-12-07T19:34:18.606+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes shared_blobs 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:18.606+0000 7f8f07187700 15 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.606+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:18.606+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:18.606+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [4,7,2,3,1,5](4) acting [4,2147483647,2,2147483647,1,5](4)) : acting set is too small 2019-12-07T19:34:18.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~1]] share_pg_info 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.606+0000 7f8f06986700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.606+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e41/41) v1 -- 0x558c54e07100 con 0x558c54c64000 2019-12-07T19:34:18.606+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3))) 2019-12-07T19:34:18.606+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.606+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.607+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e41/41) v1 -- 0x558c57390e00 con 0x558c54c19800 2019-12-07T19:34:18.607+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.607+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f03980700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c5513e000 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.043650472s + 0s) 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.607+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e41/41) v1 -- 0x558c57392300 con 0x558c54c5bc00 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.607+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e41/41) v1 -- 0x558c57391500 con 0x558c54c5b400 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.607+0000 7f8f07187700 1 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] start_peering_interval up [4,7,2,3,1,5] -> [4,7,2,3,1,5], acting [4,2147483647,2,2147483647,1,5] -> [4,7,2,3,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 40 -> 41 2019-12-07T19:34:18.607+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.608235+0000 1 0.001347 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.607+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e41/41) v1 -- 0x558c576df500 con 0x558c54c19c00 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 40 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:18.607+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~1]] exit Trimming 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] new interval newup [4,1,2,5,7,3] newacting [4,1,2,5,7,3] 2019-12-07T19:34:18.607+0000 7f8f03980700 1 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] state: transitioning to Primary 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.607+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.608271+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.607+0000 7f8f0297e700 15 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] publish_stats_to_osd 41:126 2019-12-07T19:34:18.607+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Started/Primary 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.608277+0000 1 0.000028 2019-12-07T19:34:18.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] enter NotTrimming 2019-12-07T19:34:18.607+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Started/Primary/Peering 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.607+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.608289+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.607+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.608299+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] snap_trimmer complete 2019-12-07T19:34:18.607+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] build_prior all_probe 0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] build_prior final: probe 1(2),2(4),3(5),4(1),5(3),7(0),7(4) down 0 blocked_by {} 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] up_thru 40 < same_since 41, must notify monitor 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.607+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: querying info from osd.1(2) 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 40 -> 41 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: querying info from osd.2(4) 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] new interval newup [4,1,2,5,7,3] newacting [4,1,2,5,7,3] 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: querying info from osd.3(5) 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] new interval newup [3,2,5,4,1,7] newacting [3,2,5,4,1,7] 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: querying info from osd.4(1) 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]/[4,1,2,5,7,2147483647]p4(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: querying info from osd.5(3) 2019-12-07T19:34:18.607+0000 7f8f0317f700 15 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=37'13 active+remapped mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: querying info from osd.7(4) 2019-12-07T19:34:18.607+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.608373+0000 1 0.000280 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.608372+0000 1 0.000022 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] check_prior_readable_down_osds prior_readable_down_osds osd.0 is dead as of epoch 41 2019-12-07T19:34:18.607+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.608383+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:18.607+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.608392+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [4,1,2,5,7,3](4) acting [4,1,2,5,7,2147483647](4)) up_thru 39 up_from 13 last_epoch_clean 29 interval(39-40 up [4,1,2,5,7,3](4) acting [4,1,2,5,7,2147483647](4) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.607+0000 7f8f07187700 1 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.607+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.607+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.608401+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f03980700 15 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] publish_stats_to_osd 40:91 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([39,40] acting 1(1),2(2),4(0),5(3),7(4))) 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f07187700 5 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.607+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 75 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010614337s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57391c00 con 0x558c54c19c00 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.607+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a3f0] 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.607+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.607+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active pruub 57.054502621s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 57.054502621s (mnow 42.043872491s + 15.010630130s) 2019-12-07T19:34:18.607+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010614337s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] new interval newup [3,2,5,4,1,7] newacting [3,2,5,4,1,7] 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active pruub 57.054502621s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.607+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010614337s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.5(3) 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active pruub 57.054502621s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.607+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:41 sent:41 1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e41/41) v1 -- 0x558c57390000 con 0x558c54c5b400 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]/[3,2147483647,5,4,1,2147483647]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:23.243866+0000, must = 0 2019-12-07T19:34:18.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:18.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57ff1b00 seq 9 2019-12-07T19:34:18.607+0000 7f8f0317f700 1 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active pruub 57.054502621s@ mbc={} ps=[a~1]] start_peering_interval up [4,1,2,5,7,3] -> [4,1,2,5,7,3], acting [4,1,2,5,7,2147483647] -> [4,1,2,5,7,3], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.607+0000 7f8f05183700 15 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]p3(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.607+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:18.607+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.607+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:18.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [3,2,5,4,1,7](3) acting [3,2147483647,5,4,1,2147483647](3)) : acting set is too small 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:18.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:18.607+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c57e8e780 con 0x558c54c64000 2019-12-07T19:34:18.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1b00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:18.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1b00 onodes shared_blobs 2019-12-07T19:34:18.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 prepare 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2))) 2019-12-07T19:34:18.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1b00 2019-12-07T19:34:18.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 io_done 2019-12-07T19:34:18.607+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c57d9a280 con 0x558c54c19800 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.606+0000 7f8f08189700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.2s5 1.2s5 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c57d9a500 con 0x558c54c5bc00 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.2s1 1.2s1 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c57d9a780 con 0x558c54c5b400 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c54ea3ce0] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s3 1.2s3 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c57d9aa00 con 0x558c54c19c00 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.043956210s + 0s) 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 41 send_incremental_map 40 -> 41 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.608522+0000 1 0.000306 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.607+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(41..41 src has 1..41) v4 -- 0x558c57641b00 con 0x558c53ae5400 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f03980700 10 osd.7 41 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 40 -> 41 (shared) 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.607+0000 7f8f05183700 1 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] start_peering_interval up [3,2,5,4,1,7] -> [3,2,5,4,1,7], acting [3,2147483647,5,4,1,2147483647] -> [3,2,5,4,1,7], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role -1 -> 5, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.607+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 41) e41/41) v1 -- 0x558c57d9ac80 con 0x558c53ae5400 2019-12-07T19:34:18.607+0000 7f8f08189700 1 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.607+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.608548+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57ef9200 seq 13 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.607+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 0+0+0 (unknown 0 0 0) 0x558c57d9ac80 con 0x558c53ae5400 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] release_backoffs [1:f0000000::::0,MAX) 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.607+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:18.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:18.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010614337s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:18.607+0000 7f8f1119b700 15 osd.7 41 enqueue_peering_evt 1.2s4 epoch_sent: 41 epoch_requested: 41 MQuery 1.2s4 from 7(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010614337s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.2s4 from 7(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.607+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:18.607+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.607+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.607+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9200 cost 5471 (1 ios * 4000 + 1471 bytes) 2019-12-07T19:34:18.607+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9200 onodes shared_blobs 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(41..41 src has 1..41) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c57641b00 con 0x558c53ae5400 2019-12-07T19:34:18.607+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9200 allocated 0x[] released 0x[] 2019-12-07T19:34:18.607+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 5(3) notify: (query:41 sent:41 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010614337s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f1199c700 20 osd.7 41 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.607+0000 7f8f1199c700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:18.607+0000 7f8f1199c700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.607+0000 7f8f1199c700 20 osd.7 41 _dispatch 0x558c57641b00 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.607+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 prepare 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.607+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9200 2019-12-07T19:34:18.607+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 io_done 2019-12-07T19:34:18.607+0000 7f8f1199c700 3 osd.7 41 handle_osd_map epochs [41,41], i have 41, src has [1,41] 2019-12-07T19:34:18.607+0000 7f8f1199c700 10 osd.7 41 no new maps here, dropping 2019-12-07T19:34:18.607+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.5(3) info 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010614337s), identical to ours 2019-12-07T19:34:18.607+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010630130s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f06986700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.607+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:41 sent:41 1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([39,40] acting 1(5),2(2),3(0),4(3),5(4)))) e41/41) v1 -- 0x558c54e05c00 con 0x558c54c5bc00 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c574fe600 seq 8 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.608604+0000 1 0.000306 2019-12-07T19:34:18.607+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:18.607+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(41..41 src has 1..41) v4 -- 0x558c575d4480 con 0x558c54c19800 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe600 onodes shared_blobs 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe600 allocated 0x[] released 0x[] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 prepare 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.607+0000 7f8f0317f700 1 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe600 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 io_done 2019-12-07T19:34:18.607+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.5s0 1.5s0 (query:41 sent:41 1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([39,40] acting 1(1),2(0),3(2),4(3),5(5)))) e41/41) v1 -- 0x558c54e9a300 con 0x558c54c19800 2019-12-07T19:34:18.607+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.608636+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c54d21500 seq 7 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f0317f700 5 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c5500a000 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:18.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:18.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:18.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes shared_blobs 2019-12-07T19:34:18.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[] released 0x[] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 40 -> 41 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.608675+0000 1 0.000290 2019-12-07T19:34:18.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 40 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:34:18.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c55141000 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] new interval newup [1,4,5,2,7,3] newacting [1,4,5,2,7,3] 2019-12-07T19:34:18.607+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:41 sent:41 1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) 4->0 ([28,40] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([39,40] acting 1(1),2(2),4(0),5(3),7(4)))) e41/41) v1 -- 0x558c54e06300 con 0x558c54c5b400 2019-12-07T19:34:18.607+0000 7f8f05183700 1 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.608704+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c574ffb00 seq 44 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.608710+0000 1 0.000025 2019-12-07T19:34:18.607+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 40 -> 41 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.608718+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ffb00 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ffb00 onodes shared_blobs 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ffb00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.607+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 40 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.608727+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 prepare 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ffb00 2019-12-07T19:34:18.607+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 io_done 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.607+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] new interval newup [2,1,7,3,4,5] newacting [2,1,7,3,4,5] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.607+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:41 sent:41 1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2)))) e41/41) v1 -- 0x558c548a3100 con 0x558c54c5bc00 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.608761+0000 1 0.000028 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.607+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:18.607+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57ff0900 seq 10 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] new interval newup [1,4,5,2,7,3] newacting [1,4,5,2,7,3] 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.608772+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:18.607+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.607+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:18.607+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.608783+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:18.607+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0900 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:18.607+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0900 onodes shared_blobs 2019-12-07T19:34:18.607+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0900 allocated 0x[] released 0x[] 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]/[1,4,5,2147483647,7,3]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.607+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 prepare 2019-12-07T19:34:18.607+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0900 2019-12-07T19:34:18.607+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 io_done 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=39 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f08189700 15 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=34'17 active+remapped mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=41 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c54d5f000 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=41 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [1,4,5,2,7,3](1) acting [1,4,5,2147483647,7,3](1)) up_thru 39 up_from 13 last_epoch_clean 29 interval(39-40 up [1,4,5,2,7,3](1) acting [1,4,5,2147483647,7,3](1) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=41 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=41 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] new interval newup [2,1,7,3,4,5] newacting [2,1,7,3,4,5] 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 40 -> 41 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([39,40] acting 1(0),3(5),4(1),5(2),7(4))) 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=41 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 40 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]/[2,1,7,3,4,2147483647]p2(0) r=2 lpr=41 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.607+0000 7f8f06185700 15 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,39)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] new interval newup [1,4,3,5,7,2] newacting [1,4,3,5,7,2] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active pruub 57.057641346s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 57.057641346s (mnow 42.044300249s + 15.013341097s) 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active pruub 57.057641346s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.608870+0000 1 0.000016 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active pruub 57.057641346s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [2,1,7,3,4,5](2) acting [2,1,7,3,4,2147483647](2)) up_thru 39 up_from 13 last_epoch_clean 19 interval(39-40 up [2,1,7,3,4,5](2) acting [2,1,7,3,4,2147483647](2) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.608880+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.607+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.607+0000 7f8f08189700 1 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active pruub 57.057641346s@ mbc={} ps=[a~1]] start_peering_interval up [1,4,5,2,7,3] -> [1,4,5,2,7,3], acting [1,4,5,2147483647,7,3] -> [1,4,5,2,7,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.608889+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=39) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] noting past ([18,40] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([39,40] acting 1(1),2(0),3(3),4(4),7(2))) 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c5731a000 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=39 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a150] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active pruub 57.054473552s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 57.054473552s (mnow 42.044358129s + 15.010115423s) 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active pruub 57.054473552s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s4 40 -> 41 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] new interval newup [1,4,3,5,7,2] newacting [1,4,3,5,7,2] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active pruub 57.054473552s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 40 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.607+0000 7f8f06185700 1 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active pruub 57.054473552s@ mbc={} ps=[a~1]] start_peering_interval up [2,1,7,3,4,5] -> [2,1,7,3,4,5], acting [2,1,7,3,4,2147483647] -> [2,1,7,3,4,5], acting_primary 2(0) -> 2, up_primary 2(0) -> 2, role 2 -> 2, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]/[1,4,3,2147483647,7,2]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.607+0000 7f8f05183700 15 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,39)/1 luod=0'0 crt=40'15 active+remapped mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [7,4,1,5,2,3] 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] release_backoffs [1:b0000000::::0,1:c0000000::::head) 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [1,4,3,5,7,2](1) acting [1,4,3,2147483647,7,2](1)) up_thru 39 up_from 13 last_epoch_clean 29 interval(39-40 up [1,4,3,5,7,2](1) acting [1,4,3,2147483647,7,2](1) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.608976+0000 2 0.000067 2019-12-07T19:34:18.607+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(2),4(1),7(4))) 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.607+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.608988+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.609002+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013341097s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active pruub 57.057013246s@ mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 57.057013246s (mnow 42.044454290s + 15.012558956s) 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active pruub 57.057013246s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active pruub 57.057013246s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.609028+0000 1 0.000299 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f05183700 1 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active pruub 57.057013246s@ mbc={} ps=[a~1]] start_peering_interval up [1,4,3,5,7,2] -> [1,4,3,5,7,2], acting [1,4,3,2147483647,7,2] -> [1,4,3,5,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.607+0000 7f8f08189700 1 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.609060+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f08189700 5 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] new interval newup [7,4,1,5,2,3] newacting [7,4,1,5,2,3] 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.607+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]/[2147483647,4,1,5,2147483647,3]p4(1) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.010115423s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.607+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.607+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f07988700 15 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=29'15 active+remapped mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.609097+0000 1 0.000315 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:41 sent:41 1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) 4->0 ([28,40] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([39,40] acting 1(0),3(5),4(1),5(2),7(4)))) e41/41) v1 -- 0x558c57ed2700 con 0x558c54c64000 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [7,4,1,5,2,3](7) acting [2147483647,4,1,5,2147483647,3](4)) : acting set is too small 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.607+0000 7f8f06185700 1 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57b95200 seq 45 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.609128+0000 0 0.000000 2019-12-07T19:34:18.607+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:18.607+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=39) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4))) 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:18.607+0000 7f8f06185700 5 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:18.607+0000 7f8f06185700 20 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95200 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95200 onodes shared_blobs 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95200 allocated 0x[] released 0x[] 2019-12-07T19:34:18.607+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.607+0000 7f8f05183700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 prepare 2019-12-07T19:34:18.607+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95200 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] init_hb_stamps now [] 2019-12-07T19:34:18.607+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 io_done 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.607+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.044602399s + 0s) 2019-12-07T19:34:18.607+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012558956s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.607+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.607+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.607+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.609179+0000 1 0.000287 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.607+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:41 sent:41 1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) 2->0 ([18,40] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([39,40] acting 1(1),2(0),3(3),4(4),7(2)))) e41/41) v1 -- 0x558c57ed4300 con 0x558c54c19800 2019-12-07T19:34:18.607+0000 7f8f07988700 1 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'15 active mbc={} ps=[a~1]] start_peering_interval up [7,4,1,5,2,3] -> [7,4,1,5,2,3], acting [2147483647,4,1,5,2147483647,3] -> [7,4,1,5,2,3], acting_primary 4(1) -> 7, up_primary 7(0) -> 7, role -1 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.607+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.607+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.608+0000 7f8f05183700 1 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.608+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:18.608+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c54d20600 seq 28 2019-12-07T19:34:18.608+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.609207+0000 0 0.000000 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:18.608+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.608+0000 7f8f05183700 5 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.608+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:18.608+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.608+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:18.608+0000 7f8f05183700 20 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.608+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:18.608+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:18.608+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes shared_blobs 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.608+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:18.608+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.608+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:18.608+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:18.608+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.608+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.608+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.608+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.608+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:41 sent:41 1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) 4->0 ([28,40] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(2),4(1),7(4)))) e41/41) v1 -- 0x558c57393100 con 0x558c54c64000 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:18.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57ff0c00 seq 49 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.608+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:18.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:18.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:18.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:18.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0c00 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:18.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0c00 onodes shared_blobs 2019-12-07T19:34:18.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 prepare 2019-12-07T19:34:18.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0c00 2019-12-07T19:34:18.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 io_done 2019-12-07T19:34:18.608+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:18.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14453c 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001542992s (0.000000094s flush + 0.001542898s kv commit) 2019-12-07T19:34:18.608+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 13 submitting 13 deferred done 0 stable 0 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2558 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3849 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.609341+0000 1 0.000350 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:18.608+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4999 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6290 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7605 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 1 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8874 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:18.608+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.609372+0000 0 0.000000 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10143 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 11412 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12703 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13972 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 15044 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 15252 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 16639 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 16662 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 16662 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.608+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14453c~4116 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14453c 0x14453c~4116 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11453c 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x53c 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x652 and padding block with 0x9ae 2019-12-07T19:34:18.608+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 -- 0x558c57453800 con 0x558c53ae5400 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c57b94300 seq 29 2019-12-07T19:34:18.608+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._biginfo' <- _biginfo 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59b00> 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94300 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94300 onodes shared_blobs 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94300 allocated 0x[] released 0x[] 2019-12-07T19:34:18.608+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 0+0+0 (unknown 0 0 0) 0x558c57453800 con 0x558c53ae5400 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 prepare 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94300 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 io_done 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.2s4 from 7(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.2s4 from 7(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c5731a000 2019-12-07T19:34:18.608+0000 7f8f1119b700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.608+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.2s4 from 7(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x148652 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.608+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 13 cleaned 0 in 0.000194514s (0.000000081s flush + 0.000194433s kv commit) 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 -- 0x558c58122000 con 0x558c53ae5400 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c57b95b00 seq 30 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x148652~522 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x148652 0x148652~522 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x118652 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x652 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb74 and padding block with 0x48c 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95b00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95b00 onodes shared_blobs 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95b00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.608+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 0+0+0 (unknown 0 0 0) 0x558c58122000 con 0x558c53ae5400 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 prepare 2019-12-07T19:34:18.608+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95b00 2019-12-07T19:34:18.608+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 io_done 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.608+0000 7f8f1119b700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.608+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fdb00,0x558c5740af00,0x558c57ff1800,0x558c5740a600,0x558c57ff1b00,0x558c57ef9200,0x558c574fe600,0x558c54d21500,0x558c574ffb00,0x558c57ff0900,0x558c57b95200,0x558c54d20600,0x558c57ff0c00> 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fdb00 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fdb00 onodes 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fdb00 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fdb00 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.7(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1800 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1800 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1800 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:18.608+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1800 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:05.248753+0000, must = 0 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] osd.7(4) has stray content: 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1b00 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1b00 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1b00 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1b00 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.608+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9200 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9200 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9200 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9200 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 7 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe600 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe600 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe600 done 2019-12-07T19:34:18.608+0000 7f8f07988700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe600 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ffb00 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ffb00 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ffb00 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ffb00 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 7(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0900 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0900 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0900 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0900 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95200 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95200 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95200 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:18.608+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.7(4) info 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41), identical to ours 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95200 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:18.608+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0c00 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0c00 onodes 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0c00 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0c00 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.608+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x148b74 2019-12-07T19:34:18.608+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 76 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558c576df500 con 0x558c54c19c00 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000191229s (0.000000075s flush + 0.000191154s kv commit) 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:18.608+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x148b74~416 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x148b74 0x148b74~416 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x118b74 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb74 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf8a and padding block with 0x76 2019-12-07T19:34:18.608+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 5(3) notify: (query:41 sent:41 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.608+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.5(3) info 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41), identical to ours 2019-12-07T19:34:18.608+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94300> 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94300 2019-12-07T19:34:18.606+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 finishing 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94300 onodes 2019-12-07T19:34:18.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94300 done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95b00 kv_submitted 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94300 [] 2019-12-07T19:34:18.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.608+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c54d98000 2019-12-07T19:34:18.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x148f8a 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] handle_advance_map: 41 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 40 -> 41 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123161s (0.000000066s flush + 0.000123095s kv commit) 2019-12-07T19:34:18.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:18.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] new interval newup [1,3,4,5,7,2] newacting [1,3,4,5,7,2] 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:18.609867+0000 1 0.000012 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] exit Started/ReplicaActive 2019-12-07T19:34:18.609873+0000 0 0.000000 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] exit Started 2019-12-07T19:34:18.609878+0000 0 0.000000 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] enter Reset 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=39 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] state: Reset advmap 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] new interval newup [1,3,4,5,7,2] newacting [1,3,4,5,7,2] 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]/[1,3,4,5,2147483647,2]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] set_last_peering_reset 41 2019-12-07T19:34:18.608+0000 7f8f05984700 15 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]p1(0) r=-1 lpr=41 pi=[28,39)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] clear_stats 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] start_peering_interval: check_new_interval output: check_new_interval interval(39-40 up [1,3,4,5,7,2](1) acting [1,3,4,5,2147483647,2](1)) up_thru 39 up_from 13 last_epoch_clean 29 interval(39-40 up [1,3,4,5,7,2](1) acting [1,3,4,5,2147483647,2](1) maybe_went_rw) : primary up 13-39 includes interval 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=39) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] noting past ([28,40] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(1),4(2),5(3))) 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] on_new_interval 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] init_hb_stamps now [0x558c5523a230] 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] on_new_interval prior_readable_until_ub 0s (mnow 42.045402178s + 0s) 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:18.608+0000 7f8f05984700 1 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] start_peering_interval up [1,3,4,5,7,2] -> [1,3,4,5,7,2], acting [1,3,4,5,2147483647,2] -> [1,3,4,5,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role -1 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] clear_primary_state 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] release_backoffs [1:70000000::::0,1:80000000::::head) 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] on_change_cleanup 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] on_change 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_role_change 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.608+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 41 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] exit Reset 2019-12-07T19:34:18.610118+0000 1 0.000226 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] enter Started 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] enter Start 2019-12-07T19:34:18.608+0000 7f8f05984700 1 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] state: transitioning to Stray 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] exit Start 2019-12-07T19:34:18.610143+0000 0 0.000000 2019-12-07T19:34:18.608+0000 7f8f05984700 5 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] enter Started/Stray 2019-12-07T19:34:18.608+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 40 while current is 41 2019-12-07T19:34:18.608+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.608+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:18.608+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:41 sent:41 1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(1),4(2),5(3)))) e41/41) v1 -- 0x558c574e7800 con 0x558c54c64000 2019-12-07T19:34:18.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:18.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c57ef8900 seq 6 2019-12-07T19:34:18.609+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:18.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:18.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:18.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:18.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:18.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8900 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:18.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8900 onodes shared_blobs 2019-12-07T19:34:18.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8900 allocated 0x[] released 0x[] 2019-12-07T19:34:18.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 prepare 2019-12-07T19:34:18.609+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8900 2019-12-07T19:34:18.609+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 io_done 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 NullEvt) prio 255 cost 10 e41) pg 0x558c54d5c000 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_advance_map: 41 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 40 -> 41 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 40 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] state: Started advmap 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:18.609+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] plpg_on_pool_change 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] hit_set_clear 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] agent_stop 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 41 advance_pg new pool opts old pool opts 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] handle_activate_map: 41 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] activate_map 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] activate_map: Not dirtying info: last_persisted is 39 while current is 41 2019-12-07T19:34:18.609+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:18.609+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 NullEvt 2019-12-07T19:34:18.609+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:18.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.609+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:18.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:18.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:18.609+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x148f8a~50c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x148f8a 0x148f8a~50c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x118f8a 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf8a 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.609+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x496 and padding block with 0xb6a 2019-12-07T19:34:18.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x149496 2019-12-07T19:34:18.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000836295s (0.000000110s flush + 0.000836185s kv commit) 2019-12-07T19:34:18.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 257 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57d9a780 con 0x558c54c64000 2019-12-07T19:34:18.610+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.es4 epoch_sent: 41 epoch_requested: 41 MQuery 1.es4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.610+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.es4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 258 ==== pg_query2(1.9s4 1.9s4 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57d9a780 con 0x558c54c64000 2019-12-07T19:34:18.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.es4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.es4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c54d98000 2019-12-07T19:34:18.610+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.9s4 epoch_sent: 41 epoch_requested: 41 MQuery 1.9s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.610+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.9s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.610+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.es4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.610+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.610+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:18.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 259 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c54e06300 con 0x558c54c64000 2019-12-07T19:34:18.610+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:41 sent:41 1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(1),4(2),5(3)))) e41/41) v1 -- 0x558c58124300 con 0x558c54c64000 2019-12-07T19:34:18.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.610+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.610+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 260 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (crc 0 0 0) 0x558c54e06300 con 0x558c54c64000 2019-12-07T19:34:18.610+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.610+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 261 ==== pg_query2(1.7s4 1.7s4 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57d9a780 con 0x558c54c64000 2019-12-07T19:34:18.610+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.7s4 epoch_sent: 41 epoch_requested: 41 MQuery 1.7s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.610+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.610+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.7s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.610+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.611+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.611+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.1(2) 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2019-12-07T19:34:18.611+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:04.384843+0000, must = 0 2019-12-07T19:34:18.611+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.611+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.611+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 262 ==== pg_query2(1.8s5 1.8s5 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57d9a780 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.8s5 epoch_sent: 41 epoch_requested: 41 MQuery 1.8s5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.611+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.8s5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 263 ==== pg_query2(1.as5 1.as5 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57d9a780 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.as5 epoch_sent: 41 epoch_requested: 41 MQuery 1.as5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.611+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.as5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.8s5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.8s5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c57380000 2019-12-07T19:34:18.611+0000 7f8f04982700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.8s5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.611+0000 7f8f04982700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.611+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 264 ==== osd_map(41..41 src has 1..41) v4 ==== 974+0+0 (crc 0 0 0) 0x558c54dd1200 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f1199c700 20 osd.7 41 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.611+0000 7f8f1199c700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:18.611+0000 7f8f1199c700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:18.611+0000 7f8f1199c700 20 osd.7 41 _dispatch 0x558c54dd1200 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.611+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:18.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 266 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e41/41) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c54e06300 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.8s0 1.8s0 (query:41 sent:41 1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([39,40] acting 1(0),2(3),3(4),4(1),5(2)))) e41/41) v1 -- 0x558c5812d800 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.0s3 epoch_sent: 41 epoch_requested: 41 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s) 2019-12-07T19:34:18.611+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e41) 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s)) prio 255 cost 10 e41) pg 0x558c5500d000 2019-12-07T19:34:18.611+0000 7f8f04982700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 52.075598409s int 16s) 2019-12-07T19:34:18.611+0000 7f8f04982700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:18.611+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:18.611+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:18.611+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c574fef00 seq 34 2019-12-07T19:34:18.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 267 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007661900s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c57393100 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:18.611+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:18.611+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:18.611+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:18.611+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007661900s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.611+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007661900s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.611+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:18.611+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fef00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:18.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fef00 onodes shared_blobs 2019-12-07T19:34:18.611+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fef00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.610+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 99 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c5812a000 con 0x558c54c19800 2019-12-07T19:34:18.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.7s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 prepare 2019-12-07T19:34:18.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.7s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c54d5f000 2019-12-07T19:34:18.611+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fef00 2019-12-07T19:34:18.611+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 io_done 2019-12-07T19:34:18.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.012400154s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.7s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.611+0000 7f8f24c58700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.611+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.611+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95b00,0x558c57ef8900> 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 kv_submitted 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95b00 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 finishing 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95b00 onodes 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95b00 done 2019-12-07T19:34:18.611+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95b00 [] 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 kv_submitted 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8900 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 finishing 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8900 onodes 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8900 done 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8900 [] 2019-12-07T19:34:18.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.611+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:41 sent:41 1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) 4->0 ([28,40] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(2),4(1),7(4)))) e41/41) v1 -- 0x558c58134000 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.as5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.as5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c57e8b000 2019-12-07T19:34:18.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:18.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:23.704196+0000, must = 0 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:18.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.611+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.as5 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:18.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:18.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.611+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:18.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.611+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.611+0000 7f8f0297e700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x149496~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x149496 0x149496~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.9s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.611+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x119496 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x496 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x92b and padding block with 0x6d5 2019-12-07T19:34:18.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.9s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c5500a000 2019-12-07T19:34:18.611+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.013178993s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.9s4 from 1(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.611+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:41 sent:41 1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4)))) e41/41) v1 -- 0x558c576de700 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:18.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:41 sent:41 1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) 4->0 ([28,40] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([39,40] acting 1(0),3(5),4(1),5(2),7(4)))) e41/41) v1 -- 0x558c57ed2000 con 0x558c54c64000 2019-12-07T19:34:18.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:18.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 268 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (crc 0 0 0) 0x558c57393100 con 0x558c54c64000 2019-12-07T19:34:18.612+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14992b 2019-12-07T19:34:18.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000188518s (0.000000095s flush + 0.000188423s kv commit) 2019-12-07T19:34:18.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fef00> 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 kv_submitted 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fef00 2019-12-07T19:34:18.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 finishing 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fef00 onodes 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fef00 done 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:18.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fef00 [] 2019-12-07T19:34:18.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.612+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:18.612+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:18.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 175 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558c57393100 con 0x558c54c5b400 2019-12-07T19:34:18.612+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 176 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558c57393100 con 0x558c54c5b400 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 1(2) notify: (query:41 sent:41 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.1(2) info 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41), identical to ours 2019-12-07T19:34:18.612+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.612+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 100 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (crc 0 0 0) 0x558c57ed4300 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 2019-12-07T19:34:18.612+0000 7f8f24c58700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:25.571339+0000, must = 0 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.612+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.612+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 102 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007787798s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c57ed4300 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.2(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:18.612+0000 7f8f24c58700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007787798s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:04.735329+0000, must = 0 2019-12-07T19:34:18.612+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007787798s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.612+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.612+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 103 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (crc 0 0 0) 0x558c57ed4300 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f24c58700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.612+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 104 ==== pg_query2(1.5s4 1.5s4 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57d9a280 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f24c58700 15 osd.7 41 enqueue_peering_evt 1.5s4 epoch_sent: 41 epoch_requested: 41 MQuery 1.5s4 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.5s4 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 177 ==== pg_query2(1.3s4 1.3s4 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57e8e780 con 0x558c54c5b400 2019-12-07T19:34:18.612+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.3s4 epoch_sent: 41 epoch_requested: 41 MQuery 1.3s4 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.3s4 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.3s4 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.3s4 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c5513e000 2019-12-07T19:34:18.612+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010466029s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.3s4 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.612+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:41 sent:41 1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) 4->0 ([28,40] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([39,40] acting 1(1),2(2),4(0),5(3),7(4)))) e41/41) v1 -- 0x558c54e04e00 con 0x558c54c5b400 2019-12-07T19:34:18.612+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.612+0000 7f8f1199c700 3 osd.7 41 handle_osd_map epochs [41,41], i have 41, src has [1,41] 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 no new maps here, dropping 2019-12-07T19:34:18.612+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 265 ==== osd_map(41..41 src has 1..41) v4 ==== 974+0+0 (crc 0 0 0) 0x558c57e03b00 con 0x558c54c64000 2019-12-07T19:34:18.612+0000 7f8f1199c700 20 osd.7 41 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:18.612+0000 7f8f1199c700 20 osd.7 41 _dispatch 0x558c57e03b00 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.612+0000 7f8f1199c700 3 osd.7 41 handle_osd_map epochs [41,41], i have 41, src has [1,41] 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 no new maps here, dropping 2019-12-07T19:34:18.612+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 174 ==== osd_map(41..41 src has 1..41) v4 ==== 974+0+0 (unknown 0 0 0) 0x558c54d2efc0 con 0x558c54c5b400 2019-12-07T19:34:18.612+0000 7f8f1199c700 20 osd.7 41 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:18.612+0000 7f8f1199c700 20 osd.7 41 _dispatch 0x558c54d2efc0 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.612+0000 7f8f1199c700 3 osd.7 41 handle_osd_map epochs [41,41], i have 41, src has [1,41] 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 no new maps here, dropping 2019-12-07T19:34:18.612+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 101 ==== osd_map(41..41 src has 1..41) v4 ==== 974+0+0 (crc 0 0 0) 0x558c575d4480 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f1199c700 20 osd.7 41 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:18.612+0000 7f8f1199c700 20 osd.7 41 _dispatch 0x558c575d4480 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.612+0000 7f8f1199c700 3 osd.7 41 handle_osd_map epochs [41,41], i have 41, src has [1,41] 2019-12-07T19:34:18.612+0000 7f8f1199c700 10 osd.7 41 no new maps here, dropping 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 2(4) notify: (query:41 sent:41 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.2(4) info 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41), identical to ours 2019-12-07T19:34:18.612+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 4(1) notify: (query:41 sent:41 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.4(1) info 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41), identical to ours 2019-12-07T19:34:18.612+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.5s4 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.5s4 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c57522000 2019-12-07T19:34:18.612+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.5s4 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.612+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.5s0 1.5s0 (query:41 sent:41 1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([39,40] acting 1(1),2(0),3(2),4(3),5(5)))) e41/41) v1 -- 0x558c57ed4a00 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 105 ==== pg_query2(1.ds2 1.ds2 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (crc 0 0 0) 0x558c57d9a280 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007661900s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f24c58700 15 osd.7 41 enqueue_peering_evt 1.ds2 epoch_sent: 41 epoch_requested: 41 MQuery 1.ds2 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.ds2 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007661900s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.ds2 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.ds2 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c55141000 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 1(2) notify: (query:41 sent:41 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007661900s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009942032s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.ds2 from 2(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.1(2) info 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007661900s), identical to ours 2019-12-07T19:34:18.612+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.612+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.612+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:41 sent:41 1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) 2->0 ([18,40] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([39,40] acting 1(1),2(0),3(3),4(4),7(2)))) e41/41) v1 -- 0x558c57ed5100 con 0x558c54c19800 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 2019-12-07T19:34:18.612+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:18.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 178 ==== pg_query2(1.bs1 1.bs1 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57e8e780 con 0x558c54c5b400 2019-12-07T19:34:18.612+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.bs1 epoch_sent: 41 epoch_requested: 41 MQuery 1.bs1 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:34:56.803413+0000, must = 0 2019-12-07T19:34:18.612+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.bs1 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.612+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 179 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57e8e780 con 0x558c54c5b400 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.bs3 epoch_sent: 41 epoch_requested: 41 MQuery 1.bs3 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007787798s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.612+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.bs3 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007787798s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 2(4) notify: (query:41 sent:41 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007787798s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.2(4) info 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.007787798s), identical to ours 2019-12-07T19:34:18.612+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.612+0000 7f8f06986700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.613+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.bs1 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.bs1 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c575ef000 2019-12-07T19:34:18.613+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.bs1 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.613+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.613+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:18.613+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:41 sent:41 1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e41/41) v1 -- 0x558c57390700 con 0x558c54c5b400 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.bs3 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.bs3 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c54d9b000 2019-12-07T19:34:18.613+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.bs3 from 4(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] update_history prior_readable_until_ub 57.054279632s (mnow 42.049758578s + 15.004521054s) 2019-12-07T19:34:18.613+0000 7f8f07187700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.613+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:18.613+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:41 sent:41 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) 3->0 ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e41/41) v1 -- 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:18.613+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:18.613+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c54ddfb00 seq 24 2019-12-07T19:34:18.613+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:18.613+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:18.613+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:18.613+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddfb00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:18.613+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddfb00 onodes shared_blobs 2019-12-07T19:34:18.613+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddfb00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.613+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 prepare 2019-12-07T19:34:18.613+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddfb00 2019-12-07T19:34:18.613+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 io_done 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14992b~416 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14992b 0x14992b~416 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11992b 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x92b 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd41 and padding block with 0x2bf 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x149d41 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000104803s (0.000000090s flush + 0.000104713s kv commit) 2019-12-07T19:34:18.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddfb00> 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 kv_submitted 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddfb00 2019-12-07T19:34:18.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 finishing 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddfb00 onodes 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddfb00 done 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:18.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddfb00 [] 2019-12-07T19:34:18.613+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.613+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:18.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 180 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57390700 con 0x558c54c5b400 2019-12-07T19:34:18.613+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 181 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004462018s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c57390700 con 0x558c54c5b400 2019-12-07T19:34:18.613+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004462018s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004462018s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:03.254176+0000, must = 0 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004462018s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004462018s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 4(5) notify: (query:41 sent:41 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004462018s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.4(5) info 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004462018s), identical to ours 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:18.613+0000 7f8f0297e700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 148 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c54e06a00 con 0x558c54c5bc00 2019-12-07T19:34:18.615+0000 7f8f23c56700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.615+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.615+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.615+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.3(1) 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 2019-12-07T19:34:18.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 149 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558c548a3100 con 0x558c54c5bc00 2019-12-07T19:34:18.615+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.4s0 register next scrub, scrub time 2019-12-07T19:35:10.499551+0000, must = 0 2019-12-07T19:34:18.615+0000 7f8f23c56700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.615+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.615+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 150 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57d9a500 con 0x558c54c5bc00 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:18.615+0000 7f8f23c56700 15 osd.7 41 enqueue_peering_evt 1.1s1 epoch_sent: 41 epoch_requested: 41 MQuery 1.1s1 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.615+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.1s1 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.615+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:18.615+0000 7f8f06986700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:18.617168+0000 10 0.000380 2019-12-07T19:34:18.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 151 ==== pg_query2(1.fs5 1.fs5 query(info 0'0 epoch_sent 41) e41/41) v1 ==== 158+0+0 (unknown 0 0 0) 0x558c57d9a500 con 0x558c54c5bc00 2019-12-07T19:34:18.615+0000 7f8f23c56700 15 osd.7 41 enqueue_peering_evt 1.fs5 epoch_sent: 41 epoch_requested: 41 MQuery 1.fs5 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.615+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.fs5 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) 2019-12-07T19:34:18.615+0000 7f8f06986700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:18.615+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 2019-12-07T19:34:18.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 152 ==== pg_notify2(1.2s0 1.2s0 (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e41/41) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558c548a3100 con 0x558c54c5bc00 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.3(1) 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 2019-12-07T19:34:18.616+0000 7f8f23c56700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.616+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.7(0) 1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] calc_acting prefer osd.7(0) because it is current primary 2019-12-07T19:34:18.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 153 ==== pg_notify2(1.4s0 1.4s0 (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e41/41) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c548a3100 con 0x558c54c5bc00 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 3(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 4(5) 2019-12-07T19:34:18.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f23c56700 15 osd.7 41 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.fs5 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.616+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.fs5 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c57383000 2019-12-07T19:34:18.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.fs5 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:34:18.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_async_recovery_ec result want=[7,3,1,5,2,4] async_recovery= 2019-12-07T19:34:18.616+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 154 ==== osd_map(41..41 src has 1..41) v4 ==== 974+0+0 (unknown 0 0 0) 0x558c57570d80 con 0x558c54c5bc00 2019-12-07T19:34:18.616+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:18.616+0000 7f8f1199c700 20 osd.7 41 OSD::ms_dispatch: osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.616+0000 7f8f1199c700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:18.616+0000 7f8f1199c700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:18.616+0000 7f8f1199c700 20 osd.7 41 _dispatch 0x558c57570d80 osd_map(41..41 src has 1..41) v4 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] acting_recovery_backfill is 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:18.616+0000 7f8f1199c700 3 osd.7 41 handle_osd_map epochs [41,41], i have 41, src has [1,41] 2019-12-07T19:34:18.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 155 ==== MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=114, reqid=osd.3.0:114, at_version=41'37, trim_to=0'0, roll_forward_to=39'35)) v2 ==== 3417+0+0 (unknown 0 0 0) 0x558c57dff800 con 0x558c54c5bc00 2019-12-07T19:34:18.616+0000 7f8f1199c700 10 osd.7 41 no new maps here, dropping 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting want=[7,3,1,5,2,4] backfill_targets= async_recovery_targets= 2019-12-07T19:34:18.616+0000 7f8f23c56700 15 osd.7 41 enqueue_op 0x558c57f3e2c0 prio 127 cost 0 latency 0.000027 epoch 41 MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=114, reqid=osd.3.0:114, at_version=41'37, trim_to=0'0, roll_forward_to=39'35)) v2 2019-12-07T19:34:18.616+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:41 sent:41 1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2)))) e41/41) v1 -- 0x558c54e98e00 con 0x558c54c5bc00 2019-12-07T19:34:18.616+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=114, reqid=osd.3.0:114, at_version=41'37, trim_to=0'0, roll_forward_to=39'35)) v2) prio 127 cost 0 e41) 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: leaving GetLog 2019-12-07T19:34:18.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f06986700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:18.617340+0000 0 0.000000 2019-12-07T19:34:18.616+0000 7f8f06986700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetMissing 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.1(2) has no missing, identical log 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=114, reqid=osd.3.0:114, at_version=41'37, trim_to=0'0, roll_forward_to=39'35)) v2) prio 127 cost 0 e41) queued 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.2(4) has no missing, identical log 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=114, reqid=osd.3.0:114, at_version=41'37, trim_to=0'0, roll_forward_to=39'35)) v2) prio 127 cost 0 e41) pg 0x558c54d5c000 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.3(1) has no missing, identical log 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.4(5) has no missing, identical log 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 41 dequeue_op 0x558c57f3e2c0 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=114, reqid=osd.3.0:114, at_version=41'37, trim_to=0'0, roll_forward_to=39'35)) v2 pg pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.5(3) has no missing, identical log 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: still need up_thru update before going active 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=114, reqid=osd.3.0:114, at_version=41'37, trim_to=0'0, roll_forward_to=39'35)) v2 2019-12-07T19:34:18.616+0000 7f8f06986700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetMissing 2019-12-07T19:34:18.617397+0000 0 0.000000 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6c205177:::smithi13140796-8:10 c,e was 2019-12-07T19:34:18.616+0000 7f8f06986700 5 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:18.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:18.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:18.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d59200 seq 25 2019-12-07T19:34:18.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:34:18.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes shared_blobs 2019-12-07T19:34:18.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[] released 0x[] 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:10 c,e,10 2019-12-07T19:34:18.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:18.616+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:18.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6c205177:::smithi13140796-8:10 c,e 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.4s0 from 3(1) notify: (query:41 sent:41 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x149d41~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f06986700 7 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: handle_pg_notify from osd.3(1) 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x149d41 0x149d41~2e to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 39'35 (0'0,39'35] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] append_log log((0'0,39'35], crt=39'35) [41'36 (39'31) modify 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:18.607795+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,41'37 (39'35) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:18.607795+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x119d41 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd41 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.3(1) info 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41), identical to ours 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd6f and padding block with 0x291 2019-12-07T19:34:18.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:18.616+0000 7f8f06986700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'36 (0'0,41'36] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'35 crt=39'35 lcod 39'35 active mbc={} ps=[a~1,d~1]] add_log_entry 41'36 (39'31) modify 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:18.607795+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'35 crt=39'35 lcod 39'35 active mbc={} ps=[a~1,d~1]] add_log_entry 41'37 (39'35) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:18.607795+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'35 crt=39'35 lcod 39'35 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 37 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'35 crt=39'35 lcod 39'35 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:18.616+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 41'36, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x149d6f 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730ac00 seq 66 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000108243s (0.000000080s flush + 0.000108163s kv commit) 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# 1 keys 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# 1 keys = 0 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# hinfo_key (42 bytes) 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# hinfo_key (42 bytes) = 0 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:18.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- SNA_1_0000000000000010_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000041.00000000000000000036' <- 0000000041.00000000000000000036 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000041.00000000000000000037' <- 0000000041.00000000000000000037 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730ac00 cost 8396 (1 ios * 4000 + 4396 bytes) 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730ac00 onodes 0x558c574a1b80,0x558c575ba280 shared_blobs 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c574a1c88) update 2#1:6c205177:::smithi13140796-8:10# 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:10# is 687 (411 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 1592 (563 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 prepare 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730ac00 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 io_done 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=39'35 crt=39'35 lcod 39'35 active mbc={} ps=[a~1,d~1]] op_applied version 41'37 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.616+0000 7f8f0197c700 10 osd.7 41 dequeue_op 0x558c57f3e2c0 finish 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5137 < min_flush_size 524288 2019-12-07T19:34:18.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1b80 had 1 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5160 < min_flush_size 524288 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5160 < min_flush_size 524288 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x149d6f~1428 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x149d6f 0x149d6f~1428 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x119d6f 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd6f 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x197 and padding block with 0xe69 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14b197 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123848s (0.000000059s flush + 0.000123789s kv commit) 2019-12-07T19:34:18.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got osd.3(5) 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730ac00> 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 kv_submitted 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730ac00 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 finishing 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730ac00 onodes 0x558c574a1b80,0x558c575ba280 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730ac00 done 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730ac00 [] 2019-12-07T19:34:18.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:23.320297+0000, must = 0 2019-12-07T19:34:18.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.616+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 41/28 ECSubWriteReply(tid=114, last_complete=41'37, committed=1, applied=1)) v2 -- 0x558c5841f200 con 0x558c54c5bc00 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:18.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:18.616+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:18.617844+0000 12 0.000432 2019-12-07T19:34:18.616+0000 7f8f07988700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.1(2) 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009633327s) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.2(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.3(5) 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.5(3) 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.7(0) 1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting all_info osd.7(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=41) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] For position 0: selecting up[i]: 7(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_async_recovery_ec result want=[7,4,1,5,2,3] async_recovery= 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] acting_recovery_backfill is 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] choose_acting want=[7,4,1,5,2,3] backfill_targets= async_recovery_targets= 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: requesting log from osd.1(2) 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:18.616+0000 7f8f07988700 15 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] publish_stats_to_osd 41:92 2019-12-07T19:34:18.616+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(log 29'15 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 epoch_sent 41) e41/41) v1 -- 0x558c57e8ea00 con 0x558c54c64000 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41)> waiting <> waiting_peering {} 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MNotifyRec 1.2s0 from 3(5) notify: (query:41 sent:41 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:18.616+0000 7f8f07988700 7 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: handle_pg_notify from osd.3(5) 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] got dup osd.3(5) info 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41), identical to ours 2019-12-07T19:34:18.616+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:18.616+0000 7f8f07988700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:18.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 269 ==== PGlog(1.2s0 log log((29'15,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) e41/41) v6 ==== 2024+0+0 (crc 0 0 0) 0x558c54897000 con 0x558c54c64000 2019-12-07T19:34:18.617+0000 7f8f24457700 15 osd.7 41 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 MLogRec from 1(2) log log((29'15,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) +create_info 2019-12-07T19:34:18.617+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MLogRec from 1(2) log log((29'15,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e41) 2019-12-07T19:34:18.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MLogRec from 1(2) log log((29'15,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MLogRec from 1(2) log log((29'15,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MLogRec from 1(2) log log((29'15,29'15], crt=29'15) pi ([0,0] all_participants= intervals=) +create_info 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: GetLog: received master log from osd.1(2) 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: leaving GetLog 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: processing master log 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] proc_master_log for osd.1(2): log((29'15,29'15], crt=29'15) missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.617+0000 7f8f03980700 10 merge_log log((29'15,29'15], crt=29'15) from osd.1(2) into log((0'0,29'15], crt=29'15) 2019-12-07T19:34:18.617+0000 7f8f03980700 10 merge_log copying olog dups to log 21'1 to 29'15 2019-12-07T19:34:18.617+0000 7f8f03980700 10 merge_log removed dups overlapping log entries (0'0,29'15] 2019-12-07T19:34:18.617+0000 7f8f03980700 10 merge_log result log((0'0,29'15], crt=29'15) missing(0 may_include_deletes = 1) changed=1 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] peer osd.1(2) now 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) missing(0 may_include_deletes = 1) 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:08.239446+0000, must = 0 2019-12-07T19:34:18.617+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:18.618536+0000 2 0.000254 2019-12-07T19:34:18.617+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/GetMissing 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.1(2) has no missing, identical log 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.2(4) has no missing, identical log 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.3(5) has no missing, identical log 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.4(1) has no missing, identical log 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: osd.5(3) has no missing, identical log 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: still need up_thru update before going active 2019-12-07T19:34:18.617+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/GetMissing 2019-12-07T19:34:18.618601+0000 0 0.000000 2019-12-07T19:34:18.617+0000 7f8f03980700 5 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:18.617+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:18.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57ef8000 seq 14 2019-12-07T19:34:18.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore.onode(0x558c57e8e280).flush flush done 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range remove range start: 0x0000000000000493'.dup_0000000000.00000000000000000000' end: 0x0000000000000493'.dup_4294967295.18446744073709551615' 2019-12-07T19:34:18.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkey_range 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:18.617+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:18.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:18.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8000 cost 5658 (1 ios * 4000 + 1658 bytes) 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8000 onodes shared_blobs 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8000 allocated 0x[] released 0x[] 2019-12-07T19:34:18.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 prepare 2019-12-07T19:34:18.617+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8000 2019-12-07T19:34:18.617+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 io_done 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1358 < min_flush_size 524288 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1381 < min_flush_size 524288 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1381 < min_flush_size 524288 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.617+0000 7f8f03980700 10 osd.7 41 queue_want_up_thru want 41 <= queued 41, currently 40 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14b197~565 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14b197 0x14b197~565 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11b197 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x197 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6fc and padding block with 0x904 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14b6fc 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000101787s (0.000000077s flush + 0.000101710s kv commit) 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8000> 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 kv_submitted 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8000 2019-12-07T19:34:18.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 finishing 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8000 onodes 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8000 done 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:18.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8000 [] 2019-12-07T19:34:18.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:18.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.1s1 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MQuery 1.1s1 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41)) prio 255 cost 10 e41) pg 0x558c5736a000 2019-12-07T19:34:18.617+0000 7f8f04181700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MQuery 1.1s1 from 3(0) query_epoch 41 query: query(info 0'0 epoch_sent 41) 2019-12-07T19:34:18.617+0000 7f8f04181700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] sending info 2019-12-07T19:34:18.617+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:18.617+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:41 sent:41 1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([39,40] acting 1(5),2(2),3(0),4(3),5(4)))) e41/41) v1 -- 0x558c576b8700 con 0x558c54c5bc00 2019-12-07T19:34:18.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:18.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 156 ==== MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=115, reqid=osd.3.0:114, at_version=0'0, trim_to=0'0, roll_forward_to=41'37)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558c57dff000 con 0x558c54c5bc00 2019-12-07T19:34:18.618+0000 7f8f23c56700 15 osd.7 41 enqueue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000019 epoch 41 MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=115, reqid=osd.3.0:114, at_version=0'0, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:18.618+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=115, reqid=osd.3.0:114, at_version=0'0, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e41) 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=115, reqid=osd.3.0:114, at_version=0'0, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e41) queued 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=115, reqid=osd.3.0:114, at_version=0'0, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e41) pg 0x558c54d5c000 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 osd.7 41 dequeue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000043 MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=115, reqid=osd.3.0:114, at_version=0'0, trim_to=0'0, roll_forward_to=41'37)) v2 pg pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.6s2 41/28 ECSubWrite(tid=115, reqid=osd.3.0:114, at_version=0'0, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=39'35 active mbc={} ps=[a~1,d~1]] append_log log((0'0,41'37], crt=39'35) [] 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1]] rollforward: entry=41'36 (39'31) modify 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:18.607795+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1]] rollforward: entry=41'37 (39'35) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:18.607795+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 37 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:18.618+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5730a900 seq 67 2019-12-07T19:34:18.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a900 onodes shared_blobs 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a900 allocated 0x[] released 0x[] 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 prepare 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a900 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 io_done 2019-12-07T19:34:18.618+0000 7f8f0197c700 10 osd.7 41 dequeue_op 0x558c57334dc0 finish 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3859480576 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.99844) bluestore 86 GiB free (0.959897), bluefs_ratio 0.0399446 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14b6fc~471 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14b6fc 0x14b6fc~471 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11b6fc 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6fc 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb6d and padding block with 0x493 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14bb6d 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000130761s (0.000000059s flush + 0.000130702s kv commit) 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a900> 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 kv_submitted 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a900 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 finishing 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a900 onodes 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a900 done 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a900 [] 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.618+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 41/28 ECSubWriteReply(tid=115, last_complete=41'37, committed=1, applied=1)) v2 -- 0x558c5841f400 con 0x558c54c5bc00 2019-12-07T19:34:18.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 157 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e41/41) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c576b8700 con 0x558c54c5bc00 2019-12-07T19:34:18.618+0000 7f8f23c56700 15 osd.7 41 enqueue_peering_evt 1.6s2 epoch_sent: 41 epoch_requested: 41 MInfoRec from 3(0) info: 1.6s0( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:18.618+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MInfoRec from 3(0) info: 1.6s0( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e41) 2019-12-07T19:34:18.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:18.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MInfoRec from 3(0) info: 1.6s0( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e41) queued 2019-12-07T19:34:18.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:18.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 MInfoRec from 3(0) info: 1.6s0( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e41) pg 0x558c54d5c000 2019-12-07T19:34:18.618+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 MInfoRec from 3(0) info: 1.6s0( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:18.618+0000 7f8f05984700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:18.618+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:18.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:18.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57ef8300 seq 68 2019-12-07T19:34:18.618+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:18.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:18.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:18.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:18.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8300 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:18.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8300 onodes shared_blobs 2019-12-07T19:34:18.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8300 allocated 0x[] released 0x[] 2019-12-07T19:34:18.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 prepare 2019-12-07T19:34:18.618+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8300 2019-12-07T19:34:18.618+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 io_done 2019-12-07T19:34:18.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14bb6d~47d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14bb6d 0x14bb6d~47d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11bb6d 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb6d 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfea and padding block with 0x16 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14bfea 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:18.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000086423s (0.000000086s flush + 0.000086337s kv commit) 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:18.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8300> 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 kv_submitted 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8300 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 finishing 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8300 onodes 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8300 done 2019-12-07T19:34:18.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8300 [] 2019-12-07T19:34:18.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:18.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:18.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:18.656+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:18.656+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:18.656+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 30 0x2000~1000 2019-12-07T19:34:18.656+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 2fe41629 2019-12-07T19:34:18.656+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 30 0x1b000~1000 2019-12-07T19:34:18.656+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 41a1cd18 2019-12-07T19:34:18.656+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.656+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:18.656+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:18.706+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 10 osd.7 41 tick_without_osd_lock 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f690000/0x40000000/0x1680000000, data 0x9b11ad/0x96e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 25 osd.7 41 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674802+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 25 osd.7 41 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674781+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 25 osd.7 41 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674735+0000 last_rx_front 2019-12-07T19:34:15.674747+0000 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 25 osd.7 41 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674761+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 25 osd.7 41 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674643+0000 last_rx_front 2019-12-07T19:34:15.674689+0000 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 20 osd.7 41 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 20 osd.7 41 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 20 osd.7 41 scrub_load_below_threshold loadavg per cpu 0.04875 < max 5 = yes 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 20 osd.7 41 sched_scrub load_is_low=1 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 10 osd.7 41 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:08.239446+0000 > 2019-12-07T19:34:18.736779+0000 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 20 osd.7 41 sched_scrub done 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 10 osd.7 41 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 20 osd.7 41 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:18.735+0000 7f8f1fbbe700 10 osd.7 41 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:18.756+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.806+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.856+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.906+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:18.915+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:18.944+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 9 ==== osd_ping(ping e41 up_from 13 ping_stamp 2019-12-07T19:34:18.945007+0000/42.411331313s send_stamp 42.411331313s delta_ub 0.031101585s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841f200 con 0x558c54c5ac00 2019-12-07T19:34:18.944+0000 7f8f23c56700 20 osd.7 41 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030651718s,0.031101585s]) 2019-12-07T19:34:18.944+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e41 up_from 13 ping_stamp 2019-12-07T19:34:18.945007+0000/42.411331313s send_stamp 42.380679595s delta_ub 18446744073.678899898s) v5 -- 0x558c5841f400 con 0x558c54c5ac00 2019-12-07T19:34:18.944+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 9 ==== osd_ping(ping e41 up_from 13 ping_stamp 2019-12-07T19:34:18.945007+0000/42.411331313s send_stamp 42.411331313s delta_ub 0.031101585s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57519a00 con 0x558c54c5a400 2019-12-07T19:34:18.944+0000 7f8f24c58700 20 osd.7 41 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030497313s,0.031101585s]) 2019-12-07T19:34:18.944+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e41 up_from 13 ping_stamp 2019-12-07T19:34:18.945007+0000/42.411331313s send_stamp 42.380834000s delta_ub 18446744073.679054303s) v5 -- 0x558c53a5e800 con 0x558c54c5a400 2019-12-07T19:34:18.956+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.007+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.057+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.107+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.119+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:19.157+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.207+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.257+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.307+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.357+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.390+0000 7f8f21451700 10 osd.7 41 tick 2019-12-07T19:34:19.390+0000 7f8f21451700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:19.390+0000 7f8f21451700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:19.390+0000 7f8f21451700 20 osd.7 41 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:19.407+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.458+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.508+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.558+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1029274 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 26 ==== osd_map(41..42 src has 1..42) v4 ==== 1611+0+0 (crc 0 0 0) 0x558c57354240 con 0x558c5476d000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 osd.7 41 OSD::ms_dispatch: osd_map(41..42 src has 1..42) v4 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 osd.7 41 do_waiters -- start 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 osd.7 41 do_waiters -- finish 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 osd.7 41 _dispatch 0x558c57354240 osd_map(41..42 src has 1..42) v4 2019-12-07T19:34:19.603+0000 7f8f164bb700 3 osd.7 41 handle_osd_map epochs [41,42], i have 41, src has [1,42] 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 osd.7 41 handle_osd_map got inc map for epoch 42 2019-12-07T19:34:19.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3e08e826:::osdmap.41:0# 0x0~0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efc770 blob([0x96e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57efc7e0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efc850 blob([!~1000,0x96f000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57efc8c0 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3e08e826:::osdmap.41:0# 0x0~1464 = 5220 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 osd.7 41 add_map_bl 41 5220 bytes 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 osd.7 41 got_full_map 42, nothing requested 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 osd.7 41 handle_osd_map pg_num_history pg_num_history(e42 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {42={}} 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 osd.7 41 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e42 [1,42] lci=[13,42]) 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54d58600 seq 33 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cc17972f:::inc_osdmap.42:0# key 0x7f7fffffffffffffffcc1797'/!inc_osdmap.42!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cc17972f:::inc_osdmap.42:0# 0x0~275 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1208 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cc17972f:::inc_osdmap.42:0# 0x0~275 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~275 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd8b on front/back, now 0x0~1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58600 1 blobs 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x970000~1000/1000,1000,0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x970000~1000] 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550601c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efca10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550601c0 blob([0x970000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efca10 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c550601c0) get_ref 0x0~275 Blob(0x558c550601c0 blob([0x970000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efca10 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c550601c0) get_ref init 0x1000, 1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~275: 0x0~275 Blob(0x558c550601c0 blob([0x970000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x275) SharedBlob(0x558c57efca10 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efca28 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x275 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c888) dirty_range mark inline shard dirty 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cc17972f:::inc_osdmap.42:0# 0x0~275 = 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3088e826:::osdmap.42:0# key 0x7f7fffffffffffffff3088e8'&!osdmap.42!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3088e826:::osdmap.42:0# 0x0~1448 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1209 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3088e826:::osdmap.42:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58600 2 blobs 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x971000~2000/1000,2000,0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x971000~2000] 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efcaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efcb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efcaf0 blob([0x971000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efcb60 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57efcaf0) get_ref 0x0~1000 Blob(0x558c57efcaf0 blob([0x971000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efcb60 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57efcaf0) get_ref init 0x1000, 1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57efcaf0 blob([0x971000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57efcb60 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efcb78 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efcbd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efcc40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efcbd0 blob([!~1000,0x972000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efcc40 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57efcbd0) get_ref 0x1000~448 Blob(0x558c57efcbd0 blob([!~1000,0x972000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57efcc40 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c57efcbd0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c57efcbd0 blob([!~1000,0x972000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57efcc40 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efcc58 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738d008) dirty_range mark inline shard dirty 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3088e826:::osdmap.42:0# 0x0~1448 = 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58600 0 blobs 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58600 0 blobs 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58600 cost 23302 (4 ios * 4000 + 7302 bytes) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58600 onodes 0x558c53a91900,0x558c55276c80,0x558c5738c780,0x558c5738cf00 shared_blobs 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c888) update #-1:cc17972f:::inc_osdmap.42:0# 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c888) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cc17972f:::inc_osdmap.42:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738d008) update #-1:3088e826:::osdmap.42:0# 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738d008) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3088e826:::osdmap.42:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58600 allocated 0x[970000~3000] released 0x[] 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 freelist allocate 0x970000~3000 2019-12-07T19:34:19.603+0000 7f8f164bb700 20 freelist _xor first_key 0x900000 last_key 0x900000 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 prepare 2019-12-07T19:34:19.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58600 2019-12-07T19:34:19.604+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 aio_wait 2019-12-07T19:34:19.604+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58600 2019-12-07T19:34:19.604+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 io_done 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738c780 had 1 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738cf00 had 1 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8848 < min_flush_size 524288 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8848 < min_flush_size 524288 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14bfea~2290 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14bfea 0x14bfea~2290 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11bfea 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfea 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x27a and padding block with 0xd86 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14e27a 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000178079s (0.000052999s flush + 0.000125080s kv commit) 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58600> 2019-12-07T19:34:19.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c57ee1cc0> 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 kv_submitted 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58600 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54d58600 osr 0x558c54ea4b40 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7 41 _committed_osd_maps 42..42 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7 41 advance to epoch 42 (<= last 42 <= newest_map 42) 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 31 0x1b000~1000 crc 6ec1381f 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 31 0x2000~1000 crc e7455c9f 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 deferred_cleanup 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd200 onodes 0x558c53a91900,0x558c55276c80,0x558c57efa500,0x558c58093180 2019-12-07T19:34:19.604+0000 7f8f141a1700 7 osd.7 42 consume_map version 42 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314638 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64ea0 space 0x558c57314638 0x0~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.8s5 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bd518 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64f30 space 0x558c575bd518 0x0~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.0s3 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efc398 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64e10 space 0x558c57efc398 0x1000~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.1s1 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.9s4 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd200 done 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.as5 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 deferred_cleanup 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.2s0 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd200 [] 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.2s4 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.bs3 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.bs1 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.3s4 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.cs0 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.4s0 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 deferred_cleanup 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.5s4 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.ds2 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 0x558c53a91900,0x558c55276c80,0x558c57efa280,0x558c57efac80 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.es4 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bdb38 in 0x558c53a59340) _finish_write discard buffer(0x558c57e64c60 space 0x558c575bdb38 0x0~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.6s2 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f3dd8 in 0x558c53a59340) _finish_write discard buffer(0x558c579a73b0 space 0x558c577f3dd8 0x0~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.fs5 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 identify_splits_and_merges 1.7s4 e41 to e42 pg_nums {18=16} 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efc558 in 0x558c53a59340) _finish_write discard buffer(0x558c579a7320 space 0x558c57efc558 0x1000~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 42 prune_pg_created 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:0.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 deferred_cleanup 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:1.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:2.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:3.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 deferred_cleanup 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:4.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:5.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 0x558c53a91900,0x558c55276c80,0x558c5738c500,0x558c57efaa00 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efc6a8 in 0x558c53a59340) _finish_write discard buffer(0x558c54e14cf0 space 0x558c57efc6a8 0x0~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:6.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efc7f8 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4f200 space 0x558c57efc7f8 0x0~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7:7.consume_map 42 (was 41) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efc8d8 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4f560 space 0x558c57efc8d8 0x1000~1000 writing nocache) 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 deferred_queued 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:34:19.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.604+0000 7f8f141a1700 10 osd.7 42 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:19.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.0s3 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c57380000 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.9s4 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.as5 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c5500d000 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.2s4 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c5736a000 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c57e8b000 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 41 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.bs3 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 41 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.cs0 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:19.604+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c57676000 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.5s4 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.ds2 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.6s2 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 41 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] state: Active advmap 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f141a1700 15 osd.7 42 enqueue_peering_evt 1.7s4 epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.604+0000 7f8f141a1700 7 osd.7 42 activate_map version 42 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:19.604+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f0918b700 20 osd.7 42 agent_entry empty queue 2019-12-07T19:34:19.604+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c57522000 2019-12-07T19:34:19.604+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cc17972f:::inc_osdmap.42:0# 0x0~0 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~275 size 0x275 (629) 2019-12-07T19:34:19.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f08189700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:19.604+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550601c0 blob([0x970000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x275) SharedBlob(0x558c57efca10 sbid 0x0)) need 0x0~275 cache has 0x[0~275] 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c57383000 2019-12-07T19:34:19.604+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cc17972f:::inc_osdmap.42:0# 0x0~275 = 629 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 42 add_map_inc_bl 42 629 bytes 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c54d98000 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c54d9b000 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] state: Active: handling ActMap 2019-12-07T19:34:19.604+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(42..42 src has 1..42) v4 -- 0x558c575d4240 con 0x558c54c64000 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] Active: kicking snap trim 2019-12-07T19:34:19.604+0000 7f8f04982700 10 osd.7 42 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 41 -> 42 (shared) 2019-12-07T19:34:19.604+0000 7f8f06986700 7 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] state: handle ActMap primary 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs3 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:19.604+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.8s0 1.8s0 (query:42 sent:42 1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),7(5) intervals=([39,40] acting 1(0),2(3),3(4),4(1),5(2)))) e42/42) v1 -- 0x558c576dee00 con 0x558c54c64000 2019-12-07T19:34:19.604+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] publish_stats_to_osd 41:127 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:19.604+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:19.604+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs3_head options 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:42 sent:42 1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(1),4(2),5(3)))) e42/42) v1 -- 0x558c57378e00 con 0x558c54c64000 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(42..42 src has 1..42) v4 -- 0x558c575d46c0 con 0x558c54c19800 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.004521054s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c54d5c000 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 42 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 41 -> 42 (shared) 2019-12-07T19:34:19.604+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.5s0 1.5s0 (query:42 sent:42 1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(4),1(1),2(0),3(2),4(3),5(5),7(4) intervals=([39,40] acting 1(1),2(0),3(2),4(3),5(5)))) e42/42) v1 -- 0x558c57ed5800 con 0x558c54c19800 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c55141000 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 41 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 41 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(42..42 src has 1..42) v4 -- 0x558c57580480 con 0x558c54c5b400 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 42 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 41 -> 42 (shared) 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004935355s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:42 sent:42 1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) 3->0 ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e42/42) v1 -- 0x558c54e05500 con 0x558c54c5b400 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c575ef000 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 41 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:19.604+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Peering advmap 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] adjust_need_up_thru now 41, need_up_thru now false 2019-12-07T19:34:19.604+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(42..42 src has 1..42) v4 -- 0x558c576418c0 con 0x558c54c64000 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.ds0 1.ds0 (query:42 sent:42 1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) 2->0 ([18,40] all_participants=0(5),1(1),2(0),3(3),4(4),5(5),7(2) intervals=([39,40] acting 1(1),2(0),3(3),4(4),7(2)))) e42/42) v1 -- 0x558c54e04700 con 0x558c54c19800 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.604+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:42 sent:42 1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(3),1(0),2(2),3(4),3(5),4(1),5(3),5(4),7(5) intervals=([28,37] acting 0(3),1(0),2(2),3(5),4(1),5(4)))) e42/42) v1 -- 0x558c548a1500 con 0x558c54c64000 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0297e700 7 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f0297e700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] publish_stats_to_osd 41:78 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.604+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f0297e700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:19.605946+0000 3 0.000339 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] state: Leaving Peering 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f0297e700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering 2019-12-07T19:34:19.605963+0000 0 0.000000 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0297e700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Started/Primary/Active 2019-12-07T19:34:19.604+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] state: In Active, about to call activate 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:19.604+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(42..42 src has 1..42) v4 -- 0x558c57303d40 con 0x558c54c5bc00 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 37'4 -> 37'4 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate - purged_snaps [a~1] removed_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Peering advmap 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 42 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 41 -> 42 (shared) 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] adjust_need_up_thru now 41, need_up_thru now false 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:42 sent:42 1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(1),1(4),2(1),2(5),3(0),4(3),5(2),7(5) intervals=([28,37] acting 0(1),1(4),2(5),3(0),4(3),5(2)))) e42/42) v1 -- 0x558c58134700 con 0x558c54c5bc00 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:19.604+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:42 sent:42 1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(1),1(4),2(2),3(3),4(0),5(5),7(1),7(3) intervals=([28,37] acting 0(1),1(4),2(2),4(0),5(5),7(3)))) e42/42) v1 -- 0x558c576e1100 con 0x558c54c5b400 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041434037s) 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 41 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c5513e000 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.1(2) is up to date, queueing in pending_activators 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.1(2) 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.012689290s) uptodate 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) 2019-12-07T19:34:19.604+0000 7f8f03980700 7 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 41 -> 42 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.2(4) is up to date, queueing in pending_activators 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.008595348s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.2(4) 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.009743360s) uptodate 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 41 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.3(1) 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 2019-12-07T19:34:19.604+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.3(1) is up to date, queueing in pending_activators 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.604+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:42 sent:42 1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) 4->0 ([28,40] all_participants=0(3),1(0),2(5),3(2),4(1),5(3),7(4) intervals=([39,40] acting 1(0),2(5),3(2),4(1),7(4)))) e42/42) v1 -- 0x558c58134e00 con 0x558c54c64000 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.3(1) 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) uptodate 2019-12-07T19:34:19.604+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] publish_stats_to_osd 41:93 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.604+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.604+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) 2019-12-07T19:34:19.604+0000 7f8f03980700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:19.606092+0000 2 0.000188 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.4(5) is up to date, queueing in pending_activators 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] state: Leaving Peering 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.4(5) 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.004537266s) uptodate 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.604+0000 7f8f03980700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 peering mbc={} ps=[a~1]] exit Started/Primary/Peering 2019-12-07T19:34:19.606109+0000 0 0.000000 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.5(3) is up to date, queueing in pending_activators 2019-12-07T19:34:19.604+0000 7f8f03980700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] enter Started/Primary/Active 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.005422919s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.5(3) 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=15.014313011s) uptodate 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] state: In Active, about to call activate 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 1(2) 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 2(4) 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 3(1) 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 29'15 -> 29'15 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 4(5) 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 5(3) 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate - purged_snaps [a~1] removed_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 7(0) 2019-12-07T19:34:19.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 unknown mbc={} ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] release_backoffs [1:20000000::::0,1:30000000::::head) 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:42 sent:42 1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) 4->0 ([28,40] all_participants=0(5),1(1),2(2),3(5),4(0),5(3),7(4) intervals=([39,40] acting 1(1),2(2),4(0),5(3),7(4)))) e42/42) v1 -- 0x558c57392a00 con 0x558c54c5b400 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.604+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041594632s) 2019-12-07T19:34:19.604+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(42..42 src has 1..42) v4 -- 0x558c5738b680 con 0x558c54c5bc00 2019-12-07T19:34:19.604+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.604+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.1(2) 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:42 sent:42 1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),7(1) intervals=([39,40] acting 1(5),2(2),3(0),4(3),5(4)))) e42/42) v1 -- 0x558c57ed3500 con 0x558c54c5bc00 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.1(2) is up to date, queueing in pending_activators 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c5500a000 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.1(2) 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) uptodate 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.2(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.2(4) is up to date, queueing in pending_activators 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.2(4) 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) uptodate 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 41 -> 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.3(5) 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 41 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.3(5) is up to date, queueing in pending_activators 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.3(5) 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.010911057s) uptodate 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.605+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.4(1) is up to date, queueing in pending_activators 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.4(1) 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.006015431s) uptodate 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.5(3) 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.009093588s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.605+0000 7f8f0297e700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 41:79 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.5(3) is up to date, queueing in pending_activators 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate peer osd.5(3) 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=15.011727212s) uptodate 2019-12-07T19:34:19.605+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f0297e700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] enter Started/Primary/Active/Activating 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 1(2) 2019-12-07T19:34:19.605+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.605+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 2(4) 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 3(5) 2019-12-07T19:34:19.605+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:42 sent:42 1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) 4->0 ([28,40] all_participants=0(3),1(0),2(3),3(5),4(1),5(2),7(4) intervals=([39,40] acting 1(0),3(5),4(1),5(2),7(4)))) e42/42) v1 -- 0x558c57ed3c00 con 0x558c54c64000 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 4(1) 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 5(3) 2019-12-07T19:34:19.605+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] activate setting up missing_loc from shard 7(0) 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 unknown mbc={} ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:19.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) e42/42) v1 -- 0x558c576e0300 con 0x558c54c64000 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:19.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) e42/42) v1 -- 0x558c57392300 con 0x558c54c19800 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) e42/42) v1 -- 0x558c57393800 con 0x558c54c5bc00 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) e42/42) v1 -- 0x558c58124300 con 0x558c54c5b400 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(42..42 src has 1..42) v4 -- 0x558c57e03d40 con 0x558c54c19c00 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 osd.7 42 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 41 -> 42 (shared) 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) e42/42) v1 -- 0x558c574e5500 con 0x558c54c19c00 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c548f7b00 seq 26 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.605+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 5585 (1 ios * 4000 + 1585 bytes) 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes shared_blobs 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:19.605+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1429 < min_flush_size 524288 2019-12-07T19:34:19.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1452 < min_flush_size 524288 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1452 < min_flush_size 524288 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14e27a~5ac to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14e27a 0x14e27a~5ac to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 41:94 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11e27a 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x27a 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x826 and padding block with 0x7da 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.605+0000 7f8f03980700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] enter Started/Primary/Active/Activating 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.605+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 27 ==== osd_map(42..42 src has 1..42) v4 ==== 673+0+0 (crc 0 0 0) 0x558c57303d40 con 0x558c5476d000 2019-12-07T19:34:19.605+0000 7f8f164bb700 20 osd.7 42 OSD::ms_dispatch: osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.605+0000 7f8f164bb700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.605+0000 7f8f164bb700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.605+0000 7f8f164bb700 20 osd.7 42 _dispatch 0x558c57303d40 osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.605+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 -- 0x558c548a3800 con 0x558c54c64000 2019-12-07T19:34:19.605+0000 7f8f164bb700 3 osd.7 42 handle_osd_map epochs [42,42], i have 42, src has [1,42] 2019-12-07T19:34:19.605+0000 7f8f164bb700 10 osd.7 42 no new maps here, dropping 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14e826 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.605+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 -- 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000142435s (0.000000130s flush + 0.000142305s kv commit) 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.605+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 -- 0x558c576df500 con 0x558c54c5bc00 2019-12-07T19:34:19.605+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 -- 0x558c576dfc00 con 0x558c54c5b400 2019-12-07T19:34:19.605+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 -- 0x558c58137100 con 0x558c54c19c00 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7b00> 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.605+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:19.605+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57ef9800 seq 15 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:19.605+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.605+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:19.605+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:19.605+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:19.605+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9800 cost 5618 (1 ios * 4000 + 1618 bytes) 2019-12-07T19:34:19.605+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9800 onodes shared_blobs 2019-12-07T19:34:19.605+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9800 allocated 0x[] released 0x[] 2019-12-07T19:34:19.605+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 prepare 2019-12-07T19:34:19.605+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9800 2019-12-07T19:34:19.605+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 io_done 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 NullEvt) prio 255 cost 10 e42) pg 0x558c5731a000 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f06986700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map: 42 2019-12-07T19:34:19.605+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s4 41 -> 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 41 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:19.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:19.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:19.605+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s4_head options 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 42 advance_pg new pool opts old pool opts 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] handle_activate_map: 42 2019-12-07T19:34:19.605+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 42 2019-12-07T19:34:19.605+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: _activate_committed 42 peer_activated now 7(0) last_interval_started 39 last_epoch_started 40 same_interval_since 41 2019-12-07T19:34:19.605+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 NullEvt 2019-12-07T19:34:19.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 42 send_incremental_map 41 -> 42 to 0x558c53ae5400 v2:172.21.15.109:6826/36939 2019-12-07T19:34:19.605+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(42..42 src has 1..42) v4 -- 0x558c57641440 con 0x558c53ae5400 2019-12-07T19:34:19.605+0000 7f8f03980700 10 osd.7 42 maybe_share_map con 0x558c53ae5400 v2:172.21.15.109:6826/36939 map epoch 41 -> 42 (shared) 2019-12-07T19:34:19.605+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_notify2(1.2s0 1.2s0 (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e42/42) v1 -- 0x558c58137800 con 0x558c53ae5400 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1462 < min_flush_size 524288 2019-12-07T19:34:19.605+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== pg_notify2(1.2s0 1.2s0 (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e42/42) v1 ==== 0+0+0 (unknown 0 0 0) 0x558c58137800 con 0x558c53ae5400 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1485 < min_flush_size 524288 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1485 < min_flush_size 524288 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14e826~5cd to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14e826 0x14e826~5cd to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd_map(42..42 src has 1..42) v4 ==== 0+0+0 (unknown 0 0 0) 0x558c57641440 con 0x558c53ae5400 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11e826 2019-12-07T19:34:19.605+0000 7f8f1199c700 20 osd.7 42 OSD::ms_dispatch: osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x826 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.605+0000 7f8f1199c700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdf3 and padding block with 0x20d 2019-12-07T19:34:19.605+0000 7f8f1199c700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.605+0000 7f8f1199c700 20 osd.7 42 _dispatch 0x558c57641440 osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.605+0000 7f8f1119b700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 7(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.605+0000 7f8f1199c700 3 osd.7 42 handle_osd_map epochs [42,42], i have 42, src has [1,42] 2019-12-07T19:34:19.605+0000 7f8f1199c700 10 osd.7 42 no new maps here, dropping 2019-12-07T19:34:19.605+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 7(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 7(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 7(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.605+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 7(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.605+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 7(4), already have info from that osd, ignoring 2019-12-07T19:34:19.605+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14edf3 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.605+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127192s (0.000000093s flush + 0.000127099s kv commit) 2019-12-07T19:34:19.605+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9800> 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 kv_submitted 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9800 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 finishing 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9800 onodes 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9800 done 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9800 [] 2019-12-07T19:34:19.605+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.605+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.605+0000 7f8f07988700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 158 ==== pg_notify2(1.4s0 1.4s0 (query:42 sent:42 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e42/42) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c576df500 con 0x558c54c5bc00 2019-12-07T19:34:19.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.606+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.606+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: _activate_committed 42 peer_activated now 7(0) last_interval_started 39 last_epoch_started 40 same_interval_since 41 2019-12-07T19:34:19.606+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.606+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 3(1) notify: (query:42 sent:42 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.606+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 3(1) notify: (query:42 sent:42 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.606+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 159 ==== osd_map(42..42 src has 1..42) v4 ==== 673+0+0 (unknown 0 0 0) 0x558c5738b680 con 0x558c54c5bc00 2019-12-07T19:34:19.606+0000 7f8f1199c700 20 osd.7 42 OSD::ms_dispatch: osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.606+0000 7f8f1199c700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.606+0000 7f8f1199c700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.606+0000 7f8f1199c700 20 osd.7 42 _dispatch 0x558c5738b680 osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.606+0000 7f8f1199c700 3 osd.7 42 handle_osd_map epochs [42,42], i have 42, src has [1,42] 2019-12-07T19:34:19.606+0000 7f8f1199c700 10 osd.7 42 no new maps here, dropping 2019-12-07T19:34:19.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 3(1) notify: (query:42 sent:42 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 3(1) notify: (query:42 sent:42 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 3(1) notify: (query:42 sent:42 1.4s1( v 37'4 (0'0,37'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 1->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 3(1), already have info from that osd, ignoring 2019-12-07T19:34:19.606+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:19.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 160 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c576df500 con 0x558c54c5bc00 2019-12-07T19:34:19.606+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064307871s int 16s) 2019-12-07T19:34:19.606+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064307871s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 161 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c576df500 con 0x558c54c5bc00 2019-12-07T19:34:19.606+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064164481s int 16s) 2019-12-07T19:34:19.606+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064164481s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 162 ==== pg_notify2(1.2s0 1.2s0 (query:42 sent:42 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e42/42) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558c576df500 con 0x558c54c5bc00 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064164481s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064164481s int 16s)) prio 255 cost 10 e42) pg 0x558c57383000 2019-12-07T19:34:19.606+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 3(5) notify: (query:42 sent:42 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.606+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 3(5) notify: (query:42 sent:42 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064164481s int 16s) 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] state: got info from osd.3(0) 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 3(5) notify: (query:42 sent:42 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.064164481s int 16s) 2019-12-07T19:34:19.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 3(5) notify: (query:42 sent:42 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.021880519s -> ruub 59.042283962s 2019-12-07T19:34:19.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064307871s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.042283962s 2019-12-07T19:34:19.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064307871s int 16s)) prio 255 cost 10 e42) pg 0x558c5736a000 2019-12-07T19:34:19.606+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 3(5) notify: (query:42 sent:42 1.2s5( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 5->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.606+0000 7f8f0117b700 5 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.608028+0000 6 0.000083 2019-12-07T19:34:19.606+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.064307871s int 16s) 2019-12-07T19:34:19.606+0000 7f8f0117b700 5 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.606+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 3(5), already have info from that osd, ignoring 2019-12-07T19:34:19.606+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] state: got info from osd.3(0) 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.606+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.606+0000 7f8f0117b700 5 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.606+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.064307871s int 16s) 2019-12-07T19:34:19.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.606+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.021880519s -> ruub 59.042427352s 2019-12-07T19:34:19.606+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.042427352s 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 29'12 -> 29'12 2019-12-07T19:34:19.606+0000 7f8f04181700 5 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.608064+0000 6 0.000083 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.606+0000 7f8f04181700 5 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.606+0000 7f8f04181700 5 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:19.606+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57b95500 seq 11 2019-12-07T19:34:19.606+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 29'9 -> 29'9 2019-12-07T19:34:19.606+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:19.606+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:19.606+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95500 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95500 onodes shared_blobs 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 prepare 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95500 2019-12-07T19:34:19.606+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 io_done 2019-12-07T19:34:19.606+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:19.606+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c54ea1200 seq 9 2019-12-07T19:34:19.606+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:19.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.606+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:19.606+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:19.606+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:19.606+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:19.606+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:19.606+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:34:19.606+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.606+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:19.606+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:19.606+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:19.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 182 ==== pg_notify2(1.4s0 1.4s0 (query:42 sent:42 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.011464467s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e42/42) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c5b400 2019-12-07T19:34:19.607+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 4(5) notify: (query:42 sent:42 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.011464467s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.607+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 4(5) notify: (query:42 sent:42 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.011464467s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 4(5) notify: (query:42 sent:42 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.011464467s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 4(5) notify: (query:42 sent:42 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.011464467s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.607+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 4(5) notify: (query:42 sent:42 1.4s5( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.011464467s) 5->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.607+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 4(5), already have info from that osd, ignoring 2019-12-07T19:34:19.607+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.607+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:19.607+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:19.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2560 < min_flush_size 524288 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:19.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2583 < min_flush_size 524288 2019-12-07T19:34:19.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2583 < min_flush_size 524288 2019-12-07T19:34:19.607+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14edf3~a17 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.607+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14edf3 0x14edf3~a17 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11edf3 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdf3 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.607+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x80a and padding block with 0x7f6 2019-12-07T19:34:19.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 184 ==== pg_notify2(1.2s0 1.2s0 (query:42 sent:42 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e42/42) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c5b400 2019-12-07T19:34:19.607+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 4(1) notify: (query:42 sent:42 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.607+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 4(1) notify: (query:42 sent:42 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 185 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c5b400 2019-12-07T19:34:19.607+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.056742504s int 16s) 2019-12-07T19:34:19.607+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.056742504s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 4(1) notify: (query:42 sent:42 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 4(1) notify: (query:42 sent:42 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.607+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 4(1) notify: (query:42 sent:42 1.2s1( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 1->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.607+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 4(1), already have info from that osd, ignoring 2019-12-07T19:34:19.607+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.607+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 163 ==== pg_info2(1.4s0 1.4s0 1.4s1( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) e42/42) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c576df500 con 0x558c54c5bc00 2019-12-07T19:34:19.607+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(1) info: 1.4s1( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.607+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(1) info: 1.4s1( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.607+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 164 ==== pg_info2(1.2s0 1.2s0 1.2s5( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c576df500 con 0x558c54c5bc00 2019-12-07T19:34:19.607+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(5) info: 1.2s5( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.607+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(5) info: 1.2s5( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.607+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.056742504s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.607+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.056742504s int 16s)) prio 255 cost 10 e42) pg 0x558c5513e000 2019-12-07T19:34:19.607+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.056742504s int 16s) 2019-12-07T19:34:19.607+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] state: got info from osd.4(0) 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.607+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.056742504s int 16s) 2019-12-07T19:34:19.607+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] peer_clock_delta_lb 0.013502867s -> ruub 59.043239637s 2019-12-07T19:34:19.607+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.043239637s 2019-12-07T19:34:19.607+0000 7f8f07187700 5 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.608914+0000 6 0.000127 2019-12-07T19:34:19.607+0000 7f8f07187700 5 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.607+0000 7f8f07187700 5 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.607+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown NOTIFY pruub 57.054502621s@ mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.607+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] activate - no missing, moving last_complete 37'13 -> 37'13 2019-12-07T19:34:19.607+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.607+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.607+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:19.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54ddec00 seq 45 2019-12-07T19:34:19.607+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:19.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:19.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:19.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:19.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:19.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddec00 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:19.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddec00 onodes shared_blobs 2019-12-07T19:34:19.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddec00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 prepare 2019-12-07T19:34:19.607+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddec00 2019-12-07T19:34:19.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 77 ==== pg_notify2(1.2s0 1.2s0 (query:42 sent:42 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e42/42) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c19c00 2019-12-07T19:34:19.607+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 io_done 2019-12-07T19:34:19.607+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.607+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 5(3) notify: (query:42 sent:42 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.607+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 5(3) notify: (query:42 sent:42 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(1) info: 1.4s1( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(1) info: 1.4s1( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.607+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(1) info: 1.4s1( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:19.607+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041434037s) 2019-12-07T19:34:19.608+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.3(1) activated and committed 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.608+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 183 ==== osd_map(42..42 src has 1..42) v4 ==== 673+0+0 (unknown 0 0 0) 0x558c576418c0 con 0x558c54c5b400 2019-12-07T19:34:19.608+0000 7f8f1199c700 20 osd.7 42 OSD::ms_dispatch: osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.608+0000 7f8f1199c700 20 osd.7 42 _dispatch 0x558c576418c0 osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.608+0000 7f8f1199c700 3 osd.7 42 handle_osd_map epochs [42,42], i have 42, src has [1,42] 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 no new maps here, dropping 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.608+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 78 ==== osd_map(42..42 src has 1..42) v4 ==== 673+0+0 (unknown 0 0 0) 0x558c57e03d40 con 0x558c54c19c00 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.608+0000 7f8f1199c700 20 osd.7 42 OSD::ms_dispatch: osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.608+0000 7f8f1199c700 20 osd.7 42 _dispatch 0x558c57e03d40 osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.608+0000 7f8f1199c700 3 osd.7 42 handle_osd_map epochs [42,42], i have 42, src has [1,42] 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 no new maps here, dropping 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.608+0000 7f8f0297e700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:80 2019-12-07T19:34:19.608+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 107 ==== pg_notify2(1.4s0 1.4s0 (query:42 sent:42 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009911271s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e42/42) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:19.608+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 106 ==== osd_map(42..42 src has 1..42) v4 ==== 673+0+0 (crc 0 0 0) 0x558c575d46c0 con 0x558c54c19800 2019-12-07T19:34:19.608+0000 7f8f1199c700 20 osd.7 42 OSD::ms_dispatch: osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.608+0000 7f8f1199c700 20 osd.7 42 _dispatch 0x558c575d46c0 osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.608+0000 7f8f1199c700 3 osd.7 42 handle_osd_map epochs [42,42], i have 42, src has [1,42] 2019-12-07T19:34:19.608+0000 7f8f1199c700 10 osd.7 42 no new maps here, dropping 2019-12-07T19:34:19.608+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 2(4) notify: (query:42 sent:42 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009911271s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.608+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 2(4) notify: (query:42 sent:42 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009911271s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14f80a 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000953144s (0.000000147s flush + 0.000952997s kv commit) 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:19.608+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 108 ==== pg_notify2(1.2s0 1.2s0 (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e42/42) v1 ==== 1237+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.608+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 2(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.608+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 2(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14f80a~50c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14f80a 0x14f80a~50c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11f80a 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x80a 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd16 and padding block with 0x2ea 2019-12-07T19:34:19.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 2(4) notify: (query:42 sent:42 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009911271s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 2(4) notify: (query:42 sent:42 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009911271s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95500,0x558c54ea1200> 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 kv_submitted 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95500 2019-12-07T19:34:19.608+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 2(4) notify: (query:42 sent:42 1.4s4( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009911271s) 4->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 finishing 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95500 onodes 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95500 done 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95500 [] 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.608+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:19.608+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 2(4), already have info from that osd, ignoring 2019-12-07T19:34:19.608+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:19.608+0000 7f8f05183700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:19.608+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.608+0000 7f8f05183700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x14fd16 2019-12-07T19:34:19.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c57383000 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119188s (0.000000106s flush + 0.000119082s kv commit) 2019-12-07T19:34:19.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddec00> 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.608+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 kv_submitted 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddec00 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 finishing 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddec00 onodes 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddec00 done 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddec00 [] 2019-12-07T19:34:19.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.608+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 crt=29'12 unknown mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.608+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1030758 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.608+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(5) info: 1.2s5( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(5) info: 1.2s5( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.608+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.fs0 1.fs0 1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c58135500 con 0x558c54c5bc00 2019-12-07T19:34:19.608+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(5) info: 1.2s5( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:19.608+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.608+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f08189700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:19.608+0000 7f8f08189700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c5736a000 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:19.608+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041594632s) 2019-12-07T19:34:19.608+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.608+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 crt=29'9 unknown mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.608+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.3(5) activated and committed 2019-12-07T19:34:19.608+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.1s0 1.1s0 1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c576ba300 con 0x558c54c5bc00 2019-12-07T19:34:19.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f07187700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c5513e000 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.608+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 crt=37'13 unknown pruub 57.054502621s@ mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.608+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.608+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 pruub=14.012933555s) e42/42) v1 -- 0x558c57378000 con 0x558c54c5b400 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.608+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:95 2019-12-07T19:34:19.608+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 5(3) notify: (query:42 sent:42 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 5(3) notify: (query:42 sent:42 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.608+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 5(3) notify: (query:42 sent:42 1.2s3( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 3->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.608+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 5(3), already have info from that osd, ignoring 2019-12-07T19:34:19.608+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 2(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 2(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.608+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 2(4) notify: (query:42 sent:42 1.2s4( v 29'15 (0'0,29'15] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 4->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.608+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 2(4), already have info from that osd, ignoring 2019-12-07T19:34:19.608+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:19.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 79 ==== pg_info2(1.2s0 1.2s0 1.2s3( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c19c00 2019-12-07T19:34:19.608+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.2s3( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.608+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.2s3( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.2s3( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.2s3( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.608+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.2s3( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 80 ==== pg_notify2(1.4s0 1.4s0 (query:42 sent:42 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009441352s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e42/42) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c19c00 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 59.041594632s 2019-12-07T19:34:19.608+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 5(3) notify: (query:42 sent:42 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009441352s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.608+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 5(3) notify: (query:42 sent:42 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009441352s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 5(3) notify: (query:42 sent:42 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009441352s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 5(3) notify: (query:42 sent:42 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009441352s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041594632s) 2019-12-07T19:34:19.608+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.5(3) activated and committed 2019-12-07T19:34:19.608+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 5(3) notify: (query:42 sent:42 1.4s3( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.009441352s) 3->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 186 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c5b400 2019-12-07T19:34:19.608+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 5(3), already have info from that osd, ignoring 2019-12-07T19:34:19.608+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.608+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.057851957s int 16s) 2019-12-07T19:34:19.608+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.057851957s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.057851957s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.057851957s int 16s)) prio 255 cost 10 e42) pg 0x558c575ef000 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.057851957s int 16s) 2019-12-07T19:34:19.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 187 ==== pg_info2(1.4s0 1.4s0 1.4s5( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.011464467s) e42/42) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c5b400 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] state: got info from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.011464467s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.608+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.011464467s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.057851957s int 16s) 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.013502867s -> ruub 59.044349090s 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.044349090s 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 188 ==== pg_info2(1.2s0 1.2s0 1.2s1( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1173+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c5b400 2019-12-07T19:34:19.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.610093+0000 6 0.000084 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.608+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.608+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.608+0000 7f8f0317f700 5 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 21'12 -> 21'12 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c574ff500 seq 10 2019-12-07T19:34:19.608+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 81 ==== pg_info2(1.4s0 1.4s0 1.4s3( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009441352s) e42/42) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c548a3800 con 0x558c54c19c00 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.608+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:19.608+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009441352s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:19.608+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009441352s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:19.608+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:96 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff500 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff500 onodes shared_blobs 2019-12-07T19:34:19.608+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 prepare 2019-12-07T19:34:19.608+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff500 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.608+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 io_done 2019-12-07T19:34:19.608+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.609+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(1) info: 1.2s1( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 59.041594632s 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 746 < min_flush_size 524288 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 59.041594632s 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x14fd16~52d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x14fd16 0x14fd16~52d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:17.815332+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _allocate len 0x243 from 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _allocate len 0x243 from 1 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd280000 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd280000~10000/10000,10000,acd280000 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041594632s) 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x160000 to include allocated 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 43 (was clean) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x11fd16 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd16 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x243 and padding block with 0xdbd 2019-12-07T19:34:19.609+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.4(1) activated and committed 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.011464467s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x160000 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.011464467s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.609+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(5) info: 1.4s5( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.011464467s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041434037s) 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.4(5) activated and committed 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x150243 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.609+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:97 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.609+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.609+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.609+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 109 ==== pg_info2(1.4s0 1.4s0 1.4s4( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009911271s) e42/42) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.609+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009911271s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (43) on file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]), flushing log 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:19.609+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009911271s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 43 len 0x49 crc 0x197012f8) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x2a000~1000 to file(ino 1 size 0x2a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x2a000 0x2a000~1000 to file(ino 1 size 0x2a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x2b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2a000 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.609+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 110 ==== pg_info2(1.2s0 1.2s0 1.2s4( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.609+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:81 2019-12-07T19:34:19.609+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.2s4( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.609+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.2s4( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009441352s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009441352s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x2b000 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.2s4( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 5(3) info: 1.4s3( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009441352s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.2s4( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.609+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 111 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.5s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.081729084s int 16s) 2019-12-07T19:34:19.609+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.081729084s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.609+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.2s4( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000029 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 59.041594632s 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 59.041594632s 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.081729084s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041434037s) 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 59.041594632s 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.081729084s int 16s)) prio 255 cost 10 e42) pg 0x558c57522000 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:19.609+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.5(3) activated and committed 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041594632s) 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.081729084s int 16s) 2019-12-07T19:34:19.609+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.2(4) activated and committed 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 43 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] state: got info from osd.2(0) 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.081729084s int 16s) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000367581s (0.000000119s flush + 0.000367462s kv commit) 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.035982138s -> ruub 59.045746946s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.045746946s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 5 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.610588+0000 6 0.000087 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 5 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 5 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 27'8 -> 27'8 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c577fcf00 seq 8 2019-12-07T19:34:19.609+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:98 2019-12-07T19:34:19.609+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff500> 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 kv_submitted 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff500 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 finishing 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff500 onodes 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff500 done 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:19.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff500 [] 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcf00 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcf00 onodes shared_blobs 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcf00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.609+0000 7f8f07187700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.609+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:82 2019-12-07T19:34:19.609+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.609+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 prepare 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcf00 2019-12-07T19:34:19.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 io_done 2019-12-07T19:34:19.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009911271s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c575ef000 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009911271s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(4) info: 1.4s4( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.009911271s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.609+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 crt=21'12 unknown mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c58137800 con 0x558c54c5b400 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 59.041434037s 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 0s (sent 59.041434037s) 2019-12-07T19:34:19.609+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.2(4) activated and committed 2019-12-07T19:34:19.609+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x150243~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x150243 0x150243~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x120243 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x243 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x74f and padding block with 0x8b1 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.609+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:83 2019-12-07T19:34:19.609+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15074f 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.609+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000074121s (0.000000122s flush + 0.000073999s kv commit) 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcf00> 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 kv_submitted 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcf00 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 finishing 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcf00 onodes 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcf00 done 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcf00 [] 2019-12-07T19:34:19.609+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.609+0000 7f8f06185700 15 osd.7 42 enqueue_peering_evt 1.5s4 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.609+0000 7f8f06185700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c57522000 2019-12-07T19:34:19.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.609+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.609+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 crt=27'8 unknown mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.609+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.609+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.5s0 1.5s0 1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c54e06a00 con 0x558c54c19800 2019-12-07T19:34:19.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.610+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 112 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) e42/42) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:19.610+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.ds2 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) pg_lease(ru 0s ub 59.082500675s int 16s) 2019-12-07T19:34:19.610+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) pg_lease(ru 0s ub 59.082500675s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) pg_lease(ru 0s ub 59.082500675s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) pg_lease(ru 0s ub 59.082500675s int 16s)) prio 255 cost 10 e42) pg 0x558c55141000 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) pg_lease(ru 0s ub 59.082500675s int 16s) 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] state: got info from osd.2(0) 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.082500675s int 16s) 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] peer_clock_delta_lb 0.035982138s -> ruub 59.046518537s 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.046518537s 2019-12-07T19:34:19.610+0000 7f8f0217d700 5 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.611446+0000 6 0.000079 2019-12-07T19:34:19.610+0000 7f8f0217d700 5 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.610+0000 7f8f0217d700 5 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown NOTIFY pruub 57.054473552s@ mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] activate - no missing, moving last_complete 21'6 -> 21'6 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c548f6300 seq 29 2019-12-07T19:34:19.610+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes shared_blobs 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:19.610+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15074f~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15074f 0x15074f~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12074f 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x74f 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc5b and padding block with 0x3a5 2019-12-07T19:34:19.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x150c5b 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129988s (0.000000109s flush + 0.000129879s kv commit) 2019-12-07T19:34:19.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6300> 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:19.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:19.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:19.610+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.610+0000 7f8f06185700 15 osd.7 42 enqueue_peering_evt 1.ds2 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.610+0000 7f8f06185700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c55141000 2019-12-07T19:34:19.610+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.610+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 crt=21'6 unknown pruub 57.054473552s@ mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.610+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.610+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.ds0 1.ds0 1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.013160602s) e42/42) v1 -- 0x558c54e06300 con 0x558c54c19800 2019-12-07T19:34:19.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 270 ==== pg_notify2(1.4s0 1.4s0 (query:42 sent:42 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.007835865s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) e42/42) v1 ==== 1215+0+0 (crc 0 0 0) 0x558c58137800 con 0x558c54c64000 2019-12-07T19:34:19.610+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 1(2) notify: (query:42 sent:42 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.007835865s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.610+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 1(2) notify: (query:42 sent:42 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.007835865s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 1(2) notify: (query:42 sent:42 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.007835865s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process 0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 1(2) notify: (query:42 sent:42 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.007835865s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 271 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58137800 con 0x558c54c64000 2019-12-07T19:34:19.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.4s0 from 1(2) notify: (query:42 sent:42 1.4s2( v 37'4 (0'0,37'4] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.007835865s) 2->0 ([18,40] all_participants=0(1),1(2),2(4),3(1),4(5),5(3),7(0) intervals=([39,40] acting 1(2),2(4),4(5),5(3),7(0)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 1(2), already have info from that osd, ignoring 2019-12-07T19:34:19.610+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:19.610+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068156525s int 16s) 2019-12-07T19:34:19.610+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068156525s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068156525s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068156525s int 16s)) prio 255 cost 10 e42) pg 0x558c54d98000 2019-12-07T19:34:19.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 272 ==== pg_info2(1.4s0 1.4s0 1.4s2( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.007835865s) e42/42) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c58137800 con 0x558c54c64000 2019-12-07T19:34:19.610+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.007835865s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.610+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.007835865s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.610+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068156525s int 16s) 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.007835865s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.610+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] state: got info from osd.1(0) 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.007835865s) pg_lease_ack(ruub 59.041434037s)) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.610+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 273 ==== osd_map(42..42 src has 1..42) v4 ==== 673+0+0 (crc 0 0 0) 0x558c57580480 con 0x558c54c64000 2019-12-07T19:34:19.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 274 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58137800 con 0x558c54c64000 2019-12-07T19:34:19.610+0000 7f8f1199c700 20 osd.7 42 OSD::ms_dispatch: osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.610+0000 7f8f1199c700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.610+0000 7f8f1199c700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.610+0000 7f8f1199c700 20 osd.7 42 _dispatch 0x558c57580480 osd_map(42..42 src has 1..42) v4 2019-12-07T19:34:19.610+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.4s2( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 pruub=14.007835865s) pg_lease_ack(ruub 59.041434037s) 2019-12-07T19:34:19.610+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068340106s int 16s) 2019-12-07T19:34:19.610+0000 7f8f1199c700 3 osd.7 42 handle_osd_map epochs [42,42], i have 42, src has [1,42] 2019-12-07T19:34:19.610+0000 7f8f1199c700 10 osd.7 42 no new maps here, dropping 2019-12-07T19:34:19.610+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068340106s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 59.041434037s 2019-12-07T19:34:19.610+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.068156525s int 16s) 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 59.041434037s 2019-12-07T19:34:19.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 275 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58137800 con 0x558c54c64000 2019-12-07T19:34:19.610+0000 7f8f05984700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.021415354s -> ruub 59.046741171s 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 59.041434037s 2019-12-07T19:34:19.610+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.9s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068413248s int 16s) 2019-12-07T19:34:19.610+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068413248s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.610+0000 7f8f05984700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.046741171s 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 59.041434037s 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 59.041434037s (sent 59.041434037s) 2019-12-07T19:34:19.610+0000 7f8f05984700 5 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.612110+0000 6 0.000079 2019-12-07T19:34:19.610+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.1(2) activated and committed 2019-12-07T19:34:19.610+0000 7f8f05984700 5 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.610+0000 7f8f05984700 5 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.610+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=0/0 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068413248s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.610+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 21'4 -> 21'4 2019-12-07T19:34:19.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068413248s int 16s)) prio 255 cost 10 e42) pg 0x558c5500a000 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.610+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.610+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068413248s int 16s) 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.610+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:19.610+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] state: got info from osd.1(0) 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.610+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:19.610+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c57ef8600 seq 7 2019-12-07T19:34:19.610+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.068413248s int 16s) 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.610+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:19.610+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] peer_clock_delta_lb 0.021415354s -> ruub 59.046997894s 2019-12-07T19:34:19.610+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.610+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.046997894s 2019-12-07T19:34:19.610+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:19.610+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:19.610+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.611+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:19.611+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:19.610+0000 7f8f04181700 5 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.612193+0000 6 0.000088 2019-12-07T19:34:19.611+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8600 onodes shared_blobs 2019-12-07T19:34:19.611+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8600 allocated 0x[] released 0x[] 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.611+0000 7f8f04181700 5 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.611+0000 7f8f04181700 5 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.611+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 prepare 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.611+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8600 2019-12-07T19:34:19.611+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=39/40 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown NOTIFY pruub 57.057641346s@ mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.611+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 io_done 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] activate - no missing, moving last_complete 34'17 -> 34'17 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.611+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:84 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x150c5b~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x150c5b 0x150c5b~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x120c5b 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: all_activated_and_committed 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc5b 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068340106s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.611+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea0c00 seq 46 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x167 and padding block with 0xe99 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068340106s int 16s)) prio 255 cost 10 e42) pg 0x558c57380000 2019-12-07T19:34:19.611+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.611+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.611+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:19.611+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068340106s int 16s) 2019-12-07T19:34:19.611+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:19.611+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes shared_blobs 2019-12-07T19:34:19.611+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] state: got info from osd.1(0) 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.611+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:19.611+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.068340106s int 16s) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.021415354s -> ruub 59.046924752s 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.046924752s 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 5 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.612333+0000 6 0.000093 2019-12-07T19:34:19.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 276 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58137800 con 0x558c54c64000 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.611+0000 7f8f0898a700 5 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.611+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.7s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068577717s int 16s) 2019-12-07T19:34:19.611+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068577717s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.611+0000 7f8f0898a700 5 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=0/0 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 36'33 -> 36'33 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x151167 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000152660s (0.000000107s flush + 0.000152553s kv commit) 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c577fc000 seq 19 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] state: mnow 43.047839213s >= prior_readable_until_ub 0s 2019-12-07T19:34:19.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] share_pg_info 2019-12-07T19:34:19.611+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:19.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068577717s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068577717s int 16s)) prio 255 cost 10 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) e42/42) v1 -- 0x558c57450000 con 0x558c54c64000 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x151167~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc000 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc000 onodes shared_blobs 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc000 allocated 0x[] released 0x[] 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x151167 0x151167~50c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.068577717s int 16s) 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x121167 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x167 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x673 and padding block with 0x98d 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 prepare 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc000 2019-12-07T19:34:19.611+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 io_done 2019-12-07T19:34:19.611+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] state: got info from osd.1(0) 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) e42/42) v1 -- 0x558c5737aa00 con 0x558c54c19800 2019-12-07T19:34:19.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.068577717s int 16s) 2019-12-07T19:34:19.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] peer_clock_delta_lb 0.021415354s -> ruub 59.047162363s 2019-12-07T19:34:19.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.047162363s 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) e42/42) v1 -- 0x558c5737b100 con 0x558c54c5bc00 2019-12-07T19:34:19.611+0000 7f8f05183700 5 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.612464+0000 6 0.000081 2019-12-07T19:34:19.611+0000 7f8f05183700 5 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) e42/42) v1 -- 0x558c5737b800 con 0x558c54c5b400 2019-12-07T19:34:19.611+0000 7f8f05183700 5 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.611+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=39/40 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown NOTIFY pruub 57.057013246s@ mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.611+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] activate - no missing, moving last_complete 40'15 -> 40'15 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) e42/42) v1 -- 0x558c55052000 con 0x558c54c19c00 2019-12-07T19:34:19.611+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8600> 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 kv_submitted 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8600 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 finishing 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8600 onodes 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8600 done 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8600 [] 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.611+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.611+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:19.611+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57ff0f00 seq 50 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f05984700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f05984700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.611+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:19.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.611+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:19.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:19.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c54d98000 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0f00 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:19.611+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0f00 onodes shared_blobs 2019-12-07T19:34:19.611+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0f00 prepare 2019-12-07T19:34:19.611+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0f00 2019-12-07T19:34:19.611+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0f00 io_done 2019-12-07T19:34:19.611+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=21'4 unknown mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.611+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:19.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.611+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.es0 1.es0 1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c58124a00 con 0x558c54c64000 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.611+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:85 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] check_local 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] needs_backfill does not need backfill 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] activate all replicas clean, no recovery 2019-12-07T19:34:19.611+0000 7f8f06986700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered) prio 255 cost 10 e42) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.611+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:86 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] agent_stop 2019-12-07T19:34:19.611+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54d59b00 seq 27 2019-12-07T19:34:19.611+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5332 (1 ios * 4000 + 1332 bytes) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered) prio 255 cost 10 e42) pg 0x558c548ab000 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered 2019-12-07T19:34:19.611+0000 7f8f06986700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] exit Started/Primary/Active/Activating 2019-12-07T19:34:19.612818+0000 13 0.001848 2019-12-07T19:34:19.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.611+0000 7f8f06986700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] enter Started/Primary/Active/Recovered 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(1),4(5),5(3),7(0) upset 1(2),2(4),3(1),4(5),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(1),4(5),5(3),7(0) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x151673 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000480540s (0.000000099s flush + 0.000480441s kv commit) 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(1) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(5) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2656 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(1),4(5),5(3),7(0)=2} 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3819 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3842 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3842 < min_flush_size 524288 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(1) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x151673~f02 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(5) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x151673 0x151673~f02 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x121673 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x673 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x575 and padding block with 0xa8b 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.611+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:87 2019-12-07T19:34:19.611+0000 7f8f06986700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] exit Started/Primary/Active/Recovered 2019-12-07T19:34:19.612974+0000 0 0.000000 2019-12-07T19:34:19.611+0000 7f8f06986700 5 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 pi=[18,41)/1 crt=37'4 lcod 21'2 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] enter Started/Primary/Active/Clean 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] share_pg_info 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55052700 con 0x558c54c64000 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55052e00 con 0x558c54c19800 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55053500 con 0x558c54c5bc00 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55053c00 con 0x558c54c5b400 2019-12-07T19:34:19.611+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55054300 con 0x558c54c19c00 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x152575 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000185593s (0.000000101s flush + 0.000185492s kv commit) 2019-12-07T19:34:19.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.611+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:88 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] exit NotTrimming 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] enter Trimming 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:19.611+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.611+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:89 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] finish_recovery 2019-12-07T19:34:19.611+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] clear_recovery_state 2019-12-07T19:34:19.611+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c5730b500 seq 28 2019-12-07T19:34:19.611+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b500 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b500 onodes shared_blobs 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc000,0x558c57ff0f00,0x558c54d59b00> 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 kv_submitted 2019-12-07T19:34:19.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc000 2019-12-07T19:34:19.611+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 prepare 2019-12-07T19:34:19.611+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b500 2019-12-07T19:34:19.612+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 io_done 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 finishing 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc000 onodes 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc000 done 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc000 [] 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.612+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f0898a700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0f00 kv_submitted 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0f00 2019-12-07T19:34:19.612+0000 7f8f0898a700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0f00 finishing 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0f00 onodes 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0f00 done 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:19.612+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0f00 [] 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.612+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.612+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:19.612+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c57380000 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:19.612+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: NotTrimming: trimming d 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:19.612+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 kv_queued 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.612+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.612+0000 7f8f05183700 15 osd.7 42 enqueue_peering_evt 1.7s4 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.612+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.612+0000 7f8f05183700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.612+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:90 2019-12-07T19:34:19.612+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=36'33 unknown mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.612+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:19.612+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:19.612+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.612+0000 7f8f0b990700 10 osd.7 42 queueing pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] for snaptrim 2019-12-07T19:34:19.612+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.612+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.612+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.8s0 1.8s0 1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c57451500 con 0x558c54c64000 2019-12-07T19:34:19.612+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.612+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:91 2019-12-07T19:34:19.612+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.612+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=40'15 unknown pruub 57.057013246s@ mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.612+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.612+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 pruub=14.015507853s) e42/42) v1 -- 0x558c58135c00 con 0x558c54c64000 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:19.612+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.612+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x152575~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x152575 0x152575~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x122575 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x575 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9d2 and padding block with 0x62e 2019-12-07T19:34:19.612+0000 7f8f08189700 15 osd.7 42 enqueue_peering_evt 1.9s4 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.612+0000 7f8f08189700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c5500a000 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) pg 0x558c548ab000 2019-12-07T19:34:19.612+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.612+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 crt=34'17 unknown pruub 57.057641346s@ mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.612+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] snap_trimmer posting 2019-12-07T19:34:19.612+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 pruub=14.015878610s) e42/42) v1 -- 0x558c576b8000 con 0x558c54c64000 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap d 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:19.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630300) upper_bound after SNA_1_000000000000000D_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.4' 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1529d2 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630300) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000209134s (0.000000103s flush + 0.000209031s kv commit) 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b500> 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 kv_submitted 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b500 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 finishing 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b500 onodes 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 done 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b500 [] 2019-12-07T19:34:19.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000D_.0_0000000000000001.4) returns 0 SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239760 in raw 0x558c55239760 len 70 nref 1) ) 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630300) upper_bound after SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. key 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630300) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22..) returns 0 SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239760 in raw 0x558c55239760 len 70 nref 1) ) 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_object_context: obc NOT found in cache: 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# _ 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# _ = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# snapset 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# snapset = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] populate_obc_watchers 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_object_context: creating obc from disk: 0x558c5524c480 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_object_context: 0x558c5524c480 1:276d5164:::smithi13140796-22:e rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:e(37'3 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_object_context: obc NOT found in cache: 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# _ 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# _ = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] populate_obc_watchers 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_object_context: creating obc from disk: 0x558c57945440 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_object_context: 0x558c57945440 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e,d,c]} 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:e old_snaps c,d,e old snapset e=[]:{e=[e,d,c]} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] simple_opc_create 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:e snaps c,d,e -> c,e 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:e new snapset e=[]:{e=[e,c]} on 1:276d5164:::smithi13140796-22:head(37'4 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 1:276d5164:::smithi13140796-22:e writing updated snapset on 1:276d5164:::smithi13140796-22:head, snapset is e=[]:{e=[e,c]} 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] new_repop rep_tid 26 (no op) 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] new_repop: repgather(0x558c576b6360 0'0 rep_tid=26 committed?=0 r=0) 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] simple_opc_submit 0x558c576b6360 2019-12-07T19:34:19.612+0000 7f8f0297e700 7 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] issue_repop rep_tid 26 o 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] submit_transaction: op Op(1:276d5164:::smithi13140796-22:e v=42'6 tt=0'0 tid=26 reqid=osd.7.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:19.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_hash_info: found on disk, size 90112 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:e projected size 360448 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:head projected size 311296 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] start_rmw: Op(1:276d5164:::smithi13140796-22:e v=42'6 tt=0'0 tid=26 reqid=osd.7.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_state_to_reads: Op(1:276d5164:::smithi13140796-22:e v=42'6 tt=0'0 tid=26 reqid=osd.7.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:e v=42'6 tt=0'0 tid=26 reqid=osd.7.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] generate_transactions: 1:276d5164:::smithi13140796-22:e, current size is 360448 buffers are {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): new_size start 360448 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): to_append: {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:e resetting hinfo to logical size 360448 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] generate_transactions: 1:276d5164:::smithi13140796-22:head, current size is 311296 buffers are {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): new_size start 311296 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): to_append: {} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] operator(): 1:276d5164:::smithi13140796-22:head resetting hinfo to logical size 311296 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: written: {1:276d5164:::smithi13140796-22:e={},1:276d5164:::smithi13140796-22:head={}} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:e v=42'6 tt=0'0 tid=26 reqid=osd.7.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: written_set: {1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:e,{} 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:head,{} 2019-12-07T19:34:19.612+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 42/41 ECSubWrite(tid=26, reqid=osd.7.0:26, at_version=42'6, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c579e1800 con 0x558c54c64000 2019-12-07T19:34:19.612+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 42/41 ECSubWrite(tid=26, reqid=osd.7.0:26, at_version=42'6, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c579e2000 con 0x558c54c19800 2019-12-07T19:34:19.612+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.4s1 42/41 ECSubWrite(tid=26, reqid=osd.7.0:26, at_version=42'6, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c579e3000 con 0x558c54c5bc00 2019-12-07T19:34:19.612+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 42/41 ECSubWrite(tid=26, reqid=osd.7.0:26, at_version=42'6, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c579e0000 con 0x558c54c5b400 2019-12-07T19:34:19.612+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 42/41 ECSubWrite(tid=26, reqid=osd.7.0:26, at_version=42'6, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c54e40000 con 0x558c54c19c00 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.update_snaps 1:276d5164:::smithi13140796-22:e c,e was 2019-12-07T19:34:19.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:276d5164:::smithi13140796-22:e c,d,e 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.set_snaps 1:276d5164:::smithi13140796-22:e c,e 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 snap_mapper.update_snaps rm SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 trim 42'5 (37'3) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 20 trim 42'6 (37'4) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 37'4 (0'0,37'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] append_log log((0'0,37'4], crt=37'4) [42'5 (37'3) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,42'6 (37'4) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'5 (0'0,42'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 lua=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] add_log_entry 42'5 (37'3) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 lua=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] add_log_entry 42'6 (37'4) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 lua=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] append_log approx pg log length = 6 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 lua=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.612+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.612+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c548f7800 seq 29 2019-12-07T19:34:19.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 1 keys 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 1 keys = 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys = 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000D_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000042.00000000000000000005' <- 0000000042.00000000000000000005 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000042.00000000000000000006' <- 0000000042.00000000000000000006 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 8355 (1 ios * 4000 + 4355 bytes) 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes 0x558c57f35900,0x558c58093400 shared_blobs 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35a08) update 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:head# is 745 (549 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58093508) update 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:e# is 718 (412 bytes onode + 2 bytes spanning blobs + 304 bytes inline extents) 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[] released 0x[] 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 lua=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] op_applied version 42'6 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4288 < min_flush_size 524288 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] eval_repop repgather(0x558c576b6360 42'6 rep_tid=26 committed?=0 r=0) (no op) 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35900 had 1 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] calc_trim_to_aggressive limit = 37'4 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093400 had 1 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4311 < min_flush_size 524288 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4311 < min_flush_size 524288 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1529d2~10d7 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] enter Trimming/WaitRepops 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1529d2 0x1529d2~10d7 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x1229d2 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9d2 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] snap_trimmer complete 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xaa9 and padding block with 0x557 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] _finish_recovery 2019-12-07T19:34:19.613+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] purge_strays 2019-12-07T19:34:19.613+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.613+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:92 2019-12-07T19:34:19.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x153aa9 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000156379s (0.000000090s flush + 0.000156289s kv commit) 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7800> 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 finishing 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 0x558c57f35900,0x558c58093400 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 82 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5763de00 con 0x558c54c19c00 2019-12-07T19:34:19.613+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000013 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 277 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) e42/42) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c55053c00 con 0x558c54c64000 2019-12-07T19:34:19.613+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.as5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.069490446s int 16s) 2019-12-07T19:34:19.613+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.069490446s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 278 ==== pg_notify2(1.2s0 1.2s0 (query:42 sent:42 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) e42/42) v1 ==== 1237+0+0 (crc 0 0 0) 0x558c576b8000 con 0x558c54c64000 2019-12-07T19:34:19.613+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 1(2) notify: (query:42 sent:42 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.613+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 1(2) notify: (query:42 sent:42 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.069490446s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.069490446s int 16s)) prio 255 cost 10 e42) pg 0x558c57e8b000 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) pg_lease(ru 0s ub 59.069490446s int 16s) 2019-12-07T19:34:19.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 189 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfbc00 con 0x558c54c5b400 2019-12-07T19:34:19.613+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581ccc60 prio 196 cost 0 latency 0.000010 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.613+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] state: got info from osd.1(0) 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] proc_lease pg_lease(ru 0s ub 59.069490446s int 16s) 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] peer_clock_delta_lb 0.021415354s -> ruub 59.048075092s 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] proc_lease readable_until_ub now 59.048075092s 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.613+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.614850+0000 6 0.000095 2019-12-07T19:34:19.613+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000236 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 2019-12-07T19:34:19.613+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581ccb00 finish 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=0/0 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown NOTIFY mbc={} ps=[a~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] activate - no missing, moving last_complete 21'2 -> 21'2 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] state: Activate Finished 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581ccc60 prio 196 cost 0 latency 0.000090 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.613+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581ccc60 finish 2019-12-07T19:34:19.613+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:19.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c57b95200 seq 6 2019-12-07T19:34:19.613+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:19.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:19.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:19.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:19.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:19.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95200 cost 5458 (1 ios * 4000 + 1458 bytes) 2019-12-07T19:34:19.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95200 onodes shared_blobs 2019-12-07T19:34:19.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 prepare 2019-12-07T19:34:19.613+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95200 2019-12-07T19:34:19.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 io_done 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 1(2) notify: (query:42 sent:42 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:34:19.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process 0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42)> waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 1(2) notify: (query:42 sent:42 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1325 < min_flush_size 524288 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MNotifyRec 1.2s0 from 1(2) notify: (query:42 sent:42 1.2s2( v 29'15 (0'0,29'15] local-lis/les=39/40 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) 2->0 ([28,40] all_participants=0(0),1(2),2(4),3(5),4(1),5(3),7(0),7(4) intervals=([28,37] acting 0(0),1(2),3(5),4(1),5(3),7(4)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x153aa9~52d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x153aa9 0x153aa9~52d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x123aa9 2019-12-07T19:34:19.613+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: Active: got notify from 1(2), already have info from that osd, ignoring 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xaa9 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.613+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfd6 and padding block with 0x2a 2019-12-07T19:34:19.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x153fd6 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.613+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000091787s (0.000000085s flush + 0.000091702s kv commit) 2019-12-07T19:34:19.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95200> 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 kv_submitted 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95200 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 finishing 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95200 onodes 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95200 done 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95200 [] 2019-12-07T19:34:19.613+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f07988700 15 osd.7 42 enqueue_peering_evt 1.as5 epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.613+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42) prio 255 cost 10 e42) pg 0x558c57e8b000 2019-12-07T19:34:19.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 ActivateCommitted from 42 processed at 42 2019-12-07T19:34:19.613+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 crt=21'2 unknown mbc={} ps=[a~1]] state: react 42 telling primary 2019-12-07T19:34:19.613+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.613+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.as0 1.as0 1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c58122700 con 0x558c54c64000 2019-12-07T19:34:19.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 279 ==== pg_info2(1.2s0 1.2s0 1.2s2( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c576b8000 con 0x558c54c64000 2019-12-07T19:34:19.614+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.2s2( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.614+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.2s2( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.2s2( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.2s2( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s)) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.614+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(2) info: 1.2s2( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease_ack(ruub 59.041594632s) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.4 ruub 59.041594632s 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.1 ruub 59.041594632s 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.5 ruub 59.041594632s 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.2 ruub 59.041594632s 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] recalc_readable_until readable_until[_ub] 59.041594632s (sent 59.041594632s) 2019-12-07T19:34:19.614+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: peer osd.1(2) activated and committed 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.614+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:99 2019-12-07T19:34:19.614+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] state: all_activated_and_committed 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 activating mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.614+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] state: mnow 43.051436095s >= prior_readable_until_ub 0s 2019-12-07T19:34:19.614+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] share_pg_info 2019-12-07T19:34:19.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c54e9a300 con 0x558c54c64000 2019-12-07T19:34:19.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c57390e00 con 0x558c54c19800 2019-12-07T19:34:19.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c57392300 con 0x558c54c5bc00 2019-12-07T19:34:19.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c57390700 con 0x558c54c5b400 2019-12-07T19:34:19.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 -- 0x558c574e5500 con 0x558c54c19c00 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.614+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.614+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:100 2019-12-07T19:34:19.614+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] check_local 2019-12-07T19:34:19.614+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] checking 1:41df217a:::smithi13140796-39:head at 29'15 2019-12-07T19:34:19.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:19.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:41df217a:::smithi13140796-39:head# key 0x80800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:19.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] checking 1:41df217a:::smithi13140796-39:4 at 29'14 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:41df217a:::smithi13140796-39:4# key 0x80800000000000000141df21'z!smithi13140796-39!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] checking 1:4c9e444c:::smithi13140796-48:head at 23'10 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] needs_backfill does not need backfill 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] activate all replicas clean, no recovery 2019-12-07T19:34:19.615+0000 7f8f03980700 15 osd.7 42 enqueue_peering_evt 1.2s0 epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered) prio 255 cost 10 e42) 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.615+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:101 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] hit_set_clear 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] agent_stop 2019-12-07T19:34:19.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c54ea1500 seq 16 2019-12-07T19:34:19.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5343 (1 ios * 4000 + 1343 bytes) 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered) prio 255 cost 10 e42) pg 0x558c57e88000 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1174 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:19.615+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 AllReplicasRecovered 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1197 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1197 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] exit Started/Primary/Active/Activating 2019-12-07T19:34:19.616510+0000 14 0.001953 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x153fd6~4ad to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x153fd6 0x153fd6~4ad to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] enter Started/Primary/Active/Recovered 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x123fd6 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfd6 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x483 and padding block with 0xb7d 2019-12-07T19:34:19.615+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] needs_recovery is recovered 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats actingset 1(2),2(4),3(5),4(1),5(3),7(0) upset 1(2),2(4),3(5),4(1),5(3),7(0) acting_recovery_backfill 1(2),2(4),3(5),4(1),5(3),7(0) 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(0) primary objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 1(2) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 2(4) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x154483 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 3(5) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 4(1) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000110526s (0.000000110s flush + 0.000110416s kv commit) 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 5(3) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats no peer_missing found for 7(4) 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats shard 7(4) objects 2 missing 0 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats object_location_counts {1(2),2(4),3(5),4(1),5(3),7(0)=2} 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats missing shard 7(0) missing= 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats degraded 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] update_calc_stats misplaced 0 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.615+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:102 2019-12-07T19:34:19.615+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] exit Started/Primary/Active/Recovered 2019-12-07T19:34:19.616662+0000 0 0.000000 2019-12-07T19:34:19.615+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 pi=[28,41)/1 crt=29'15 mlcod 0'0 active mbc={} trimq=[d~1,10~1] ps=[a~1]] enter Started/Primary/Active/Clean 2019-12-07T19:34:19.615+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] share_pg_info 2019-12-07T19:34:19.615+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c58122e00 con 0x558c54c64000 2019-12-07T19:34:19.615+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c58123500 con 0x558c54c19800 2019-12-07T19:34:19.615+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c58123c00 con 0x558c54c5bc00 2019-12-07T19:34:19.615+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 113 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57519a00 con 0x558c54c19800 2019-12-07T19:34:19.615+0000 7f8f24c58700 15 osd.7 42 enqueue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000009 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.615+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.615+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c57ed4300 con 0x558c54c5b400 2019-12-07T19:34:19.615+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c57ed4a00 con 0x558c54c19c00 2019-12-07T19:34:19.615+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 114 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c57390e00 con 0x558c54c19800 2019-12-07T19:34:19.615+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.5s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.615+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.615+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:103 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) pg 0x558c57522000 2019-12-07T19:34:19.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:19.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:19.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.615+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] exit NotTrimming 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] enter Trimming 2019-12-07T19:34:19.615+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:19.615+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000080 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:19.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c548f6900 seq 9 2019-12-07T19:34:19.615+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.615+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:34:19.615+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:19.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:19.615+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:104 2019-12-07T19:34:19.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:19.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6900 onodes shared_blobs 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.615+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] finish_recovery 2019-12-07T19:34:19.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 prepare 2019-12-07T19:34:19.615+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] clear_recovery_state 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6900 2019-12-07T19:34:19.615+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 io_done 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.615+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:19.615+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57b94300 seq 17 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:19.615+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x154483~416 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x154483 0x154483~416 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94300 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.615+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94300 onodes shared_blobs 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94300 allocated 0x[] released 0x[] 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x124483 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x483 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x899 and padding block with 0x767 2019-12-07T19:34:19.615+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 prepare 2019-12-07T19:34:19.615+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94300 2019-12-07T19:34:19.615+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 io_done 2019-12-07T19:34:19.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:19.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: NotTrimming: trimming d 2019-12-07T19:34:19.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[d~1,10~1] ps=[a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:19.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:105 2019-12-07T19:34:19.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x154899 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.615+0000 7f8f0b990700 10 osd.7 42 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[d~1,10~1] ps=[a~1]] for snaptrim 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.615+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123892s (0.000000092s flush + 0.000123800s kv commit) 2019-12-07T19:34:19.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:106 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6900> 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 kv_submitted 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6900 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 finishing 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6900 onodes 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 done 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6900 [] 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x154899~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x154899 0x154899~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x124899 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x899 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcf6 and padding block with 0x30a 2019-12-07T19:34:19.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.615+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=42) prio 5 cost 1048576 e42) pg 0x558c57e88000 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] snap_trimmer posting 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap d 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c575c8b00) upper_bound after SNA_1_000000000000000D_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.2' 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c575c8b00) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' 2019-12-07T19:34:19.615+0000 7f8f03980700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000D_.0_0000000000000001.2) returns 0 SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239290 in raw 0x558c55239290 len 70 nref 1) ) 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1]] SnapTrimmer state: adding snap d to purged_snaps 2019-12-07T19:34:19.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: purged_snaps now [a~1,d~1], snap_trimq now [10~1] 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c577fd200 seq 18 2019-12-07T19:34:19.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd200 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd200 onodes shared_blobs 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 prepare 2019-12-07T19:34:19.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd200 2019-12-07T19:34:19.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 io_done 2019-12-07T19:34:19.615+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] share_pg_info 2019-12-07T19:34:19.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c54e04700 con 0x558c54c64000 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c54e06a00 con 0x558c54c19800 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c54e06300 con 0x558c54c5bc00 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c54e05500 con 0x558c54c5b400 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c548a1c00 con 0x558c54c19c00 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] exit Trimming 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.616+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:107 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] enter NotTrimming 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] exit NotTrimming 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming/WaitReservation 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.616+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:108 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] snap_trimmer complete 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 165 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c58123c00 con 0x558c54c5bc00 2019-12-07T19:34:19.616+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.616+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.616+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 115 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c57390e00 con 0x558c54c19800 2019-12-07T19:34:19.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) pg 0x558c5736a000 2019-12-07T19:34:19.616+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.5s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.616+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) pg 0x558c57522000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x154cf6 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.616+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.616+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.616+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.616+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000400094s (0.000000100s flush + 0.000399994s kv commit) 2019-12-07T19:34:19.616+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: NotTrimming: trimming 10 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.616+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] exit Trimming/WaitReservation 2019-12-07T19:34:19.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:19.616+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c57b94900 seq 10 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:109 2019-12-07T19:34:19.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x154cf6~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:19.616+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x154cf6 0x154cf6~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:19.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94900 onodes shared_blobs 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x124cf6 2019-12-07T19:34:19.616+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:19.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcf6 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.616+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5740b200 seq 10 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x163 and padding block with 0xe9d 2019-12-07T19:34:19.616+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:19.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 prepare 2019-12-07T19:34:19.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94900 2019-12-07T19:34:19.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 io_done 2019-12-07T19:34:19.616+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:19.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:19.616+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:19.616+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b200 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.616+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b200 onodes shared_blobs 2019-12-07T19:34:19.616+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94300> 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 kv_submitted 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94300 2019-12-07T19:34:19.616+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 prepare 2019-12-07T19:34:19.616+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b200 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 finishing 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94300 onodes 2019-12-07T19:34:19.616+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 io_done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94300 done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd200 kv_submitted 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94300 [] 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x155163 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000126859s (0.000000088s flush + 0.000126771s kv commit) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2117 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2140 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2140 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x155163~85c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x155163 0x155163~85c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x125163 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x163 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.616+0000 7f8f0b990700 10 osd.7 42 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[10~1] ps=[a~1,d~1]] for snaptrim 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9bf and padding block with 0x641 2019-12-07T19:34:19.616+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.616+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.616+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:110 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd200> 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 kv_submitted 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd200 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 finishing 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd200 onodes 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd200 done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd200 [] 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] _finish_recovery 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] purge_strays 7(4) 2019-12-07T19:34:19.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 166 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558c58123c00 con 0x558c54c5bc00 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] sending PGRemove to osd.7(4) 2019-12-07T19:34:19.616+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1559bf 2019-12-07T19:34:19.616+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000136388s (0.000000082s flush + 0.000136306s kv commit) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94900,0x558c5740b200> 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 kv_submitted 2019-12-07T19:34:19.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94900 2019-12-07T19:34:19.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) pg 0x558c57383000 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 finishing 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94900 onodes 2019-12-07T19:34:19.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd pg remove(epoch 42; pg1.2s4; ) v3 -- 0x558c54d9f860 con 0x558c53ae5400 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94900 done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94900 [] 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 kv_submitted 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b200 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 finishing 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b200 onodes 2019-12-07T19:34:19.616+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b200 done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b200 [] 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.616+0000 7f8f05183700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] remove_stray_recovery_sources remove osd 7(4) from missing_loc 2019-12-07T19:34:19.616+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.616+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:19.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57ff1200 seq 12 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] update_heartbeat_peers 1,2,3,4,5,7 unchanged 2019-12-07T19:34:19.616+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:19.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:19.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1200 onodes shared_blobs 2019-12-07T19:34:19.616+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:111 2019-12-07T19:34:19.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 prepare 2019-12-07T19:34:19.616+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1200 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 io_done 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=42) prio 5 cost 1048576 e42) pg 0x558c57e88000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] snap_trimmer posting 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== osd pg remove(epoch 42; pg1.2s4; ) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c54d9f860 con 0x558c53ae5400 2019-12-07T19:34:19.616+0000 7f8f1119b700 7 osd.7 42 handle_fast_pg_remove osd pg remove(epoch 42; pg1.2s4; ) v3 from osd.7 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 10 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:19.616+0000 7f8f1119b700 15 osd.7 42 enqueue_peering_evt 1.2s4 epoch_sent: 42 epoch_requested: 42 DeleteStart 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteStart) prio 255 cost 10 e42) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1559bf~416 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1559bf 0x1559bf~416 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteStart) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c5517b840) upper_bound after SNA_1_0000000000000010_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.0_0000000000000001.2' 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x1259bf 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9bf 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdd5 and padding block with 0x22b 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteStart) prio 255 cost 10 e42) pg 0x558c5731a000 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c5517b840) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19..' 2019-12-07T19:34:19.616+0000 7f8f03980700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000010_.0_0000000000000001.2) returns 0 SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239290 in raw 0x558c55239290 len 70 nref 1) ) 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteStart 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:19.616+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.617812+0000 6 0.000087 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1]] SnapTrimmer state: adding snap 10 to purged_snaps 2019-12-07T19:34:19.616+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ToDelete 2019-12-07T19:34:19.616+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.616+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ToDelete/WaitDeleteReseved 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~1]] SnapTrimmer state: purged_snaps now [a~1,d~1,10~1], snap_trimq now [] 2019-12-07T19:34:19.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c577fcf00 seq 19 2019-12-07T19:34:19.616+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:19.616+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:19.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:19.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcf00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcf00 onodes shared_blobs 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcf00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 prepare 2019-12-07T19:34:19.616+0000 7f8f0b990700 15 osd.7 42 enqueue_peering_evt 1.2s4 epoch_sent: 42 epoch_requested: 42 DeleteReserved 2019-12-07T19:34:19.616+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcf00 2019-12-07T19:34:19.616+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 io_done 2019-12-07T19:34:19.616+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteReserved) prio 255 cost 10 e42) 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~1]] share_pg_info 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 167 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5841f200 con 0x558c54c5bc00 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteReserved) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c57334f20 prio 196 cost 0 latency 0.000012 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c576de700 con 0x558c54c64000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x155dd5 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteReserved) prio 255 cost 10 e42) pg 0x558c5731a000 2019-12-07T19:34:19.616+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteReserved 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138918s (0.000000070s flush + 0.000138848s kv commit) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c576dee00 con 0x558c54c19800 2019-12-07T19:34:19.616+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] exit Started/ToDelete/WaitDeleteReseved 2019-12-07T19:34:19.617893+0000 1 0.000035 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f07988700 5 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter Started/ToDelete/Deleting 2019-12-07T19:34:19.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c576e1100 con 0x558c54c5bc00 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] on_removal 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] on_shutdown 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c57334f20 prio 196 cost 0 latency 0.000047 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x155dd5~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c576dfc00 con 0x558c54c5b400 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x155dd5 0x155dd5~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x125dd5 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdd5 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.616+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c57334f20 finish 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x252 and padding block with 0xdae 2019-12-07T19:34:19.616+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c576e0300 con 0x558c54c19c00 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:19.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] clear_backoffs 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~1]] exit Trimming 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff1200> 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 kv_submitted 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1200 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 finishing 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1200 onodes 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] on_change 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1200 done 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1200 [] 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] clear_async_reads 2019-12-07T19:34:19.616+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:112 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] enter NotTrimming 2019-12-07T19:34:19.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 168 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c54e06300 con 0x558c54c5bc00 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:19.616+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.616+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] snap_trimmer complete 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:19.616+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x156252 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.616+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:19.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c57b95b00 seq 31 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132930s (0.000000057s flush + 0.000132873s kv commit) 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcf00> 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) pg 0x558c5736a000 2019-12-07T19:34:19.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._epoch' <- _epoch 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 kv_submitted 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcf00 2019-12-07T19:34:19.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045c'._info' <- _info 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 finishing 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcf00 onodes 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcf00 done 2019-12-07T19:34:19.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcf00 [] 2019-12-07T19:34:19.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95b00 cost 5210 (1 ios * 4000 + 1210 bytes) 2019-12-07T19:34:19.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95b00 onodes shared_blobs 2019-12-07T19:34:19.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.616+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95b00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 prepare 2019-12-07T19:34:19.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95b00 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 io_done 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1047 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:19.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1070 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c54d21b00 seq 11 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1070 < min_flush_size 524288 2019-12-07T19:34:19.616+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x156252~42e to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:19.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x156252 0x156252~42e to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x126252 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x252 2019-12-07T19:34:19.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes shared_blobs 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x680 and padding block with 0x980 2019-12-07T19:34:19.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:19.616+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:19.616+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:19.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x156680 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000143430s (0.000000077s flush + 0.000143353s kv commit) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 169 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558c54e06300 con 0x558c54c5bc00 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x156680~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x156680 0x156680~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x126680 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x680 2019-12-07T19:34:19.617+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xadd and padding block with 0x523 2019-12-07T19:34:19.617+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.617+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 116 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c58123500 con 0x558c54c19800 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95b00> 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 kv_submitted 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95b00 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 finishing 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) pg 0x558c57383000 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95b00 onodes 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95b00 done 2019-12-07T19:34:19.617+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.5s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95b00 [] 2019-12-07T19:34:19.617+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.617+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.617+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:19.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57b95800 seq 13 2019-12-07T19:34:19.617+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:19.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:19.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95800 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95800 onodes shared_blobs 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95800 allocated 0x[] released 0x[] 2019-12-07T19:34:19.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 prepare 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95800 2019-12-07T19:34:19.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 io_done 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f07988700 10 osd.7 42 queue_for_pg_delete on 1.2s4 e 42 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) pg 0x558c57522000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGDelete(1.2s4 e42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGDelete(1.2s4 e42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGDelete(1.2s4 e42) prio 5 cost 1048576 e42) pg 0x558c5731a000 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.617+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteSome 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:19.617+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_delete_work 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5740bb00 seq 11 2019-12-07T19:34:19.617+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:19.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.2s4_head start GHMIN end GHMAX max 30 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:19.617+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 117 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c576dee00 con 0x558c54c19800 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list range 0x847ffffffffffffffd40000000 to 0x847ffffffffffffffd50000000 and 0x84800000000000000140000000 to 0x84800000000000000150000000 start GHMIN 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.617+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.5s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.617+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list pend 0x847ffffffffffffffd50000000 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000014000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e42) pg 0x558c57522000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 4#1:40000000::::head# end GHMAX 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 4#1:41fe8a11:::smithi13140796-45:head# end GHMAX 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 4#1:4986b721:::smithi13140796-43:head# end GHMAX 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x156add 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.2s4_head start GHMAX end GHMAX max 30 = 0, ls.size() = 3, next = GHMAX 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5740b500 seq 12 2019-12-07T19:34:19.617+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_delete_work [4#1:40000000::::head#,4#1:41fe8a11:::smithi13140796-45:head#,4#1:4986b721:::smithi13140796-43:head#] 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:19.617+0000 7f8f07988700 20 snap_mapper.remove_oid 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:19.617+0000 7f8f07988700 20 snap_mapper._remove_oid 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000311685s (0.000000082s flush + 0.000311603s kv commit) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:19.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes shared_blobs 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:19.617+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f07988700 20 snap_mapper.get_snaps 1:41fe8a11:::smithi13140796-45:head got.empty() 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2204 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f07988700 20 snap_mapper.remove_oid 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 2 2019-12-07T19:34:19.617+0000 7f8f07988700 20 snap_mapper._remove_oid 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2227 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2227 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x156add~8b3 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x156add 0x156add~8b3 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x126add 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xadd 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.617+0000 7f8f07988700 20 snap_mapper.get_snaps 1:4986b721:::smithi13140796-43:head got.empty() 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x390 and padding block with 0xc70 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_delete_work deleting 2 objects 2019-12-07T19:34:19.617+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c57b95500 seq 32 2019-12-07T19:34:19.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# onode 0x558c54d96780 txc 0x558c57b95500 2019-12-07T19:34:19.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# 0x0 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21b00> 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c574fa4d0) put_ref 0x0~2000 Blob(0x558c574fa4d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8b60 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c54cf9730) put_ref 0x0~2000 Blob(0x558c54cf9730 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd650 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df5e0) put_ref 0x0~2000 Blob(0x558c578df5e0 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df6c0) put_ref 0x0~2000 Blob(0x558c578df6c0 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df730 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df7a0) put_ref 0x0~2000 Blob(0x558c578df7a0 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df810 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df880) put_ref 0x0~2000 Blob(0x558c578df880 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df8f0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df960) put_ref 0x0~2000 Blob(0x558c578df960 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df9d0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578dfa40) put_ref 0x0~2000 Blob(0x558c578dfa40 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dfab0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0000) put_ref 0x0~2000 Blob(0x558c578f0000 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0070 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f00e0) put_ref 0x0~2000 Blob(0x558c578f00e0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f01c0) put_ref 0x0~2000 Blob(0x558c578f01c0 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0230 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f02a0) put_ref 0x0~2000 Blob(0x558c578f02a0 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0310 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0380) put_ref 0x0~2000 Blob(0x558c578f0380 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f03f0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0460) put_ref 0x0~2000 Blob(0x558c578f0460 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f04d0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0540) put_ref 0x0~2000 Blob(0x558c578f0540 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f05b0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0620) put_ref 0x0~2000 Blob(0x558c578f0620 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0690 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0700) put_ref 0x0~2000 Blob(0x558c578f0700 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0770 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f07e0) put_ref 0x0~2000 Blob(0x558c578f07e0 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0850 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f08c0) put_ref 0x0~2000 Blob(0x558c578f08c0 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0930 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x157390 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f09a0) put_ref 0x0~2000 Blob(0x558c578f09a0 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0a80) put_ref 0x0~2000 Blob(0x558c578f0a80 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0af0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0b60) put_ref 0x0~2000 Blob(0x558c578f0b60 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0bd0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0c40) put_ref 0x0~2000 Blob(0x558c578f0c40 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0cb0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.extentmap(0x558c54d96888) dirty_range mark inline shard dirty 2019-12-07T19:34:19.617+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 170 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c58134700 con 0x558c54c5bc00 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574fa4d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8b60 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x309000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c574fa4d0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c54cf8b78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fb710 space 0x558c54cf8b78 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x309000~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000135893s (0.000000093s flush + 0.000135800s kv commit) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c54cf8b60) put 0x558c54cf8b60 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c54cf8b78 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.617+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54cf9730 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bd650 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x30b000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c54cf9730) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c575bd668 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c54c9a120 space 0x558c575bd668 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x30b000~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c575bd650) put 0x558c575bd650 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c575bd668 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c578df5e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x30d000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df5e0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df668 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fb7a0 space 0x558c578df668 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x30d000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578df650) put 0x558c578df650 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df668 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c578df6c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578df730 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x30f000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df6c0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df748 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fb830 space 0x558c578df748 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x30f000~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578df730) put 0x558c578df730 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df748 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c578df7a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578df810 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x311000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df7a0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df828 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fb8c0 space 0x558c578df828 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x311000~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578df810) put 0x558c578df810 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df828 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c578df880 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578df8f0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x313000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df880) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95800,0x558c5740bb00> 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df908 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fb950 space 0x558c578df908 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x313000~2000 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 kv_submitted 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x157390~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578df8f0) put 0x558c578df8f0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95800 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df908 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c578df960 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578df9d0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x315000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578df960) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df9e8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 finishing 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fb9e0 space 0x558c578df9e8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95800 onodes 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x315000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578df9d0) put 0x558c578df9d0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x157390 0x157390~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578df9e8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95800 done 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c578dfa40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578dfab0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x317000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578dfa40) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578dfac8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95800 [] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fba70 space 0x558c578dfac8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x317000~2000 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578dfab0) put 0x558c578dfab0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578dfac8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c578f0000 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0070 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:19.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x319000~2000] 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0000) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0088 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbb00 space 0x558c578f0088 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) pg 0x558c5736a000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x319000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0070) put 0x558c578f0070 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0088 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c578f00e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x31b000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f00e0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0168 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbb90 space 0x558c578f0168 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x31b000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0150) put 0x558c578f0150 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0168 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c578f01c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0230 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x31d000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f01c0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbc20 space 0x558c578f0248 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x31d000~2000 2019-12-07T19:34:19.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0230) put 0x558c578f0230 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0248 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c578f02a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0310 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x31f000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f02a0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0328 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbcb0 space 0x558c578f0328 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x31f000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0310) put 0x558c578f0310 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0328 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c578f0380 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f03f0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x321000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0380) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0408 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbd40 space 0x558c578f0408 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x321000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f03f0) put 0x558c578f03f0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0408 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c578f0460 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f04d0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x323000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0460) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f04e8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbdd0 space 0x558c578f04e8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x323000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f04d0) put 0x558c578f04d0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f04e8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c578f0540 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f05b0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x325000~2000] 2019-12-07T19:34:19.617+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0540) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f05c8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbe60 space 0x558c578f05c8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x325000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f05b0) put 0x558c578f05b0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f05c8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c578f0620 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0690 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x327000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0620) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f06a8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fbef0 space 0x558c578f06a8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x327000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0690) put 0x558c578f0690 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c57b94c00 seq 12 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f06a8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c578f0700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0770 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x329000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0700) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57f4e000 space 0x558c578f0788 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x329000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0770) put 0x558c578f0770 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0788 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c578f07e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0850 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x32b000~2000] 2019-12-07T19:34:19.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f07e0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0868 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57f4e090 space 0x558c578f0868 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x32b000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0850) put 0x558c578f0850 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0868 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94c00 onodes shared_blobs 2019-12-07T19:34:19.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94c00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c578f08c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0930 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x32d000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f08c0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0948 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57f4e120 space 0x558c578f0948 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 prepare 2019-12-07T19:34:19.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94c00 2019-12-07T19:34:19.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 io_done 2019-12-07T19:34:19.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x32d000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0930) put 0x558c578f0930 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0948 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c578f09a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x127390 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x32f000~2000] 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x390 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f09a0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0a28 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x80d and padding block with 0x7f3 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57f4e1b0 space 0x558c578f0a28 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x32f000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0a10) put 0x558c578f0a10 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0a28 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c578f0a80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0af0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x331000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0a80) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0b08 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57f4e240 space 0x558c578f0b08 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x331000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0af0) put 0x558c578f0af0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0b08 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c578f0b60 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0bd0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x333000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0b60) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0be8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57f4e2d0 space 0x558c578f0be8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x333000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0bd0) put 0x558c578f0bd0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0be8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c578f0c40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0cb0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x335000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578f0c40) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0cc8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57f4e360 space 0x558c578f0cc8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x335000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578f0cb0) put 0x558c578f0cb0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578f0cc8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:41fe8a11:::smithi13140796-45:head# = 0 2019-12-07T19:34:19.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# onode 0x558c54d96c80 txc 0x558c57b95500 2019-12-07T19:34:19.617+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# 0x0 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578182a0) put_ref 0x0~2000 Blob(0x558c578182a0 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818230 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578181c0) put_ref 0x0~2000 Blob(0x558c578181c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578180e0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818070) put_ref 0x0~2000 Blob(0x558c57818070 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818000 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818310) put_ref 0x0~2000 Blob(0x558c57818310 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818380 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578183f0) put_ref 0x0~2000 Blob(0x558c578183f0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818460 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578184d0) put_ref 0x0~2000 Blob(0x558c578184d0 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818540 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578185b0) put_ref 0x0~2000 Blob(0x558c578185b0 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818690) put_ref 0x0~2000 Blob(0x558c57818690 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818700 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818770) put_ref 0x0~2000 Blob(0x558c57818770 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818850) put_ref 0x0~2000 Blob(0x558c57818850 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818930) put_ref 0x0~2000 Blob(0x558c57818930 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818a10) put_ref 0x0~2000 Blob(0x558c57818a10 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15780d 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818af0) put_ref 0x0~2000 Blob(0x558c57818af0 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818bd0) put_ref 0x0~2000 Blob(0x558c57818bd0 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818cb0) put_ref 0x0~2000 Blob(0x558c57818cb0 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818d20 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818d90) put_ref 0x0~2000 Blob(0x558c57818d90 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818e00 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000240623s (0.000000065s flush + 0.000240558s kv commit) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818e70) put_ref 0x0~2000 Blob(0x558c57818e70 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818ee0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818f50) put_ref 0x0~2000 Blob(0x558c57818f50 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818fc0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57819030) put_ref 0x0~2000 Blob(0x558c57819030 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57819110) put_ref 0x0~2000 Blob(0x558c57819110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819180 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578191f0) put_ref 0x0~2000 Blob(0x558c578191f0 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819260 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578192d0) put_ref 0x0~2000 Blob(0x558c578192d0 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.extentmap(0x558c54d96d88) dirty_range mark inline shard dirty 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.617+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c578182a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818230 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b6000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578182a0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818248 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57ece630 space 0x558c57818248 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15780d~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b6000~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15780d 0x15780d~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818230) put 0x558c57818230 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12780d 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x80d 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc7a and padding block with 0x386 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818248 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c578181c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578180e0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b8000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578181c0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578180f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c580cd710 space 0x558c578180f8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b8000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578180e0) put 0x558c578180e0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578180f8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57818070 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818000 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ba000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818070) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818018 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c578ee090 space 0x558c57818018 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ba000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818000) put 0x558c57818000 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818018 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57818310 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818380 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2bc000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818310) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818398 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57ecfe60 space 0x558c57818398 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2bc000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818380) put 0x558c57818380 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818398 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c578183f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818460 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2be000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578183f0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818478 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c57ecfef0 space 0x558c57818478 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2be000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818460) put 0x558c57818460 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818478 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c578184d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818540 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2c0000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578184d0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818558 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa000 space 0x558c57818558 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2c0000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818540) put 0x558c57818540 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818558 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c578185b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2c2000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578185b0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818638 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa090 space 0x558c57818638 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2c2000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818620) put 0x558c57818620 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818638 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57818690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818700 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2c4000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818690) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818718 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa120 space 0x558c57818718 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2c4000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818700) put 0x558c57818700 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818718 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57818770 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2c6000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818770) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578187f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa1b0 space 0x558c578187f8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2c6000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578187e0) put 0x558c578187e0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578187f8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57818850 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2c8000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818850) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578188d8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa240 space 0x558c578188d8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2c8000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578188c0) put 0x558c578188c0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578188d8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57818930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ca000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818930) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578189b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa2d0 space 0x558c578189b8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ca000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578189a0) put 0x558c578189a0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578189b8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57818a10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2cc000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818a10) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818a98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa360 space 0x558c57818a98 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2cc000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818a80) put 0x558c57818a80 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818a98 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57818af0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ce000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818af0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818b78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa3f0 space 0x558c57818b78 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ce000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818b60) put 0x558c57818b60 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818b78 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57818bd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2d0000~2000] 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x157c7a 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818bd0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818c58 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa480 space 0x558c57818c58 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2d0000~2000 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818c40) put 0x558c57818c40 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818c58 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57818cb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818d20 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2d2000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818cb0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818d38 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa510 space 0x558c57818d38 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2d2000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818d20) put 0x558c57818d20 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818d38 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112909s (0.000000087s flush + 0.000112822s kv commit) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c57818d90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818e00 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94c00> 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2d4000~2000] 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818d90) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818e18 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa5a0 space 0x558c57818e18 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2d4000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818e00) put 0x558c57818e00 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 kv_submitted 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818e18 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94c00 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 finishing 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94c00 onodes 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94c00 done 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94c00 [] 2019-12-07T19:34:19.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57818e70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818ee0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2d6000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818e70) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818ef8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa630 space 0x558c57818ef8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2d6000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818ee0) put 0x558c57818ee0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818ef8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57818f50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57818fc0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2d8000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57818f50) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818fd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa6c0 space 0x558c57818fd8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2d8000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57818fc0) put 0x558c57818fc0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57818fd8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c57819030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2da000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57819030) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578190b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa750 space 0x558c578190b8 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2da000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c578190a0) put 0x558c578190a0 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578190b8 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c57819110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819180 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2dc000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c57819110) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 280 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819198 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa7e0 space 0x558c57819198 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2dc000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57819180) put 0x558c57819180 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819198 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581ccdc0 prio 196 cost 0 latency 0.000010 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c578191f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819260 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2de000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578191f0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819278 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa870 space 0x558c57819278 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2de000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57819260) put 0x558c57819260 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819278 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c578192d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e0000~2000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.blob(0x558c578192d0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819358 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 _rm buffer(0x558c579fa900 space 0x558c57819358 0x0~2000 clean) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2e0000~2000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.sharedblob(0x558c57819340) put 0x558c57819340 removing self from set 0x558c54d39930 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57819358 in 0x558c53a595e0) _clear 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:4986b721:::smithi13140796-43:head# = 0 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95500 cost 4354 (1 ios * 4000 + 354 bytes) 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95500 onodes shared_blobs 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95500 allocated 0x[] released 0x[2b6000~2c000,309000~2e000] 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x2b6000~2c000 2019-12-07T19:34:19.617+0000 7f8f07988700 10 freelist release 0x2b6000~2c000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x309000~2e000 2019-12-07T19:34:19.617+0000 7f8f07988700 10 freelist release 0x309000~2e000 2019-12-07T19:34:19.617+0000 7f8f07988700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 prepare 2019-12-07T19:34:19.617+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95500 2019-12-07T19:34:19.617+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 io_done 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 242 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96780 had 1 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 265 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 265 < min_flush_size 524288 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x157c7a~109 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x157c7a 0x157c7a~109 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x127c7a 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc7a 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd83 and padding block with 0x27d 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c581ccdc0 prio 196 cost 0 latency 0.000152 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x157d83 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=26, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:276d5164:::smithi13140796-22:e v=42'6 tt=0'0 tid=26 reqid=osd.7.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112040s (0.000000070s flush + 0.000111970s kv commit) 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=37'4 crt=37'4 lcod 21'2 mlcod 0'0 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] repop_all_committed: repop tid 26 all committed 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] eval_repop repgather(0x558c576b6360 42'6 rep_tid=26 committed?=1 r=0) (no op) 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] commit: repgather(0x558c576b6360 42'6 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.618+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:93 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] removing repgather(0x558c576b6360 42'6 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] q front is repgather(0x558c576b6360 42'6 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] exit Trimming/WaitRepops 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95500> 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 kv_submitted 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95500 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 finishing 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95500 onodes 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95500 done 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95500 [2b6000~2c000,309000~2e000] 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x2b6000~2c000 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x309000~2e000 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 190 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558c576de700 con 0x558c54c5b400 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 42 queueing pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] for snaptrim 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.618+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.618+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:19.618+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] publish_stats_to_osd 42:94 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] remove_repop repgather(0x558c576b6360 42'6 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) pg 0x558c575ef000 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:e v=42'6 tt=0'0 tid=26 reqid=osd.7.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:19.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=27 reqid=osd.7.0:26 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:19.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c574fec00 seq 11 2019-12-07T19:34:19.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:19.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:19.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fec00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fec00 onodes shared_blobs 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fec00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 prepare 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=27 reqid=osd.7.0:26 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fec00 2019-12-07T19:34:19.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 io_done 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 42/41 ECSubWrite(tid=27, reqid=osd.7.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c57a45800 con 0x558c54c64000 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 637 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1034 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 42/41 ECSubWrite(tid=27, reqid=osd.7.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c574a4000 con 0x558c54c19800 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:19.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.4s1 42/41 ECSubWrite(tid=27, reqid=osd.7.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c574a5800 con 0x558c54c5bc00 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3859415040 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:19.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 42/41 ECSubWrite(tid=27, reqid=osd.7.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c574a5000 con 0x558c54c5b400 2019-12-07T19:34:19.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 42/41 ECSubWrite(tid=27, reqid=osd.7.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c5748e000 con 0x558c54c19c00 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998423) bluestore 86 GiB free (0.959901), bluefs_ratio 0.0399438 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=37'4 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] append_log log((0'0,42'6], crt=37'4) [] 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1057 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1057 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] rollforward: entry=42'5 (37'3) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x157d83~421 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x157d83 0x157d83~421 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] rollforward: entry=42'6 (37'4) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:19.613722+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x127d83 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd83 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] append_log approx pg log length = 6 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1a4 and padding block with 0xe5c 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.618+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.618+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c5730bb00 seq 30 2019-12-07T19:34:19.618+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:19.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730bb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730bb00 onodes shared_blobs 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 prepare 2019-12-07T19:34:19.618+0000 7f8f07988700 10 osd.7 42 queue_for_pg_delete on 1.2s4 e 42 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730bb00 2019-12-07T19:34:19.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 io_done 2019-12-07T19:34:19.618+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s4 PGDelete(1.2s4 e42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c581ccdc0 finish 2019-12-07T19:34:19.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGDelete(1.2s4 e42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 191 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c576de700 con 0x558c54c5b400 2019-12-07T19:34:19.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s4 PGDelete(1.2s4 e42) prio 5 cost 1048576 e42) pg 0x558c5731a000 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) pg 0x558c548ab000 2019-12-07T19:34:19.618+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.618+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteSome 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] snap_trimmer posting 2019-12-07T19:34:19.618+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.618+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_delete_work 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap d 2019-12-07T19:34:19.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 171 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c58134700 con 0x558c54c5bc00 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.618+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.2s4_head start GHMIN end GHMAX max 30 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:19.618+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.618+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list range 0x847ffffffffffffffd40000000 to 0x847ffffffffffffffd50000000 and 0x84800000000000000140000000 to 0x84800000000000000150000000 start GHMIN 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f300) upper_bound after SNA_1_000000000000000D_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.0_0000000000000001.4' 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list pend 0x847ffffffffffffffd50000000 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000014000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f300) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 4#1:40000000::::head# end GHMAX 2019-12-07T19:34:19.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000D_.0_0000000000000001.4) returns 0 SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239760 in raw 0x558c55239760 len 70 nref 1) ) 2019-12-07T19:34:19.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e42) pg 0x558c5736a000 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) pg 0x558c5513e000 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.2s4_head start GHMAX end GHMAX max 30 = 0, ls.size() = 1, next = GHMAX 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[d~1,10~1] ps=[a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:19.618+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1]] SnapTrimmer state: adding snap d to purged_snaps 2019-12-07T19:34:19.618+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_delete_work [4#1:40000000::::head#] 2019-12-07T19:34:19.618+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.618+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.012933555s) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active pruub 57.054502621s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.618+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s4( v 29'15 (0'0,29'15] lb MIN local-lis/les=28/29 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [7,4,1,5,2,3]p7(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=29'15 unknown NOTIFY mbc={} ps=[a~1]] do_delete_work finished 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:19.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39860 1.2s4_head 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active pruub 57.054502621s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e8fe00 = 0x558c5730ac00 seq 33 2019-12-07T19:34:19.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:19.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c57ff0900 seq 13 2019-12-07T19:34:19.618+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:40000000::::head# onode 0x558c574a0c80 txc 0x558c5730ac00 2019-12-07T19:34:19.618+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: purged_snaps now [a~1,d~1], snap_trimq now [10~1] 2019-12-07T19:34:19.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:19.618+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.2s4_head 4#1:40000000::::head# 0x0 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore.onode(0x558c574a0c80).flush flush done 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c548f6c00 seq 31 2019-12-07T19:34:19.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:19.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:19.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54dde600 seq 46 2019-12-07T19:34:19.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:19.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:19.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:19.618+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:19.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0900 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0900 onodes shared_blobs 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:19.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:19.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes shared_blobs 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:19.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde600 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde600 onodes shared_blobs 2019-12-07T19:34:19.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde600 allocated 0x[] released 0x[] 2019-12-07T19:34:19.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 prepare 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:34:19.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0900 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:34:19.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 io_done 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:34:19.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 prepare 2019-12-07T19:34:19.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde600 2019-12-07T19:34:19.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 io_done 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] share_pg_info 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1581a4 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c574e4000 con 0x558c54c64000 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000295667s (0.000000075s flush + 0.000295592s kv commit) 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_omap_clear remove range start: 0x000000000000045c'-' end: 0x000000000000045c'~' 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.2s4_head 4#1:40000000::::head# = 0 2019-12-07T19:34:19.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c574e4700 con 0x558c54c19800 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove_collection 1.2s4_head 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore.OnodeSpace(0x558c54d399a0 in 0x558c53aaf000) map_any 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 2 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2240 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c574e5c00 con 0x558c54c5bc00 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list range 0x847ffffffffffffffd40000000 to 0x847ffffffffffffffd50000000 and 0x84800000000000000140000000 to 0x84800000000000000150000000 start GHMIN 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3350 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list pend 0x847ffffffffffffffd50000000 2019-12-07T19:34:19.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c574e6a00 con 0x558c54c5b400 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4373 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000014000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4396 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4396 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 4#1:40000000::::head# end GHMAX 2019-12-07T19:34:19.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c574e7100 con 0x558c54c19c00 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1581a4~112c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove_collection oid 4#1:40000000::::head# 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1581a4 0x1581a4~112c to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_register_zombie 0x558c54e8fe00 1.2s4_head 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x1281a4 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove_collection 1.2s4_head = 0 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1a4 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730ac00 cost 4267 (1 ios * 4000 + 267 bytes) 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2d0 and padding block with 0xd30 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730ac00 onodes shared_blobs 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] exit Trimming 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 prepare 2019-12-07T19:34:19.618+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730ac00 2019-12-07T19:34:19.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 io_done 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:95 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] enter NotTrimming 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] exit NotTrimming 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fec00> 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming/WaitReservation 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 kv_submitted 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fec00 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 finishing 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fec00 onodes 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fec00 done 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fec00 [] 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:96 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] snap_trimmer complete 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1592d0 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000187745s (0.000000080s flush + 0.000187665s kv commit) 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 956 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:19.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: NotTrimming: trimming 10 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 979 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 979 < min_flush_size 524288 2019-12-07T19:34:19.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[10~1] ps=[a~1,d~1]] exit Trimming/WaitReservation 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1592d0~3d3 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1592d0 0x1592d0~3d3 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x1292d0 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2d0 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6a3 and padding block with 0x95d 2019-12-07T19:34:19.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:97 2019-12-07T19:34:19.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:19.618+0000 7f8f0b990700 10 osd.7 42 queueing pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[10~1] ps=[a~1,d~1]] for snaptrim 2019-12-07T19:34:19.618+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730bb00,0x558c57ff0900,0x558c548f6c00,0x558c54dde600> 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 kv_submitted 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730bb00 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 finishing 2019-12-07T19:34:19.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1] 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730bb00 onodes 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730bb00 done 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 kv_submitted 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730bb00 [] 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] publish_stats_to_osd 42:98 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 kv_submitted 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0900 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 finishing 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0900 onodes 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0900 done 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=42) prio 5 cost 1048576 e42) pg 0x558c548ab000 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0900 [] 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 finishing 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] snap_trimmer posting 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 kv_submitted 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde600 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 10 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 finishing 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde600 onodes 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde600 done 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde600 [] 2019-12-07T19:34:19.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:19.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c55047340) upper_bound after SNA_1_0000000000000010_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.0_0000000000000001.4' 2019-12-07T19:34:19.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.618+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c55047340) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19..' 2019-12-07T19:34:19.618+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000010_.0_0000000000000001.4) returns 0 SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:19.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 192 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558c576de700 con 0x558c54c5b400 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[10~1] ps=[a~1,d~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:19.618+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.618+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1]] SnapTrimmer state: adding snap 10 to purged_snaps 2019-12-07T19:34:19.618+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:19.618+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~1]] SnapTrimmer state: purged_snaps now [a~1,d~1,10~1], snap_trimq now [] 2019-12-07T19:34:19.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:19.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 193 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558c574e6a00 con 0x558c54c5b400 2019-12-07T19:34:19.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c5730a000 seq 32 2019-12-07T19:34:19.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:19.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:19.619+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:19.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:19.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a000 onodes shared_blobs 2019-12-07T19:34:19.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a000 allocated 0x[] released 0x[] 2019-12-07T19:34:19.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 prepare 2019-12-07T19:34:19.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a000 2019-12-07T19:34:19.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 io_done 2019-12-07T19:34:19.619+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~1]] share_pg_info 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) pg 0x558c575ef000 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.619+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55055100 con 0x558c54c19800 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.619+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55055800 con 0x558c54c5bc00 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c574ffb00 seq 12 2019-12-07T19:34:19.619+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55053500 con 0x558c54c5b400 2019-12-07T19:34:19.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ffb00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ffb00 onodes shared_blobs 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ffb00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 prepare 2019-12-07T19:34:19.619+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 -- 0x558c55052e00 con 0x558c54c19c00 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ffb00 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 io_done 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) pg 0x558c5513e000 2019-12-07T19:34:19.619+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:19.619+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~1]] exit Trimming 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.619+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.619+0000 7f8f06986700 15 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:99 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.619+0000 7f8f06986700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] enter NotTrimming 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.619+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c57ef8900 seq 47 2019-12-07T19:34:19.619+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:19.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:19.619+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] snap_trimmer complete 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:19.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:19.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8900 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8900 onodes shared_blobs 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 prepare 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8900 2019-12-07T19:34:19.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 io_done 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f07988700 20 osd.7 42 try_finish_pg_delete 1.2s4 0x558c5731a000 2019-12-07T19:34:19.619+0000 7f8f07988700 10 osd.7:2._detach_pg 1.2s4 0x558c5731a000 2019-12-07T19:34:19.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1596a3 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 172 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c58134700 con 0x558c54c5bc00 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.619+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000471745s (0.000000077s flush + 0.000471668s kv commit) 2019-12-07T19:34:19.619+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2220 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3314 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3337 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3337 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1596a3~d09 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1596a3 0x1596a3~d09 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x1296a3 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6a3 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3ac and padding block with 0xc54 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730ac00> 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 kv_submitted 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730ac00 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 finishing 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730ac00 onodes 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _queue_reap_collection 0x558c54d39860 1.2s4_head 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730ac00 done 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e8fe00 q now empty 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730ac00 [] 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish reaping empty zombie osr 0x558c54e8fe00 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _reap_collections 0x558c54d39860 1.2s4_head 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore.OnodeSpace(0x558c54d399a0 in 0x558c53aaf000) map_any 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore.OnodeSpace(0x558c54d399a0 in 0x558c53aaf000) clear 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _reap_collections 0x558c54d39860 1.2s4_head done 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _reap_collections all reaped 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.619+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.619+0000 7f8f07988700 10 bluestore.OnodeSpace(0x558c54d399a0 in 0x558c53aaf000) clear 2019-12-07T19:34:19.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) pg 0x558c57383000 2019-12-07T19:34:19.619+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.619+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:19.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c574fef00 seq 14 2019-12-07T19:34:19.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:19.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:19.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:19.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:19.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fef00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fef00 onodes shared_blobs 2019-12-07T19:34:19.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fef00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 194 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c55054300 con 0x558c54c5b400 2019-12-07T19:34:19.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 prepare 2019-12-07T19:34:19.619+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fef00 2019-12-07T19:34:19.619+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 io_done 2019-12-07T19:34:19.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15a3ac 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000381116s (0.000000080s flush + 0.000381036s kv commit) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) pg 0x558c575ef000 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15a3ac~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15a3ac 0x15a3ac~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12a3ac 2019-12-07T19:34:19.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3ac 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x819 and padding block with 0x7e7 2019-12-07T19:34:19.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:19.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c577fc300 seq 13 2019-12-07T19:34:19.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:19.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:19.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:19.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:19.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc300 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc300 onodes shared_blobs 2019-12-07T19:34:19.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc300 allocated 0x[] released 0x[] 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a000,0x558c574ffb00,0x558c57ef8900> 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 kv_submitted 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a000 2019-12-07T19:34:19.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 prepare 2019-12-07T19:34:19.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc300 2019-12-07T19:34:19.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 io_done 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 finishing 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a000 onodes 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 done 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:19.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a000 [] 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 kv_submitted 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ffb00 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 finishing 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ffb00 onodes 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ffb00 done 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc300 kv_queued 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ffb00 [] 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 kv_submitted 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8900 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 finishing 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8900 onodes 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8900 done 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8900 [] 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15a819 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131800s (0.000000081s flush + 0.000131719s kv commit) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15a819~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15a819 0x15a819~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12a819 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x819 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc86 and padding block with 0x37a 2019-12-07T19:34:19.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 195 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c55053500 con 0x558c54c5b400 2019-12-07T19:34:19.619+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 196 ==== osd pg remove(epoch 42; pg1.bs3; ) v3 ==== 32+0+0 (unknown 0 0 0) 0x558c575cdc20 con 0x558c54c5b400 2019-12-07T19:34:19.619+0000 7f8f24457700 7 osd.7 42 handle_fast_pg_remove osd pg remove(epoch 42; pg1.bs3; ) v3 from osd.4 2019-12-07T19:34:19.619+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.bs3 epoch_sent: 42 epoch_requested: 42 DeleteStart 2019-12-07T19:34:19.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteStart) prio 255 cost 10 e42) 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fef00> 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 kv_submitted 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fef00 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 finishing 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fef00 onodes 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fef00 done 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fef00 [] 2019-12-07T19:34:19.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15ac86 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000219989s (0.000000083s flush + 0.000219906s kv commit) 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc300> 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 kv_submitted 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc300 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 finishing 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc300 onodes 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc300 done 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc300 [] 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e42) pg 0x558c575ef000 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c577fcc00 seq 14 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcc00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcc00 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcc00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 prepare 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcc00 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 io_done 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteStart) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteStart) prio 255 cost 10 e42) pg 0x558c54d9b000 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteStart 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f07187700 5 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] exit Started/Stray 2019-12-07T19:34:19.621293+0000 6 0.000108 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15ac86~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15ac86 0x15ac86~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f07187700 5 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] enter Started/ToDelete 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12ac86 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc86 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x103 and padding block with 0xefd 2019-12-07T19:34:19.620+0000 7f8f07187700 5 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] enter Started/ToDelete/WaitDeleteReseved 2019-12-07T19:34:19.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 83 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c55045200 con 0x558c54c19c00 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581ccf20 prio 196 cost 0 latency 0.000017 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f0b990700 15 osd.7 42 enqueue_peering_evt 1.bs3 epoch_sent: 42 epoch_requested: 42 DeleteReserved 2019-12-07T19:34:19.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.620+0000 7f8f0b990700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteReserved) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteReserved) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 DeleteReserved) prio 255 cost 10 e42) pg 0x558c54d9b000 2019-12-07T19:34:19.620+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581ccf20 prio 196 cost 0 latency 0.000062 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteReserved 2019-12-07T19:34:19.620+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581ccf20 finish 2019-12-07T19:34:19.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f0317f700 5 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] exit Started/ToDelete/WaitDeleteReseved 2019-12-07T19:34:19.621420+0000 1 0.000034 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15b103 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.620+0000 7f8f0317f700 5 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] enter Started/ToDelete/Deleting 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000166084s (0.000000088s flush + 0.000165996s kv commit) 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcc00> 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] on_removal 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 kv_submitted 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcc00 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 finishing 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcc00 onodes 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcc00 done 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] on_shutdown 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcc00 [] 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] cancel_copy_ops 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 197 ==== MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=71, reqid=osd.4.0:71, at_version=42'15, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 3394+0+0 (unknown 0 0 0) 0x558c574a5000 con 0x558c54c5b400 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] cancel_flush_ops 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd080 prio 127 cost 0 latency 0.000029 epoch 42 MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=71, reqid=osd.4.0:71, at_version=42'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=71, reqid=osd.4.0:71, at_version=42'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] cancel_proxy_ops 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] cancel_manifest_ops 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=71, reqid=osd.4.0:71, at_version=42'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=71, reqid=osd.4.0:71, at_version=42'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) pg 0x558c5513e000 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] clear_backoffs 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] exit NotTrimming 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 42 dequeue_op 0x558c581cd080 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=71, reqid=osd.4.0:71, at_version=42'15, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] enter NotTrimming 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=71, reqid=osd.4.0:71, at_version=42'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.620+0000 7f8f07187700 20 snap_mapper.update_snaps 1:c8ce3553:::smithi13140796-29:e c,e was 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] on_change 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] clear_async_reads 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 20 snap_mapper.get_snaps 1:c8ce3553:::smithi13140796-29:e c,d,e 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:19.620+0000 7f8f07187700 20 snap_mapper.set_snaps 1:c8ce3553:::smithi13140796-29:e c,e 2019-12-07T19:34:19.620+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:19.620+0000 7f8f07187700 20 snap_mapper.update_snaps rm SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] cancel_recovery 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] clear_recovery_state 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 37'13 (0'0,37'13] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] append_log log((0'0,37'13], crt=37'13) [42'14 (37'12) modify 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:19.620903+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,42'15 (37'13) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:19.620903+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c57ef8000 seq 25 2019-12-07T19:34:19.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'14 (0'0,42'14] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=37'13 crt=37'13 lcod 37'13 active mbc={} ps=[a~1]] add_log_entry 42'14 (37'12) modify 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:19.620903+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._epoch' <- _epoch 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000045b'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8000 cost 5210 (1 ios * 4000 + 1210 bytes) 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=37'13 crt=37'13 lcod 37'13 active mbc={} ps=[a~1]] add_log_entry 42'15 (37'13) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:19.620903+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8000 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8000 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=37'13 crt=37'13 lcod 37'13 active mbc={} ps=[a~1]] append_log approx pg log length = 15 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 prepare 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8000 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=37'13 crt=37'13 lcod 37'13 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 io_done 2019-12-07T19:34:19.620+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1047 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1070 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1070 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5730b200 seq 48 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15b103~42e to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15b103 0x15b103~42e to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 1 keys 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12b103 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 1 keys = 0 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x103 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x531 and padding block with 0xacf 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000D_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000042.00000000000000000014' <- 0000000042.00000000000000000014 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000042.00000000000000000015' <- 0000000042.00000000000000000015 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b200 cost 8355 (1 ios * 4000 + 4355 bytes) 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b200 onodes 0x558c57976500,0x558c57976780 shared_blobs 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976608) update 4#1:c8ce3553:::smithi13140796-29:e# 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:e# is 823 (412 bytes onode + 2 bytes spanning blobs + 409 bytes inline extents) 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976888) update 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:head# is 919 (549 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 prepare 2019-12-07T19:34:19.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b200 2019-12-07T19:34:19.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 io_done 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=37'13 crt=37'13 lcod 37'13 active mbc={} ps=[a~1]] op_applied version 42'15 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15b531 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 42 dequeue_op 0x558c581cd080 finish 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 281 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55053500 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000137884s (0.000000091s flush + 0.000137793s kv commit) 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4567 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976500 had 1 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976780 had 1 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4590 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4590 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 282 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15b531~11ee to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15b531 0x15b531~11ee to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12b531 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x531 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x71f and padding block with 0x8e1 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 283 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.7s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 284 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015507853s) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active pruub 57.057013246s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 285 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.9s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active pruub 57.057013246s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 286 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b95200 seq 51 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95200 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95200 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 prepare 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95200 2019-12-07T19:34:19.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 io_done 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 287 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8000> 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.9s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 kv_submitted 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8000 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8000 finishing 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8000 onodes 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8000 done 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8000 [] 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) pg 0x558c5500a000 2019-12-07T19:34:19.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 288 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.620+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.015878610s) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active pruub 57.057641346s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active pruub 57.057641346s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.620+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57b94f00 seq 47 2019-12-07T19:34:19.620+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94f00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94f00 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94f00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 prepare 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94f00 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 io_done 2019-12-07T19:34:19.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 198 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c1fe00 con 0x558c54c5b400 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd1e0 prio 196 cost 0 latency 0.000010 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) pg 0x558c5500a000 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) pg 0x558c54d98000 2019-12-07T19:34:19.620+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) pg 0x558c57380000 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57b94600 seq 48 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:19.620+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5730b500 seq 8 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94600 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94600 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94600 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 prepare 2019-12-07T19:34:19.620+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94600 2019-12-07T19:34:19.620+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 io_done 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b500 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b500 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:19.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c54d21b00 seq 20 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 prepare 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b500 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 io_done 2019-12-07T19:34:19.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) pg 0x558c54d98000 2019-12-07T19:34:19.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21b00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21b00 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21b00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 prepare 2019-12-07T19:34:19.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21b00 2019-12-07T19:34:19.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 io_done 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 42 queue_for_pg_delete on 1.bs3 e 42 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15c71f 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGDelete(1.bs3 e42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) pg 0x558c57380000 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e42) prio 5 cost 1048576 e42) pg 0x558c54d9b000 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:19.620+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000401922s (0.000000071s flush + 0.000401851s kv commit) 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5730bb00 seq 9 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:34:19.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:19.620+0000 7f8f04982700 20 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteSome 2019-12-07T19:34:19.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2164 < min_flush_size 524288 2019-12-07T19:34:19.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730bb00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730bb00 onodes shared_blobs 2019-12-07T19:34:19.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:19.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:19.620+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_delete_work 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3258 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:19.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 prepare 2019-12-07T19:34:19.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730bb00 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c577fd200 seq 21 2019-12-07T19:34:19.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 io_done 2019-12-07T19:34:19.621+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:19.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4281 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:19.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:19.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 2 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd200 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd200 onodes shared_blobs 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5304 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 2 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5327 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5327 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 prepare 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd200 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 io_done 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15c71f~14cf to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c581cd1e0 prio 196 cost 0 latency 0.000204 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15c71f 0x15c71f~14cf to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) pg 0x558c57380000 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12c71f 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x71f 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbee and padding block with 0x412 2019-12-07T19:34:19.621+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.621+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c581cd1e0 finish 2019-12-07T19:34:19.621+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f04982700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.621+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c577fcf00 seq 22 2019-12-07T19:34:19.621+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b200> 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b200 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b200 finishing 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b200 onodes 0x558c57976500,0x558c57976780 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b200 done 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcf00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcf00 onodes shared_blobs 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b200 [] 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcf00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 prepare 2019-12-07T19:34:19.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcf00 2019-12-07T19:34:19.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 io_done 2019-12-07T19:34:19.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.bs3_head start GHMIN end GHMAX max 30 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15dbee 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list range 0x837ffffffffffffffdd0000000 to 0x837ffffffffffffffde0000000 and 0x838000000000000001d0000000 to 0x838000000000000001e0000000 start GHMIN 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 5 cleaned 0 in 0.000176440s (0.000000074s flush + 0.000176366s kv commit) 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list pend 0x837ffffffffffffffde0000000 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8380000000000000010000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2188 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 3#1:d0000000::::head# end GHMAX 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 2 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 3#1:d313ddd1:::smithi13140796-49:head# end GHMAX 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3298 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 3#1:db85c185:::smithi13140796-18:head# end GHMAX 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 3#1:dce63cc3:::smithi13140796-50:head# end GHMAX 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3321 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3321 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.bs3_head start GHMAX end GHMAX max 30 = 0, ls.size() = 4, next = GHMAX 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15dbee~cf9 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15dbee 0x15dbee~cf9 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12dbee 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbee 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8e7 and padding block with 0x719 2019-12-07T19:34:19.621+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_delete_work [3#1:d0000000::::head#,3#1:d313ddd1:::smithi13140796-49:head#,3#1:db85c185:::smithi13140796-18:head#,3#1:dce63cc3:::smithi13140796-50:head#] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper.remove_oid 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper._remove_oid 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper.get_snaps 1:d313ddd1:::smithi13140796-49:head got.empty() 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper.remove_oid 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper._remove_oid 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper.get_snaps 1:db85c185:::smithi13140796-18:head got.empty() 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper.remove_oid 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper._remove_oid 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 snap_mapper.get_snaps 1:dce63cc3:::smithi13140796-50:head got.empty() 2019-12-07T19:34:19.621+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_delete_work deleting 3 objects 2019-12-07T19:34:19.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c5730af00 seq 26 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# onode 0x558c57976000 txc 0x558c5730af00 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac1c0) put_ref 0x0~2000 Blob(0x558c576ac1c0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac4d0) put_ref 0x0~2000 Blob(0x558c576ac4d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396620 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579707e0) put_ref 0x0~2000 Blob(0x558c579707e0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579708c0) put_ref 0x0~2000 Blob(0x558c579708c0 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579709a0) put_ref 0x0~2000 Blob(0x558c579709a0 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c54677260) put_ref 0x0~2000 Blob(0x558c54677260 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac2a0) put_ref 0x0~2000 Blob(0x558c576ac2a0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac230 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac380) put_ref 0x0~2000 Blob(0x558c576ac380 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c54e42700) put_ref 0x0~2000 Blob(0x558c54e42700 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b28c0) put_ref 0x0~2000 Blob(0x558c579b28c0 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2850 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b27e0) put_ref 0x0~2000 Blob(0x558c579b27e0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2700) put_ref 0x0~2000 Blob(0x558c579b2700 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2690 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2620) put_ref 0x0~2000 Blob(0x558c579b2620 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b25b0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2540) put_ref 0x0~2000 Blob(0x558c579b2540 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b24d0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2460) put_ref 0x0~2000 Blob(0x558c579b2460 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b23f0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2380) put_ref 0x0~2000 Blob(0x558c579b2380 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2310 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b22a0) put_ref 0x0~2000 Blob(0x558c579b22a0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a80 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95200,0x558c57b94f00,0x558c57b94600,0x558c5730b500,0x558c54d21b00> 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95200 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970af0) put_ref 0x0~2000 Blob(0x558c57970af0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970b60 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970bd0) put_ref 0x0~2000 Blob(0x558c57970bd0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970c40 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95200 finishing 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95200 onodes 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970cb0) put_ref 0x0~2000 Blob(0x558c57970cb0 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95200 done 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95200 [] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970d90) put_ref 0x0~2000 Blob(0x558c57970d90 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e00 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970e70) put_ref 0x0~2000 Blob(0x558c57970e70 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970ee0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94f00 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970f50) put_ref 0x0~2000 Blob(0x558c57970f50 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970fc0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94f00 finishing 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57971030) put_ref 0x0~2000 Blob(0x558c57971030 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579710a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94f00 onodes 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94f00 done 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94600 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94f00 [] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57971110) put_ref 0x0~2000 Blob(0x558c57971110 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976108) dirty_range mark inline shard dirty 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94600 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94600 finishing 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94600 onodes 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94600 done 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c576ac1c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94600 [] 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x40c000~2000] 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac1c0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b500 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac478 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57982990 space 0x558c576ac478 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b500 finishing 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b500 onodes 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b500 done 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730bb00 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x40c000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ac460) put 0x558c576ac460 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15e8e7 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac478 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c576ac4d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57396620 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x40e000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac4d0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57396638 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28ab0 space 0x558c57396638 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x40e000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57396620) put 0x558c57396620 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57396638 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000232743s (0.000000081s flush + 0.000232662s kv commit) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c579707e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x410000~2000] 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579707e0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28b40 space 0x558c57970868 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b500 [] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x410000~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970850) put 0x558c57970850 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970868 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c579708c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21b00 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x412000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579708c0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28bd0 space 0x558c57970948 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x412000~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21b00 finishing 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970930) put 0x558c57970930 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21b00 onodes 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970948 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21b00 done 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd200 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21b00 [] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c579709a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x414000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579709a0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730bb00,0x558c577fd200,0x558c577fcf00> 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28c60 space 0x558c57970a28 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730bb00 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730bb00 finishing 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x414000~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730bb00 onodes 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970a10) put 0x558c57970a10 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730bb00 done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a28 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730bb00 [] 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c54677260 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x416000~2000] 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd200 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c54677260) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad7b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c54dba000 space 0x558c576ad7b8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd200 finishing 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd200 onodes 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x416000~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd200 done 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcf00 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd200 [] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ad7a0) put 0x558c576ad7a0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 kv_submitted 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad7b8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcf00 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c576ac2a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ac230 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcf00 finishing 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x418000~2000] 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcf00 onodes 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac2a0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcf00 done 2019-12-07T19:34:19.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac248 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcf00 [] 2019-12-07T19:34:19.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c5796fb90 space 0x558c576ac248 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x418000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ac230) put 0x558c576ac230 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac248 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c576ac380 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x41a000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac380) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c5796fc20 space 0x558c576ac328 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x41a000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ac310) put 0x558c576ac310 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac328 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c54e42700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x41c000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c54e42700) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c573965c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c54dba090 space 0x558c573965c8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x41c000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c573965b0) put 0x558c573965b0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c573965c8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c579b28c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2850 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x41e000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b28c0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e29560 space 0x558c579b2868 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x41e000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2850) put 0x558c579b2850 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2868 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c579b27e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x420000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b27e0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e294d0 space 0x558c579b2788 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x420000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2770) put 0x558c579b2770 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2788 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c579b2700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2690 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x422000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2700) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b26a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e29440 space 0x558c579b26a8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x422000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2690) put 0x558c579b2690 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b26a8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c579b2620 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b25b0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x424000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2620) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b25c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e293b0 space 0x558c579b25c8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x424000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b25b0) put 0x558c579b25b0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b25c8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c579b2540 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b24d0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x426000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2540) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b24e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e29320 space 0x558c579b24e8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x426000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b24d0) put 0x558c579b24d0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b24e8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c579b2460 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b23f0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x428000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2460) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2408 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e29290 space 0x558c579b2408 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x428000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b23f0) put 0x558c579b23f0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2408 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c579b2380 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2310 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x42a000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2380) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e29200 space 0x558c579b2328 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x42a000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2310) put 0x558c579b2310 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2328 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c579b22a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970a80 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x42c000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b22a0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e29170 space 0x558c57970a98 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x42c000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970a80) put 0x558c57970a80 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a98 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57970af0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970b60 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x42e000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970af0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28cf0 space 0x558c57970b78 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x42e000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970b60) put 0x558c57970b60 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970b78 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c57970bd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970c40 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x430000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970bd0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28d80 space 0x558c57970c58 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x430000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970c40) put 0x558c57970c40 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970c58 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c57970cb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970d20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x432000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970cb0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28e10 space 0x558c57970d38 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x432000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970d20) put 0x558c57970d20 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970d38 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c57970d90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970e00 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x434000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970d90) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28ea0 space 0x558c57970e18 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x434000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970e00) put 0x558c57970e00 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970e18 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c57970e70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970ee0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x436000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970e70) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28f30 space 0x558c57970ef8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x436000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970ee0) put 0x558c57970ee0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970ef8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c57970f50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57970fc0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x438000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970f50) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e28fc0 space 0x558c57970fd8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x438000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57970fc0) put 0x558c57970fc0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970fd8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c57971030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579710a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x43a000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57971030) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579710b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e29050 space 0x558c579710b8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x43a000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579710a0) put 0x558c579710a0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579710b8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c57971110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x43c000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c57971110) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57e290e0 space 0x558c57971198 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x43c000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57971180) put 0x558c57971180 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# = 0 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# onode 0x558c575bb680 txc 0x558c5730af00 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c574f1960) put_ref 0x0~2000 Blob(0x558c574f1960 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad730) put_ref 0x0~2000 Blob(0x558c576ad730 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad880) put_ref 0x0~2000 Blob(0x558c576ad880 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad8f0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad960) put_ref 0x0~2000 Blob(0x558c576ad960 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ada40) put_ref 0x0~2000 Blob(0x558c576ada40 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adab0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adb20) put_ref 0x0~2000 Blob(0x558c576adb20 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adc00) put_ref 0x0~2000 Blob(0x558c576adc00 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adce0) put_ref 0x0~2000 Blob(0x558c576adce0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576add50 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576addc0) put_ref 0x0~2000 Blob(0x558c576addc0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ade30 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c575bb788) dirty_range mark inline shard dirty 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574f1960 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3c7000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c574f1960) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57a08b40 space 0x558c574f1208 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3c7000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c574f11f0) put 0x558c574f11f0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c576ad730 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3c9000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad730) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57983d40 space 0x558c576ad828 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3c9000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ad810) put 0x558c576ad810 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c576ad880 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ad8f0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3cb000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad880) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad908 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c551a8c60 space 0x558c576ad908 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3cb000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ad8f0) put 0x558c576ad8f0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad908 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c576ad960 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3cd000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad960) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad9e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c573d8510 space 0x558c576ad9e8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3cd000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ad9d0) put 0x558c576ad9d0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad9e8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c576ada40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576adab0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3cf000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ada40) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576adac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57983dd0 space 0x558c576adac8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3cf000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576adab0) put 0x558c576adab0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576adac8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c576adb20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3d1000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adb20) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57983e60 space 0x558c576adba8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3d1000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576adb90) put 0x558c576adb90 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c576adc00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3d3000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adc00) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576adc88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57983ef0 space 0x558c576adc88 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3d3000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576adc70) put 0x558c576adc70 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576adc88 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c576adce0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576add50 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3d5000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adce0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576add68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f774d0 space 0x558c576add68 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3d5000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576add50) put 0x558c576add50 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576add68 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c576addc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c576ade30 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3d7000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576addc0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ade48 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77560 space 0x558c576ade48 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3d7000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c576ade30) put 0x558c576ade30 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ade48 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# = 0 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# onode 0x558c57572000 txc 0x558c5730af00 2019-12-07T19:34:19.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac000) put_ref 0x0~2000 Blob(0x558c576ac000 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57566d20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c54e424d0) put_ref 0x0~2000 Blob(0x558c54e424d0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c574f1ab0) put_ref 0x0~2000 Blob(0x558c574f1ab0 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdc70 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c574fdc00) put_ref 0x0~2000 Blob(0x558c574fdc00 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdb90 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2930) put_ref 0x0~2000 Blob(0x558c579b2930 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b29a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2a10) put_ref 0x0~2000 Blob(0x558c579b2a10 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2a80 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2af0) put_ref 0x0~2000 Blob(0x558c579b2af0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2b60 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2bd0) put_ref 0x0~2000 Blob(0x558c579b2bd0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2c40 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2cb0) put_ref 0x0~2000 Blob(0x558c579b2cb0 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2d90) put_ref 0x0~2000 Blob(0x558c579b2d90 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2e70) put_ref 0x0~2000 Blob(0x558c579b2e70 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2f50) put_ref 0x0~2000 Blob(0x558c579b2f50 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2fc0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3030) put_ref 0x0~2000 Blob(0x558c579b3030 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3110) put_ref 0x0~2000 Blob(0x558c579b3110 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3180 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b31f0) put_ref 0x0~2000 Blob(0x558c579b31f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3260 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b32d0) put_ref 0x0~2000 Blob(0x558c579b32d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3340 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b33b0) put_ref 0x0~2000 Blob(0x558c579b33b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3420 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3490) put_ref 0x0~2000 Blob(0x558c579b3490 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3500 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3570) put_ref 0x0~2000 Blob(0x558c579b3570 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b35e0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3650) put_ref 0x0~2000 Blob(0x558c579b3650 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b36c0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3730) put_ref 0x0~2000 Blob(0x558c579b3730 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3810) put_ref 0x0~2000 Blob(0x558c579b3810 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b38f0) put_ref 0x0~2000 Blob(0x558c579b38f0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b39d0) put_ref 0x0~2000 Blob(0x558c579b39d0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3ab0) put_ref 0x0~2000 Blob(0x558c579b3ab0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3b20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b90) put_ref 0x0~1000 Blob(0x558c579b3b90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c57572108) dirty_range mark inline shard dirty 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c576ac000 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57566d20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3d9000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ac000) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57566d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f777a0 space 0x558c57566d38 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3d9000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c57566d20) put 0x558c57566d20 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57566d38 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54e424d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757e9a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3db000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c54e424d0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f775f0 space 0x558c5757e9b8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3db000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c5757e9a0) put 0x558c5757e9a0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5757e9b8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c574f1ab0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdc70 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3dd000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c574f1ab0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 118 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c54c19800 2019-12-07T19:34:19.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fdc88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f24c58700 15 osd.7 42 enqueue_op 0x558c57efedc0 prio 196 cost 0 latency 0.000016 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77680 space 0x558c574fdc88 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3dd000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c574fdc70) put 0x558c574fdc70 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fdc88 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574fdc00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdb90 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3df000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c574fdc00) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fdba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77710 space 0x558c574fdba8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3df000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c574fdb90) put 0x558c574fdb90 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fdba8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c579b2930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b29a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3e1000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2930) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b29b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c57efedc0 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 173 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c54e04700 con 0x558c54c5bc00 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77830 space 0x558c579b29b8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3e1000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b29a0) put 0x558c579b29a0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b29b8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c579b2a10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2a80 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3e3000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2a10) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2a98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f778c0 space 0x558c579b2a98 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3e3000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2a80) put 0x558c579b2a80 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2a98 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c57efedc0 finish 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c579b2af0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2b60 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3e5000~2000] 2019-12-07T19:34:19.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2af0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77950 space 0x558c579b2b78 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3e5000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2b60) put 0x558c579b2b60 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2b78 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c579b2bd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2c40 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3e7000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2bd0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f779e0 space 0x558c579b2c58 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3e7000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2c40) put 0x558c579b2c40 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2c58 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c579b2cb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3e9000~2000] 2019-12-07T19:34:19.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2cb0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77a70 space 0x558c579b2d38 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3e9000~2000 2019-12-07T19:34:19.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e42) pg 0x558c57383000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2d20) put 0x558c579b2d20 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2d38 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c579b2d90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3eb000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2d90) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77b00 space 0x558c579b2e18 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3eb000~2000 2019-12-07T19:34:19.621+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2e00) put 0x558c579b2e00 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2e18 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c579b2e70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3ed000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2e70) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77b90 space 0x558c579b2ef8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3ed000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2ee0) put 0x558c579b2ee0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2ef8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c579b2f50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b2fc0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3ef000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2f50) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77c20 space 0x558c579b2fd8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:19.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c574ff500 seq 15 2019-12-07T19:34:19.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:19.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3ef000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b2fc0) put 0x558c579b2fc0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2fd8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c579b3030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3f1000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3030) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77cb0 space 0x558c579b30b8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3f1000~2000 2019-12-07T19:34:19.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff500 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b30a0) put 0x558c579b30a0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff500 onodes shared_blobs 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c579b3110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3180 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3f3000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3110) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57f77d40 space 0x558c579b3198 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3f3000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3180) put 0x558c579b3180 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3198 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c579b31f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3260 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3f5000~2000] 2019-12-07T19:34:19.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 prepare 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b31f0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3278 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff500 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c45290 space 0x558c579b3278 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3f5000~2000 2019-12-07T19:34:19.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 io_done 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3260) put 0x558c579b3260 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3278 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c579b32d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3340 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3f7000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b32d0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3358 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c45320 space 0x558c579b3358 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3f7000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3340) put 0x558c579b3340 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3358 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c579b33b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3420 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3f9000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b33b0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3438 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c453b0 space 0x558c579b3438 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3f9000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3420) put 0x558c579b3420 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3438 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c579b3490 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3500 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3fb000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3490) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3518 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c45440 space 0x558c579b3518 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3fb000~2000 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3500) put 0x558c579b3500 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3518 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c579b3570 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b35e0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3fd000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3570) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b35f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c454d0 space 0x558c579b35f8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3fd000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b35e0) put 0x558c579b35e0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b35f8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c579b3650 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b36c0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3ff000~2000] 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3650) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b36d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c45560 space 0x558c579b36d8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3ff000~2000 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b36c0) put 0x558c579b36c0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b36d8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c579b3730 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15e8e7~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x401000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3730) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b37b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c455f0 space 0x558c579b37b8 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15e8e7 0x15e8e7~47d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x401000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b37a0) put 0x558c579b37a0 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b37b8 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c579b3810 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12e8e7 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x403000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3810) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8e7 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c45680 space 0x558c579b3898 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x403000~2000 2019-12-07T19:34:19.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd64 and padding block with 0x29c 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3880) put 0x558c579b3880 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3898 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c579b38f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x405000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b38f0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c45710 space 0x558c579b3978 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x405000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3960) put 0x558c579b3960 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3978 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c579b39d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x407000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b39d0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c457a0 space 0x558c579b3a58 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x407000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3a40) put 0x558c579b3a40 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3a58 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c579b3ab0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c579b3b20 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x409000~2000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3ab0) discard_unallocated 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c45830 space 0x558c579b3b38 0x0~2000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x409000~2000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3b20) put 0x558c579b3b20 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3b38 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~1000: 0x0~1000 Blob(0x558c579b3b90 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x40b000~1000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b90) discard_unallocated 0x0~1000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 _rm buffer(0x558c57c458c0 space 0x558c579b3c18 0x0~1000 clean) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x40b000~1000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.sharedblob(0x558c579b3c00) put 0x558c579b3c00 removing self from set 0x558c54d0f390 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _clear 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730af00 cost 4496 (1 ios * 4000 + 496 bytes) 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730af00 onodes shared_blobs 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730af00 allocated 0x[] released 0x[3c7000~77000] 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x3c7000~77000 2019-12-07T19:34:19.621+0000 7f8f07187700 10 freelist release 0x3c7000~77000 2019-12-07T19:34:19.621+0000 7f8f07187700 20 freelist _xor first_key 0x380000 last_key 0x400000 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 prepare 2019-12-07T19:34:19.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730af00 2019-12-07T19:34:19.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 io_done 2019-12-07T19:34:19.621+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 42/41 ECSubWriteReply(tid=71, last_complete=42'15, committed=1, applied=1)) v2 -- 0x558c54c59e00 con 0x558c54c5b400 2019-12-07T19:34:19.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 119 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55055100 con 0x558c54c19800 2019-12-07T19:34:19.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.621+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.ds2 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) pg 0x558c55141000 2019-12-07T19:34:19.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=39/18 les/c/f=40/19/0 sis=41 pruub=14.013160602s) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active pruub 57.054473552s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active pruub 57.054473552s@ mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41 2019-12-07T19:34:19.621+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:19.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c577fc000 seq 30 2019-12-07T19:34:19.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:19.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:19.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc000 onodes shared_blobs 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc000 allocated 0x[] released 0x[] 2019-12-07T19:34:19.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 prepare 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc000 2019-12-07T19:34:19.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 io_done 2019-12-07T19:34:19.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15ed64 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000232444s (0.000000098s flush + 0.000232346s kv commit) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff500> 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 kv_submitted 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff500 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 296 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff500 finishing 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff500 onodes 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff500 done 2019-12-07T19:34:19.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 289 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff500 [] 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572000 had 1 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb680 had 1 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.622+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.622+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1319 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:19.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1342 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1342 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15ed64~53e to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15ed64 0x15ed64~53e to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12ed64 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd64 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2a2 and padding block with 0xd5e 2019-12-07T19:34:19.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) pg 0x558c54d98000 2019-12-07T19:34:19.622+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.622+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.622+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:19.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:19.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c57ef8c00 seq 10 2019-12-07T19:34:19.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:19.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:19.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:19.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:19.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8c00 onodes shared_blobs 2019-12-07T19:34:19.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8c00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 prepare 2019-12-07T19:34:19.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8c00 2019-12-07T19:34:19.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 io_done 2019-12-07T19:34:19.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15f2a2 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000155117s (0.000000085s flush + 0.000155032s kv commit) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730af00,0x558c577fc000> 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 kv_submitted 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730af00 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730af00 finishing 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730af00 onodes 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730af00 done 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730af00 [3c7000~77000] 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x3c7000~77000 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 kv_submitted 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc000 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc000 finishing 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc000 onodes 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc000 done 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc000 [] 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15f2a2~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15f2a2 0x15f2a2~46d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12f2a2 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2a2 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x70f and padding block with 0x8f1 2019-12-07T19:34:19.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15f70f 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112787s (0.000000073s flush + 0.000112714s kv commit) 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.622+0000 7f8f07187700 10 osd.7 42 queue_for_pg_delete on 1.bs3 e 42 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8c00> 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGDelete(1.bs3 e42) prio 5 cost 1048576 e42) 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 kv_submitted 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8c00 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8c00 finishing 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8c00 onodes 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8c00 done 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e42) prio 5 cost 1048576 e42) queued 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8c00 [] 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e42) prio 5 cost 1048576 e42) pg 0x558c54d9b000 2019-12-07T19:34:19.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 290 ==== MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=200, reqid=client.4540.0:333, at_version=42'19, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 73044+0+0 (crc 0 0 0) 0x558c57a45800 con 0x558c54c64000 2019-12-07T19:34:19.622+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 DeleteSome 2019-12-07T19:34:19.622+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd340 prio 127 cost 0 latency 0.000023 epoch 42 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=200, reqid=client.4540.0:333, at_version=42'19, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.622+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_delete_work 2019-12-07T19:34:19.622+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=200, reqid=client.4540.0:333, at_version=42'19, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.bs3_head start GHMIN end GHMAX max 30 2019-12-07T19:34:19.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=200, reqid=client.4540.0:333, at_version=42'19, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=200, reqid=client.4540.0:333, at_version=42'19, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) pg 0x558c5500a000 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list range 0x837ffffffffffffffdd0000000 to 0x837ffffffffffffffde0000000 and 0x838000000000000001d0000000 to 0x838000000000000001e0000000 start GHMIN 2019-12-07T19:34:19.622+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c581cd340 prio 127 cost 0 latency 0.000046 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=200, reqid=client.4540.0:333, at_version=42'19, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] 2019-12-07T19:34:19.622+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=200, reqid=client.4540.0:333, at_version=42'19, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.622+0000 7f8f04181700 20 snap_mapper.add_oid 1:9b8adabe:::smithi13140796-6:11 c,e,11 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list pend 0x837ffffffffffffffde0000000 2019-12-07T19:34:19.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8380000000000000010000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 3#1:d0000000::::head# end GHMAX 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.622+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:11 got.empty() 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) collection_list 1.bs3_head start GHMAX end GHMAX max 30 = 0, ls.size() = 1, next = GHMAX 2019-12-07T19:34:19.622+0000 7f8f04181700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:11 c,e,11 2019-12-07T19:34:19.622+0000 7f8f04181700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_delete_work [3#1:d0000000::::head#] 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=28/29 n=3 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41 pruub=14.013037390s) [4,7,2,3,1,5]p4(0) r=-1 lpr=41 DELETING pi=[28,41)/1 crt=21'12 unknown NOTIFY pruub 57.054279632s@ mbc={} ps=[a~1]] do_delete_work finished 2019-12-07T19:34:19.622+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.622+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.622+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_0000000000000011_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576a28c0 = 0x558c54d20600 seq 27 2019-12-07T19:34:19.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:d0000000::::head# onode 0x558c54d2c280 txc 0x558c54d20600 2019-12-07T19:34:19.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs3_head 3#1:d0000000::::head# 0x0 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore.onode(0x558c54d2c280).flush flush done 2019-12-07T19:34:19.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 34'17 (0'0,34'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] append_log log((0'0,34'17], crt=34'17) [42'18 (34'17) clone 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,42'19 (34'17) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:333 2019-12-07T19:34:18.620957+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_omap_clear remove range start: 0x000000000000045b'-' end: 0x000000000000045b'~' 2019-12-07T19:34:19.622+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'18 (0'0,42'18] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=34'17 crt=34'17 lcod 34'17 active mbc={} ps=[a~1]] add_log_entry 42'18 (34'17) clone 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:19.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove_collection 1.bs3_head 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore.OnodeSpace(0x558c54d0f400 in 0x558c54842800) map_any 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list range 0x837ffffffffffffffdd0000000 to 0x837ffffffffffffffde0000000 and 0x838000000000000001d0000000 to 0x838000000000000001e0000000 start GHMIN 2019-12-07T19:34:19.622+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=34'17 crt=34'17 lcod 34'17 active mbc={} ps=[a~1]] add_log_entry 42'19 (34'17) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:333 2019-12-07T19:34:18.620957+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list pend 0x837ffffffffffffffde0000000 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8380000000000000010000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.622+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=34'17 crt=34'17 lcod 34'17 active mbc={} ps=[a~1]] append_log approx pg log length = 19 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list oid 3#1:d0000000::::head# end GHMAX 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _collection_list key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:19.622+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=34'17 crt=34'17 lcod 34'17 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove_collection oid 3#1:d0000000::::head# 2019-12-07T19:34:19.622+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _osr_register_zombie 0x558c576a28c0 1.bs3_head 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove_collection 1.bs3_head = 0 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d20600 cost 4267 (1 ios * 4000 + 267 bytes) 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d20600 onodes shared_blobs 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d20600 allocated 0x[] released 0x[] 2019-12-07T19:34:19.622+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'18, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 prepare 2019-12-07T19:34:19.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d20600 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 io_done 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 802 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57ff0c00 seq 49 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2c280 had 1 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 825 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 825 < min_flush_size 524288 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:11# key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0x0000000000000011ffffffffffffffff'o' 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15f70f~339 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 42 try_finish_pg_delete 1.bs3 0x558c54d9b000 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15f70f 0x15f70f~339 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f07187700 10 osd.7:3._detach_pg 1.bs3 0x558c54d9b000 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12f70f 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x70f 2019-12-07T19:34:19.622+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa48 and padding block with 0x5b8 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:11# 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1210 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.onode(0x558c57a35400).flush flush done 2019-12-07T19:34:19.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# 0x0 2019-12-07T19:34:19.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:11# 0x0~4d000 -> 0x0~4d000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x0~2000: 0x0~2000 Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10410 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f67880 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f67880) get_ref 0x0~2000 Blob(0x558c57f67880 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f67880) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57f67880 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10411 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5809ae70 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c5809ae70) get_ref 0x0~2000 Blob(0x558c5809ae70 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c5809ae70) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c5809ae70 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4000~2000: 0x0~2000 Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10412 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57314000 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57314000) get_ref 0x0~2000 Blob(0x558c57314000 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57314000) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57314000 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10413 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e42540 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15fa48 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e42540 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f678f0 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f678f0) get_ref 0x0~2000 Blob(0x558c57f678f0 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f678f0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57f678f0 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000140557s (0.000000120s flush + 0.000140437s kv commit) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x8000~2000: 0x0~2000 Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d20600> 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10414 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 kv_submitted 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d20600 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578813b0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578813b0 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d20600 finishing 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d20600 onodes 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _queue_reap_collection 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f67960 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d20600 done 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576a28c0 q now empty 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f67960) get_ref 0x0~2000 Blob(0x558c57f67960 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d20600 [] 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f67960) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57f67960 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10415 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish reaping empty zombie osr 0x558c576a28c0 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _reap_collections 0x558c54d0f2c0 1.bs3_head 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57881490 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore.OnodeSpace(0x558c54d0f400 in 0x558c54842800) map_any 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore.OnodeSpace(0x558c54d0f400 in 0x558c54842800) clear 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57881490 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f67a40 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _reap_collections 0x558c54d0f2c0 1.bs3_head done 2019-12-07T19:34:19.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _reap_collections all reaped 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f67a40) get_ref 0x0~2000 Blob(0x558c57f67a40 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57f67a40) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57f67a40 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10416 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57880bd0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57880bd0 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f07187700 10 bluestore.OnodeSpace(0x558c54d0f400 in 0x558c54842800) clear 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2e70 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2e70) get_ref 0x0~2000 Blob(0x558c579b2e70 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2e70) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c579b2e70 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0xe000~2000: 0x0~2000 Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10417 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578de310 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578de310 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2ee0 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2ee0) get_ref 0x0~2000 Blob(0x558c579b2ee0 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c579b2ee0 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x10000~2000: 0x0~2000 Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10418 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578de770 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578de770 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579700e0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579700e0) get_ref 0x0~2000 Blob(0x558c579700e0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579700e0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c579700e0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x12000~2000: 0x0~2000 Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10419 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578de5b0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578de5b0 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2150 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2150) get_ref 0x0~2000 Blob(0x558c579b2150 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2150) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c579b2150 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x14000~2000: 0x0~2000 Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10420 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578debd0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578debd0 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2d90 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2d90) get_ref 0x0~2000 Blob(0x558c579b2d90 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2d90) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c579b2d90 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x16000~2000: 0x0~2000 Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10421 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578dea10 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578dea10 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2e00 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2e00) get_ref 0x0~2000 Blob(0x558c579b2e00 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2e00) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c579b2e00 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x18000~2000: 0x0~2000 Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10422 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578decb0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578decb0 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57970070 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57970070) get_ref 0x0~2000 Blob(0x558c57970070 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57970070) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57970070 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10423 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578def50 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578def50 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c576adc70 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c576adc70) get_ref 0x0~2000 Blob(0x558c576adc70 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c576adc70) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c576adc70 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10424 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578df030 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578df030 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2cb0 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2cb0) get_ref 0x0~2000 Blob(0x558c579b2cb0 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c579b2cb0 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10425 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578df110 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578df110 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2d20 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2d20) get_ref 0x0~2000 Blob(0x558c579b2d20 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2d20) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c579b2d20 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x20000~2000: 0x0~2000 Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10426 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578df1f0 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578df1f0 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57970000 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57970000) get_ref 0x0~2000 Blob(0x558c57970000 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57970000) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c57970000 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x22000~2000: 0x0~2000 Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10427 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c070 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 120 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55055100 con 0x558c54c19800 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c576adc00 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c576adc00) get_ref 0x0~2000 Blob(0x558c576adc00 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c576adc00) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c576adc00 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x24000~2000: 0x0~2000 Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10428 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c150 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.ds2 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c150 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2bd0 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2bd0) get_ref 0x0~2000 Blob(0x558c579b2bd0 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c579b2bd0 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x26000~2000: 0x0~2000 Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10429 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c8c0 blob([0x489000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c8c0 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2c40 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2c40) get_ref 0x0~2000 Blob(0x558c579b2c40 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2c40) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c579b2c40 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x28000~2000: 0x0~2000 Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10430 2019-12-07T19:34:19.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) pg 0x558c55141000 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3f80 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f80) get_ref 0x0~2000 Blob(0x558c579b3f80 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f80) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c579b3f80 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/18 les/c/f=42/19/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10431 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578f1880 blob([0x48d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578f1880 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57818bd0 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57818bd0) get_ref 0x0~2000 Blob(0x558c57818bd0 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c57818bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f06185700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 pi=[18,41)/1 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c57818bd0 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10432 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c574e0d90 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c574e0d90 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2af0 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2af0) get_ref 0x0~2000 Blob(0x558c579b2af0 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2af0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5740a300 seq 31 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c579b2af0 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10433 2019-12-07T19:34:19.622+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:19.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c574e0930 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 sbid 0x0)) 2019-12-07T19:34:19.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c574e0930 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=1)))) 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2b60 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2b60) get_ref 0x0~2000 Blob(0x558c579b2b60 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:19.622+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:19.622+0000 7f8f04181700 20 bluestore.blob(0x558c579b2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a300 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a300 onodes shared_blobs 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c579b2b60 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a300 allocated 0x[] released 0x[] 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x30000~2000: 0x0~2000 Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10434 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578df260 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 prepare 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578df260 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a300 2019-12-07T19:34:19.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 io_done 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3f10 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f10) get_ref 0x0~2000 Blob(0x558c579b3f10 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f10) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c579b3f10 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x32000~2000: 0x0~2000 Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10435 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578ded90 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578ded90 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b21c0 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b21c0) get_ref 0x0~2000 Blob(0x558c579b21c0 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b21c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c579b21c0 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x34000~2000: 0x0~2000 Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10436 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578dee70 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578dee70 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2a10 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15fa48~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a10) get_ref 0x0~2000 Blob(0x558c579b2a10 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a10) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15fa48 0x15fa48~45d to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c579b2a10 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x12fa48 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x36000~2000: 0x0~2000 Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa48 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10437 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xea5 and padding block with 0x15b 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c700 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c700 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2a80 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a80) get_ref 0x0~2000 Blob(0x558c579b2a80 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a80) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c579b2a80 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x38000~2000: 0x0~2000 Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10438 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c620 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c620 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3ea0 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3ea0) get_ref 0x0~2000 Blob(0x558c579b3ea0 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c579b3ea0 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10439 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c540 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c540 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c576add50 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c576add50) get_ref 0x0~2000 Blob(0x558c576add50 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c576add50) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c576add50 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10440 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2930 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b2930) get_ref 0x0~2000 Blob(0x558c579b2930 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b2930) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c579b2930 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10441 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b29a0 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b29a0) get_ref 0x0~2000 Blob(0x558c579b29a0 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b29a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x15fea5 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c579b29a0 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x40000~2000: 0x0~2000 Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10442 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3e30 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000137185s (0.000000131s flush + 0.000137054s kv commit) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3e30) get_ref 0x0~2000 Blob(0x558c579b3e30 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3e30) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c579b3e30 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a300> 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 kv_submitted 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x42000~2000: 0x0~2000 Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a300 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10443 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a300 finishing 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a300 onodes 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a300 done 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:19.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a300 [] 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c576adce0 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c576adce0) get_ref 0x0~2000 Blob(0x558c576adce0 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c576adce0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c576adce0 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x44000~2000: 0x0~2000 Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10444 2019-12-07T19:34:19.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c574fdc00 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c574fdc00) get_ref 0x0~2000 Blob(0x558c574fdc00 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c574fdc00) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c574fdc00 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x46000~2000: 0x0~2000 Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10445 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c574fdb90 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c574fdb90) get_ref 0x0~2000 Blob(0x558c574fdb90 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c574fdb90) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c574fdb90 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x48000~2000: 0x0~2000 Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10446 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3dc0 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3dc0) get_ref 0x0~2000 Blob(0x558c579b3dc0 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c579b3dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c579b3dc0 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10447 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57970f50 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c57970f50) get_ref 0x0~2000 Blob(0x558c57970f50 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c57970f50) get_ref init 0x2000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c57970f50 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4c000~1000: 0x0~1000 Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10448 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 sbid 0x0)) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=1)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c578f0af0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c578f0af0) get_ref 0x0~1000 Blob(0x558c578f0af0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.blob(0x558c578f0af0) get_ref init 0x1000, 1000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4c000~1000: 0x0~1000 Blob(0x558c578f0af0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dirty_range mark inline shard dirty 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57e8f288) dirty_range mark inline shard dirty 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:11# = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# snapset 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# snapset = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# 1 keys 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# 1 keys = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# hinfo_key (42 bytes) 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 22 keys 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 22 keys = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- SNA_1_000000000000000C_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- SNA_1_000000000000000E_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- SNA_1_0000000000000011_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000042.00000000000000000018' <- 0000000042.00000000000000000018 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000042.00000000000000000019' <- 0000000042.00000000000000000019 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0c00 cost 78386 (1 ios * 4000 + 74386 bytes) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0c00 onodes 0x558c57a35400,0x558c57e8f180 shared_blobs 0x558c54e1c000,0x558c54e1c0e0,0x558c54e1c4d0,0x558c54e1c5b0,0x558c54e1c690,0x558c54e1c770,0x558c54e1c850,0x558c54e1c930,0x558c54e1ca10,0x558c54e1caf0,0x558c54e1cbd0,0x558c54e1ccb0,0x558c54e1cd90,0x558c54e1ce70,0x558c54e1cf50,0x558c54e1d030,0x558c54e1d110,0x558c574e0e00,0x558c574e1ce0,0x558c57880f50,0x558c57881180,0x558c57881340,0x558c57881420,0x558c578de3f0,0x558c578de4d0,0x558c578de690,0x558c578de930,0x558c578deaf0,0x558c578ded20,0x558c578dee00,0x558c578deee0,0x558c578defc0,0x558c578df0a0,0x558c578df180,0x558c578f1730,0x558c578f17a0,0x558c57bea070,0x558c57dcac40,0x558c57dcb6c0 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update inline shard 1053 bytes from 39 extents 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 124517 (123458 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:19.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 291 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1200+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.623+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.7s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.623+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57e8f288) update 4#1:9b8adabe:::smithi13140796-6:11# 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore.extentmap(0x558c57e8f288) update inline shard 1053 bytes from 39 extents 2019-12-07T19:34:19.623+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:11# is 55375 (54316 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 pi=[28,41)/1 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.623+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:19.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b94300 seq 52 2019-12-07T19:34:19.623+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:19.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:19.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94300 cost 5375 (1 ios * 4000 + 1375 bytes) 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94300 onodes shared_blobs 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94300 allocated 0x[] released 0x[] 2019-12-07T19:34:19.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 prepare 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94300 2019-12-07T19:34:19.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 io_done 2019-12-07T19:34:19.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 347 < min_flush_size 524288 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1223 < min_flush_size 524288 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1246 < min_flush_size 524288 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1246 < min_flush_size 524288 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x15fea5~4de to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x15fea5 0x15fea5~4de to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:19.610246+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _allocate len 0x383 from 0 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _allocate len 0x383 from 1 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd290000 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd290000~10000/10000,10000,acd290000 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x170000 to include allocated 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 44 (was clean) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:19.624914+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x12fea5 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xea5 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x383 and padding block with 0xc7d 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x170000 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ba is 11 SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bb is 11 SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c7 is 11 SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c6 is 11 SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c5 is 11 SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c4 is 11 SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bd is 11 SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bc is 11 SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c8 is 11 SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c9 is 11 SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ca is 11 SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cb is 11 SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cc is 11 SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cd is 11 SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ce is 11 SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cf is 11 SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d0 is 11 SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c1 is 11 SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c0 is 11 SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b0 is 11 SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ad is 11 SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28af is 11 SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ae is 11 SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b1 is 11 SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b3 is 11 SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b2 is 11 SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b5 is 11 SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b4 is 11 SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c2 is 11 SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c3 is 11 SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b6 is 11 SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b7 is 11 SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b8 is 11 SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b9 is 11 SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bf is 11 SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28be is 11 SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ac is 11 SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ab is 11 SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28aa is 11 SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2))) 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 prepare 2019-12-07T19:34:19.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0c00 2019-12-07T19:34:19.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 io_done 2019-12-07T19:34:19.623+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=34'17 crt=34'17 lcod 34'17 active mbc={} ps=[a~1]] op_applied version 42'19 2019-12-07T19:34:19.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.623+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c581cd340 finish 2019-12-07T19:34:19.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.623+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x160383 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (44) on file(ino 16 size 0x170000 mtime 2019-12-07T19:34:19.624914+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]), flushing log 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x170000 mtime 2019-12-07T19:34:19.624914+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 44 len 0x49 crc 0x1277410d) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x2b000~1000 to file(ino 1 size 0x2b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x2b000 0x2b000~1000 to file(ino 1 size 0x2b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x2c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2b000 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x2c000 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000026 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 44 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x170000 mtime 2019-12-07T19:34:19.624914+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000443624s (0.000000130s flush + 0.000443494s kv commit) 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 31869 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 64637 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 97405 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 130173 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 162941 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 184661 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f180 had 1 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 184684 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 184684 < min_flush_size 524288 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:19.624914+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x160383~2d16c to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:19.624914+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x160383 0x160383~2d16c to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:19.624914+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _allocate len 0x1d4ef from 0 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _allocate len 0x1d4ef from 1 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x20000/10000,20000,acd2a0000 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd2a0000~20000/10000,20000,acd2a0000 2019-12-07T19:34:19.624+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x190000 to include allocated 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 45 (was clean) 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~160000 x_off 0x130383 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x383 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4ef and padding block with 0xb11 2019-12-07T19:34:19.624+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x190000 2019-12-07T19:34:19.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 121 ==== MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=54, reqid=client.4540.0:335, at_version=42'8, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2976+0+0 (crc 0 0 0) 0x558c574a4000 con 0x558c54c19800 2019-12-07T19:34:19.624+0000 7f8f24c58700 15 osd.7 42 enqueue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000027 epoch 42 MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=54, reqid=client.4540.0:335, at_version=42'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.624+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=54, reqid=client.4540.0:335, at_version=42'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94300> 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 kv_submitted 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94300 2019-12-07T19:34:19.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94300 finishing 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94300 onodes 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=54, reqid=client.4540.0:335, at_version=42'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94300 done 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:19.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94300 [] 2019-12-07T19:34:19.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=54, reqid=client.4540.0:335, at_version=42'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) pg 0x558c55141000 2019-12-07T19:34:19.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.624+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 osd.7 42 dequeue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000077 MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=54, reqid=client.4540.0:335, at_version=42'8, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] 2019-12-07T19:34:19.624+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=54, reqid=client.4540.0:335, at_version=42'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.add_oid 1:bf81b261:::smithi13140796-37:13 c,e,11,13 2019-12-07T19:34:19.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 174 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c55271e00 con 0x558c54c5bc00 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.get_snaps 1:bf81b261:::smithi13140796-37:13 got.empty() 2019-12-07T19:34:19.624+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c57334b00 prio 196 cost 0 latency 0.000017 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.624+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.set_snaps 1:bf81b261:::smithi13140796-37:13 c,e,11,13 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_0000000000000011_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 292 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) e42/42) v1 ==== 1222+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 21'6 (0'0,21'6] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] append_log log((0'0,21'6], crt=21'6) [42'7 (21'4) clone 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:33:58.722827+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,42'8 (21'4) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:335 2019-12-07T19:34:19.608834+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 0, new_object: 0] 2019-12-07T19:34:19.624+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c57334b00 prio 196 cost 0 latency 0.000049 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'7 (0'0,42'7] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[a~1]] add_log_entry 42'7 (21'4) clone 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:33:58.722827+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.624+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.624+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.as5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.624+0000 7f8f06986700 10 osd.7 42 dequeue_op 0x558c57334b00 finish 2019-12-07T19:34:19.624+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[a~1]] add_log_entry 42'8 (21'4) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:335 2019-12-07T19:34:19.608834+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 0, new_object: 0 2019-12-07T19:34:19.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[a~1]] append_log approx pg log length = 8 2019-12-07T19:34:19.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 293 ==== MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5763c600 con 0x558c54c64000 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.624+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd4a0 prio 196 cost 0 latency 0.000011 epoch 42 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) 2019-12-07T19:34:19.624+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) queued 2019-12-07T19:34:19.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2) prio 196 cost 0 e42) pg 0x558c548ab000 2019-12-07T19:34:19.624+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.624+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.624+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) pg 0x558c57e8b000 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:19.624+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581cd4a0 prio 196 cost 0 latency 0.000041 MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 pg pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c577fc900 seq 32 2019-12-07T19:34:19.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 42/41 ECSubWriteReply(tid=27, last_complete=42'6, committed=1, applied=1)) v2 2019-12-07T19:34:19.624+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=39/28 les/c/f=40/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bf81b261:::smithi13140796-37:13# key 0x828000000000000001bf81b2'a!smithi13140796-37!='0x0000000000000013ffffffffffffffff'o' 2019-12-07T19:34:19.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=27 reqid=osd.7.0:26 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:19.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:19.624+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:19.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:19.624+0000 7f8f0297e700 10 osd.7 42 dequeue_op 0x558c581cd4a0 finish 2019-12-07T19:34:19.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:19.624+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# -> 2#1:bf81b261:::smithi13140796-37:13# 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1211 2019-12-07T19:34:19.624+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.onode(0x558c54e44f00).flush flush done 2019-12-07T19:34:19.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 0x0 2019-12-07T19:34:19.624+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54ea1200 seq 7 2019-12-07T19:34:19.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# -> 2#1:bf81b261:::smithi13140796-37:13# 0x0~25000 -> 0x0~25000 2019-12-07T19:34:19.624+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:19.624+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x0~2000: 0x0~2000 Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10449 2019-12-07T19:34:19.624+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c546772d0 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:19.624+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1200 onodes shared_blobs 2019-12-07T19:34:19.624+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1200 allocated 0x[] released 0x[] 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c546772d0 blob([0x181000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c574d2f50 blob([0x181000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 prepare 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c574d2f50) get_ref 0x0~2000 Blob(0x558c574d2f50 blob([0x181000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1200 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c574d2f50) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 io_done 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574d2f50 blob([0x181000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10450 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57538690 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57538690 blob([0x183000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c57f90fc0 blob([0x183000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f90fc0) get_ref 0x0~2000 Blob(0x558c57f90fc0 blob([0x183000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f90fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57f90fc0 blob([0x183000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x4000~2000: 0x0~2000 Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10451 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575387e0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575387e0 blob([0x185000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c57f91180 blob([0x185000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91180) get_ref 0x0~2000 Blob(0x558c57f91180 blob([0x185000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f91180) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57f91180 blob([0x185000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x6000~2000: 0x0~2000 Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10452 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575388c0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575388c0 blob([0x187000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c5750ff80 blob([0x187000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ff80) get_ref 0x0~2000 Blob(0x558c5750ff80 blob([0x187000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c5750ff80) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5750ff80 blob([0x187000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x8000~2000: 0x0~2000 Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10453 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575389a0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575389a0 blob([0x189000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f05b0 blob([0x189000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f05b0) get_ref 0x0~2000 Blob(0x558c578f05b0 blob([0x189000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f05b0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c578f05b0 blob([0x189000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10454 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57538a80 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57538a80 blob([0x18b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c57f51110 blob([0x18b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f51110) get_ref 0x0~2000 Blob(0x558c57f51110 blob([0x18b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f51110) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57f51110 blob([0x18b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10455 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583260 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583260 blob([0x18d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f0460 blob([0x18d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0460) get_ref 0x0~2000 Blob(0x558c578f0460 blob([0x18d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0460) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c578f0460 blob([0x18d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0xe000~2000: 0x0~2000 Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10456 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575835e0 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575835e0 blob([0x18f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f04d0 blob([0x18f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f04d0) get_ref 0x0~2000 Blob(0x558c578f04d0 blob([0x18f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f04d0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c578f04d0 blob([0x18f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x10000~2000: 0x0~2000 Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10457 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575836c0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575836c0 blob([0x191000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f12d0 blob([0x191000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f12d0) get_ref 0x0~2000 Blob(0x558c578f12d0 blob([0x191000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f12d0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c578f12d0 blob([0x191000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x12000~2000: 0x0~2000 Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10458 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575837a0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575837a0 blob([0x193000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c57f51180 blob([0x193000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f51180) get_ref 0x0~2000 Blob(0x558c57f51180 blob([0x193000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c57f51180) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57f51180 blob([0x193000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10459 2019-12-07T19:34:19.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583880 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 sbid 0x0)) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583880 blob([0x195000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=1)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f0380 blob([0x195000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0380) get_ref 0x0~2000 Blob(0x558c578f0380 blob([0x195000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=2)))) 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0380) get_ref init 0x2000, 1000 2019-12-07T19:34:19.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c578f0380 blob([0x195000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10460 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583960 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583960 blob([0x197000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f03f0 blob([0x197000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f03f0) get_ref 0x0~2000 Blob(0x558c578f03f0 blob([0x197000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f03f0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c578f03f0 blob([0x197000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x18000~2000: 0x0~2000 Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10461 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c54676f50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c54676f50 blob([0x199000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f1260 blob([0x199000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f1260) get_ref 0x0~2000 Blob(0x558c578f1260 blob([0x199000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f1260) get_ref init 0x2000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c578f1260 blob([0x199000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10462 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583a40 blob([0x19b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583a40 blob([0x19b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c57f511f0 blob([0x19b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c57f511f0) get_ref 0x0~2000 Blob(0x558c57f511f0 blob([0x19b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c57f511f0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57f511f0 blob([0x19b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10463 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583b20 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583b20 blob([0x19d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f02a0 blob([0x19d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f02a0) get_ref 0x0~2000 Blob(0x558c578f02a0 blob([0x19d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f02a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c578f02a0 blob([0x19d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10464 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583c00 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583c00 blob([0x19f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f0310 blob([0x19f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0310) get_ref 0x0~2000 Blob(0x558c578f0310 blob([0x19f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0310) get_ref init 0x2000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c578f0310 blob([0x19f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x20000~2000: 0x0~2000 Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10465 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583ce0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583ce0 blob([0x1a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f11f0 blob([0x1a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f11f0) get_ref 0x0~2000 Blob(0x558c578f11f0 blob([0x1a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f11f0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c578f11f0 blob([0x1a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10466 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583dc0 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583dc0 blob([0x1a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c57f51260 blob([0x1a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c57f51260) get_ref 0x0~2000 Blob(0x558c57f51260 blob([0x1a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c57f51260) get_ref init 0x2000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57f51260 blob([0x1a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup src 0x24000~1000: 0x0~1000 Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10467 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583ea0 blob([0x1a5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 sbid 0x0)) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583ea0 blob([0x1a5000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=1)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup new Blob(0x558c578f01c0 blob([0x1a5000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f01c0) get_ref 0x0~1000 Blob(0x558c578f01c0 blob([0x1a5000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.blob(0x558c578f01c0) get_ref init 0x1000, 1000 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dup dst 0x24000~1000: 0x0~1000 Blob(0x558c578f01c0 blob([0x1a5000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=2)))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dirty_range mark inline shard dirty 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57976108) dirty_range mark inline shard dirty 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# -> 2#1:bf81b261:::smithi13140796-37:13# = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# snapset 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# snapset = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bf81b261:::smithi13140796-37:head#8 key 0x828000000000000001bf81b2'a!smithi13140796-37!='0xfffffffffffffffe0000000000000008'o' 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# -> 2#1:bf81b261:::smithi13140796-37:head#8 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# -> 2#1:bf81b261:::smithi13140796-37:head#8 = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1212 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_000000000000000C_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_000000000000000E_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_0000000000000011_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_0000000000000013_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000042.00000000000000000007' <- 0000000042.00000000000000000007 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000042.00000000000000000008' <- 0000000042.00000000000000000008 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc900 cost 8485 (1 ios * 4000 + 4485 bytes) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc900 onodes 0x558c54d96c80,0x558c54e44f00,0x558c57976000 shared_blobs 0x558c54d3c540,0x558c55278a80,0x558c57538700,0x558c57538850,0x558c57538930,0x558c57538a10,0x558c575831f0,0x558c575832d0,0x558c57583650,0x558c57583730,0x558c57583810,0x558c575838f0,0x558c575839d0,0x558c57583ab0,0x558c57583b90,0x558c57583c70,0x558c57583d50,0x558c57583e30,0x558c57583f10 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96d88) update 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96d88) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head# is 499 (491 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) update 2#1:bf81b261:::smithi13140796-37:head#8 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) update inline shard 513 bytes from 19 extents 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head#8 is 1004 (485 bytes onode + 2 bytes spanning blobs + 517 bytes inline extents) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57976108) update 2#1:bf81b261:::smithi13140796-37:13# 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57976108) update inline shard 513 bytes from 19 extents 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:13# is 931 (412 bytes onode + 2 bytes spanning blobs + 517 bytes inline extents) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28dc is 11 SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d1 is 11 SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d2 is 11 SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d3 is 11 SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d4 is 11 SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d5 is 11 SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d7 is 11 SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d6 is 11 SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d8 is 11 SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d9 is 11 SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28da is 11 SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28db is 11 SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28dd is 11 SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28de is 11 SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28df is 11 SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e0 is 11 SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e1 is 11 SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e2 is 11 SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e3 is 11 SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=2))) 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 prepare 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc900 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 io_done 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[a~1]] op_applied version 42'8 2019-12-07T19:34:19.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.625+0000 7f8f0217d700 10 osd.7 42 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x18d4ef 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (45) on file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]), flushing log 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 45 len 0x49 crc 0xacf943ae) 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x2c000~1000 to file(ino 1 size 0x2c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x2c000 0x2c000~1000 to file(ino 1 size 0x2c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x2d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2c000 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x2d000 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000020 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 45 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001519837s (0.000000100s flush + 0.001519737s kv commit) 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0c00> 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 kv_submitted 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0c00 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0c00 finishing 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0c00 onodes 0x558c57a35400,0x558c57e8f180 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0c00 done 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0c00 [] 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6685 < min_flush_size 524288 2019-12-07T19:34:19.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.625+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=200, last_complete=42'19, committed=1, applied=1)) v2 -- 0x558c5795c400 con 0x558c54c64000 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44f00 had 1 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6708 < min_flush_size 524288 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6708 < min_flush_size 524288 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x18d4ef~1a34 to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x18d4ef 0x18d4ef~1a34 to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~160000 x_off 0x15d4ef 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4ef 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf23 and padding block with 0xdd 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x18ef23 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000109497s (0.000000104s flush + 0.000109393s kv commit) 2019-12-07T19:34:19.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1200,0x558c577fc900> 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 kv_submitted 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1200 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1200 finishing 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1200 onodes 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1200 done 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1200 [] 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 kv_submitted 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc900 2019-12-07T19:34:19.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc900 finishing 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc900 onodes 0x558c54d96c80,0x558c54e44f00,0x558c57976000 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc900 done 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc900 [] 2019-12-07T19:34:19.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.625+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 42/41 ECSubWriteReply(tid=54, last_complete=42'8, committed=1, applied=1)) v2 -- 0x558c5841e200 con 0x558c54c19800 2019-12-07T19:34:19.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 294 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1222+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.626+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.as5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.626+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) pg 0x558c57e8b000 2019-12-07T19:34:19.626+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/28 les/c/f=42/29/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] update_history advanced history from ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 pi=[28,41)/1 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] update_history clearing past_intervals 2019-12-07T19:34:19.626+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.626+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:19.626+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54d59800 seq 8 2019-12-07T19:34:19.626+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:19.626+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:19.626+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:19.626+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:19.626+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59800 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:19.626+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59800 onodes shared_blobs 2019-12-07T19:34:19.626+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59800 allocated 0x[] released 0x[] 2019-12-07T19:34:19.626+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 prepare 2019-12-07T19:34:19.626+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59800 2019-12-07T19:34:19.626+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 io_done 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x18ef23~45d to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x18ef23 0x18ef23~45d to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~160000 x_off 0x15ef23 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf23 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x380 and padding block with 0xc80 2019-12-07T19:34:19.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 199 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c55054a00 con 0x558c54c5b400 2019-12-07T19:34:19.626+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.626+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 200 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c55054a00 con 0x558c54c5b400 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x18f380 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.626+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.626+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000105161s (0.000000100s flush + 0.000105061s kv commit) 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) pg 0x558c5513e000 2019-12-07T19:34:19.626+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.626+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 201 ==== MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=72, reqid=osd.4.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=42'15)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c5748e000 con 0x558c54c5b400 2019-12-07T19:34:19.626+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd600 prio 127 cost 0 latency 0.000015 epoch 42 MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=72, reqid=osd.4.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=42'15)) v2 2019-12-07T19:34:19.626+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=72, reqid=osd.4.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=42'15)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.626+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5740a900 seq 49 2019-12-07T19:34:19.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a900 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a900 onodes shared_blobs 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 prepare 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a900 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 io_done 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=72, reqid=osd.4.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=42'15)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e42) pg 0x558c5513e000 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x18f380~46d to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x18f380 0x18f380~46d to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~160000 x_off 0x15f380 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x380 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7ed and padding block with 0x813 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59800> 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 kv_submitted 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59800 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c57ef8300 seq 50 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59800 finishing 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59800 onodes 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59800 done 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59800 [] 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8300 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x18f7ed 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8300 onodes shared_blobs 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8300 allocated 0x[] released 0x[] 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 prepare 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8300 2019-12-07T19:34:19.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 io_done 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000078433s (0.000000086s flush + 0.000078347s kv commit) 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a900> 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 kv_submitted 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a900 2019-12-07T19:34:19.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a900 finishing 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a900 onodes 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a900 done 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8300 kv_queued 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a900 [] 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x18f7ed~47d to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x18f7ed 0x18f7ed~47d to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~160000 x_off 0x15f7ed 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7ed 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc6a and padding block with 0x396 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=72, reqid=osd.4.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=42'15)) v2) prio 127 cost 0 e42) pg 0x558c5513e000 2019-12-07T19:34:19.626+0000 7f8f07187700 10 osd.7 42 dequeue_op 0x558c581cd600 prio 127 cost 0 latency 0.000210 MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=72, reqid=osd.4.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=42'15)) v2 pg pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.626+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.3s4 42/41 ECSubWrite(tid=72, reqid=osd.4.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=42'15)) v2 2019-12-07T19:34:19.626+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=37'13 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'15], crt=37'13) [] 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x18fc6a 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'14 (37'12) modify 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:19.620903+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000100408s (0.000000095s flush + 0.000100313s kv commit) 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8300> 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 kv_submitted 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8300 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8300 finishing 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8300 onodes 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8300 done 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:19.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8300 [] 2019-12-07T19:34:19.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.626+0000 7f8f07187700 20 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'15 (37'13) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:19.620903+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.626+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 15 2019-12-07T19:34:19.626+0000 7f8f07187700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.626+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.626+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c54d21500 seq 51 2019-12-07T19:34:19.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d21500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d21500 onodes shared_blobs 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d21500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 prepare 2019-12-07T19:34:19.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d21500 2019-12-07T19:34:19.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 io_done 2019-12-07T19:34:19.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.627+0000 7f8f07187700 10 osd.7 42 dequeue_op 0x558c581cd600 finish 2019-12-07T19:34:19.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 918 < min_flush_size 524288 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x18fc6a~47c to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x18fc6a 0x18fc6a~47c to file(ino 16 size 0x190000 mtime 2019-12-07T19:34:19.625606+0000 allocated 190000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~160000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _allocate len 0xe6 from 0 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _allocate len 0xe6 from 1 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd2c0000 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd2c0000~10000/10000,10000,acd2c0000 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x1a0000 to include allocated 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 46 (was clean) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x15fc6a 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc6a 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe6 and padding block with 0xf1a 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x1a0000 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1900e6 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (46) on file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]), flushing log 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 46 len 0x49 crc 0x4f558a58) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x2d000~1000 to file(ino 1 size 0x2d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x2d000 0x2d000~1000 to file(ino 1 size 0x2d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x2e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2d000 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x2e000 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000017 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.627+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 122 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c55055100 con 0x558c54c19800 2019-12-07T19:34:19.627+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.ds2 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.627+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) pg 0x558c55141000 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c577fc600 seq 33 2019-12-07T19:34:19.627+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc600 onodes shared_blobs 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc600 allocated 0x[] released 0x[] 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 prepare 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc600 2019-12-07T19:34:19.627+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 io_done 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 46 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000471317s (0.000000051s flush + 0.000471266s kv commit) 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d21500> 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 kv_submitted 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d21500 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d21500 finishing 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d21500 onodes 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d21500 done 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d21500 [] 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:19.627+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1900e6~46d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1900e6 0x1900e6~46d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x1600e6 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe6 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x553 and padding block with 0xaad 2019-12-07T19:34:19.627+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 42/41 ECSubWriteReply(tid=72, last_complete=42'15, committed=1, applied=1)) v2 -- 0x558c551afc00 con 0x558c54c5b400 2019-12-07T19:34:19.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x190553 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000075366s (0.000000108s flush + 0.000075258s kv commit) 2019-12-07T19:34:19.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fc600> 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 kv_submitted 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc600 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc600 finishing 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc600 onodes 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc600 done 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc600 [] 2019-12-07T19:34:19.627+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.627+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.627+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 123 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55055100 con 0x558c54c19800 2019-12-07T19:34:19.627+0000 7f8f24c58700 15 osd.7 42 enqueue_peering_evt 1.ds2 epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.627+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e42) pg 0x558c55141000 2019-12-07T19:34:19.628+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 2(0) info: 1.ds0( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:19.628+0000 7f8f06185700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.628+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:19.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:19.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57ff1b00 seq 34 2019-12-07T19:34:19.628+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:19.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:19.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:19.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:19.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1b00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1b00 onodes shared_blobs 2019-12-07T19:34:19.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 prepare 2019-12-07T19:34:19.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1b00 2019-12-07T19:34:19.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 io_done 2019-12-07T19:34:19.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x190553~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x190553 0x190553~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x160553 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x553 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9d0 and padding block with 0x630 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1909d0 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000083692s (0.000000117s flush + 0.000083575s kv commit) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 295 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.628+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.628+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e42) pg 0x558c57380000 2019-12-07T19:34:19.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 296 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:19.628+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.9s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.628+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff1b00> 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 kv_submitted 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1b00 2019-12-07T19:34:19.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1b00 finishing 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1b00 onodes 2019-12-07T19:34:19.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1b00 done 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:19.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 297 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1b00 [] 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) pg 0x558c5500a000 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57db2000 seq 23 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.628+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.628+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.628+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:19.628+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db2000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db2000 onodes shared_blobs 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db2000 allocated 0x[] released 0x[] 2019-12-07T19:34:19.628+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2000 prepare 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db2000 2019-12-07T19:34:19.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2000 io_done 2019-12-07T19:34:19.628+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:19.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740a600 seq 50 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.628+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:19.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:19.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a600 onodes shared_blobs 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a600 allocated 0x[] released 0x[] 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1909d0~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1909d0 0x1909d0~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 prepare 2019-12-07T19:34:19.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a600 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 io_done 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x1609d0 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9d0 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe4d and padding block with 0x1b3 2019-12-07T19:34:19.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e42) pg 0x558c54d98000 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x190e4d 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000082569s (0.000000054s flush + 0.000082515s kv commit) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5730ac00 seq 11 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57db2000> 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2000 kv_submitted 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db2000 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x190e4d~46d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2000 finishing 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x190e4d 0x190e4d~46d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db2000 onodes 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730ac00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db2000 done 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730ac00 onodes shared_blobs 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db2000 [] 2019-12-07T19:34:19.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x160e4d 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe4d 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2ba and padding block with 0xd46 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 prepare 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730ac00 2019-12-07T19:34:19.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 io_done 2019-12-07T19:34:19.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1912ba 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000137410s (0.000000064s flush + 0.000137346s kv commit) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a600> 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 kv_submitted 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a600 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a600 finishing 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a600 onodes 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a600 done 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a600 [] 2019-12-07T19:34:19.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:19.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1912ba~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1912ba 0x1912ba~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x1612ba 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2ba 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x737 and padding block with 0x8c9 2019-12-07T19:34:19.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x191737 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138563s (0.000000105s flush + 0.000138458s kv commit) 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730ac00> 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 kv_submitted 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730ac00 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730ac00 finishing 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730ac00 onodes 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730ac00 done 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730ac00 [] 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.629+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 298 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.629+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.as5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.629+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) pg 0x558c57e8b000 2019-12-07T19:34:19.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 299 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.629+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.629+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.7s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.629+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.629+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.629+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:19.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:19.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54ea0c00 seq 9 2019-12-07T19:34:19.629+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:19.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:19.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:19.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 300 ==== MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=201, reqid=client.4540.0:333, at_version=0'0, trim_to=0'0, roll_forward_to=42'19)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c579e1800 con 0x558c54c64000 2019-12-07T19:34:19.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:19.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:19.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0c00 onodes shared_blobs 2019-12-07T19:34:19.629+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd760 prio 127 cost 0 latency 0.000014 epoch 42 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=201, reqid=client.4540.0:333, at_version=0'0, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.629+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=201, reqid=client.4540.0:333, at_version=0'0, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 prepare 2019-12-07T19:34:19.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0c00 2019-12-07T19:34:19.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 io_done 2019-12-07T19:34:19.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=201, reqid=client.4540.0:333, at_version=0'0, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=201, reqid=client.4540.0:333, at_version=0'0, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) pg 0x558c5500a000 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:19.629+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c581cd760 prio 127 cost 0 latency 0.000044 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=201, reqid=client.4540.0:333, at_version=0'0, trim_to=0'0, roll_forward_to=42'19)) v2 pg pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x191737~46d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.629+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=201, reqid=client.4540.0:333, at_version=0'0, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x191737 0x191737~46d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x161737 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x737 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xba4 and padding block with 0x45c 2019-12-07T19:34:19.629+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=34'17 active mbc={} ps=[a~1,d~1]] append_log log((0'0,42'19], crt=34'17) [] 2019-12-07T19:34:19.629+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1]] rollforward: entry=42'18 (34'17) clone 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.629+0000 7f8f08189700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1]] rollforward: entry=42'19 (34'17) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:333 2019-12-07T19:34:18.620957+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x191ba4 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.629+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.629+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 19 2019-12-07T19:34:19.629+0000 7f8f05183700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1]] proc_primary_info updating purged_snaps to [a~1,d~1] 2019-12-07T19:34:19.629+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000092536s (0.000000099s flush + 0.000092437s kv commit) 2019-12-07T19:34:19.629+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.629+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:19.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:19.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c574fe900 seq 53 2019-12-07T19:34:19.629+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:19.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:19.629+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:19.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe900 cost 5391 (1 ios * 4000 + 1391 bytes) 2019-12-07T19:34:19.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe900 onodes shared_blobs 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0c00> 2019-12-07T19:34:19.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 kv_submitted 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0c00 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0c00 finishing 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0c00 onodes 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0c00 done 2019-12-07T19:34:19.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 prepare 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:19.629+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe900 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0c00 [] 2019-12-07T19:34:19.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.629+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 io_done 2019-12-07T19:34:19.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57b94000 seq 51 2019-12-07T19:34:19.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.629+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 301 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:19.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94000 onodes shared_blobs 2019-12-07T19:34:19.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94000 allocated 0x[] released 0x[] 2019-12-07T19:34:19.629+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.9s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.629+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 prepare 2019-12-07T19:34:19.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94000 2019-12-07T19:34:19.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 io_done 2019-12-07T19:34:19.629+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c581cd760 finish 2019-12-07T19:34:19.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 302 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e42) pg 0x558c5500a000 2019-12-07T19:34:19.629+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.as5 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.629+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.629+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.9s0( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:19.629+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.629+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:19.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d58f00 seq 52 2019-12-07T19:34:19.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:19.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58f00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58f00 onodes shared_blobs 2019-12-07T19:34:19.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58f00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.630+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 prepare 2019-12-07T19:34:19.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58f00 2019-12-07T19:34:19.630+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 io_done 2019-12-07T19:34:19.630+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e42) pg 0x558c57e8b000 2019-12-07T19:34:19.630+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:19.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.630+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:19.630+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:19.630+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c5730a900 seq 10 2019-12-07T19:34:19.630+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:19.630+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:19.630+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:19.630+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:19.630+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a900 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.630+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a900 onodes shared_blobs 2019-12-07T19:34:19.630+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.630+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 prepare 2019-12-07T19:34:19.630+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a900 2019-12-07T19:34:19.630+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 io_done 2019-12-07T19:34:19.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1228 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2342 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3468 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4594 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4617 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4617 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x191ba4~1209 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x191ba4 0x191ba4~1209 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x161ba4 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xba4 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdad and padding block with 0x253 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x192dad 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 124 ==== MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=55, reqid=client.4540.0:335, at_version=0'0, trim_to=0'0, roll_forward_to=42'8)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e2000 con 0x558c54c19800 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000176542s (0.000000130s flush + 0.000176412s kv commit) 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.630+0000 7f8f24c58700 15 osd.7 42 enqueue_op 0x558c57eff4a0 prio 127 cost 0 latency 0.000019 epoch 42 MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=55, reqid=client.4540.0:335, at_version=0'0, trim_to=0'0, roll_forward_to=42'8)) v2 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.630+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=55, reqid=client.4540.0:335, at_version=0'0, trim_to=0'0, roll_forward_to=42'8)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe900,0x558c57b94000,0x558c54d58f00,0x558c5730a900> 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 kv_submitted 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe900 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe900 finishing 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe900 onodes 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe900 done 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe900 [] 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 kv_submitted 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94000 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=55, reqid=client.4540.0:335, at_version=0'0, trim_to=0'0, roll_forward_to=42'8)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94000 finishing 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94000 onodes 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94000 done 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 kv_submitted 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=55, reqid=client.4540.0:335, at_version=0'0, trim_to=0'0, roll_forward_to=42'8)) v2) prio 127 cost 0 e42) pg 0x558c55141000 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94000 [] 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 kv_submitted 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58f00 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58f00 finishing 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58f00 onodes 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58f00 done 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58f00 [] 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 kv_submitted 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 osd.7 42 dequeue_op 0x558c57eff4a0 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=55, reqid=client.4540.0:335, at_version=0'0, trim_to=0'0, roll_forward_to=42'8)) v2 pg pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a900 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a900 finishing 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a900 onodes 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a900 done 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a900 [] 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.ds2 42/41 ECSubWrite(tid=55, reqid=client.4540.0:335, at_version=0'0, trim_to=0'0, roll_forward_to=42'8)) v2 2019-12-07T19:34:19.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=21'6 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'8], crt=21'6) [] 2019-12-07T19:34:19.630+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=201, last_complete=42'19, committed=1, applied=1)) v2 -- 0x558c5795c600 con 0x558c54c64000 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'7 (21'4) clone 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:33:58.722827+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'8 (21'4) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:335 2019-12-07T19:34:19.608834+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 0, new_object: 0 2019-12-07T19:34:19.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 8 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.630+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c577fd800 seq 35 2019-12-07T19:34:19.630+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head#8 onode 0x558c54e44f00 txc 0x558c577fd800 2019-12-07T19:34:19.630+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head#8 0x0 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c546772d0) put_ref 0x0~2000 Blob(0x558c546772d0 blob([0x181000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57538690) put_ref 0x0~2000 Blob(0x558c57538690 blob([0x183000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c575387e0) put_ref 0x0~2000 Blob(0x558c575387e0 blob([0x185000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c575388c0) put_ref 0x0~2000 Blob(0x558c575388c0 blob([0x187000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c575389a0) put_ref 0x0~2000 Blob(0x558c575389a0 blob([0x189000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57538a80) put_ref 0x0~2000 Blob(0x558c57538a80 blob([0x18b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583260) put_ref 0x0~2000 Blob(0x558c57583260 blob([0x18d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c575835e0) put_ref 0x0~2000 Blob(0x558c575835e0 blob([0x18f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c575836c0) put_ref 0x0~2000 Blob(0x558c575836c0 blob([0x191000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c575837a0) put_ref 0x0~2000 Blob(0x558c575837a0 blob([0x193000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583880) put_ref 0x0~2000 Blob(0x558c57583880 blob([0x195000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583960) put_ref 0x0~2000 Blob(0x558c57583960 blob([0x197000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c54676f50) put_ref 0x0~2000 Blob(0x558c54676f50 blob([0x199000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583a40) put_ref 0x0~2000 Blob(0x558c57583a40 blob([0x19b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583b20) put_ref 0x0~2000 Blob(0x558c57583b20 blob([0x19d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583c00) put_ref 0x0~2000 Blob(0x558c57583c00 blob([0x19f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ce0) put_ref 0x0~2000 Blob(0x558c57583ce0 blob([0x1a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583dc0) put_ref 0x0~2000 Blob(0x558c57583dc0 blob([0x1a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ea0) put_ref 0x0~1000 Blob(0x558c57583ea0 blob([0x1a5000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e45008) dirty_range mark inline shard dirty 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c546772d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x181000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57538690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x183000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c575387e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x185000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c575388c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x187000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c575389a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x189000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57538a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x18b000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57583260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x18d000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c575835e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x18f000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c575836c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x191000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c575837a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x193000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57583880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x195000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57583960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x197000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c54676f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x199000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57583a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x19b000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57583b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x19d000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c57583c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x19f000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57583ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1a1000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57583dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1a3000~2000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~1000: 0x0~1000 Blob(0x558c57583ea0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=2)))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1a5000~1000] 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3c540,0x558c55278a80,0x558c57538700,0x558c57538850,0x558c57538930,0x558c57538a10,0x558c575831f0,0x558c575832d0,0x558c57583650,0x558c57583730,0x558c57583810,0x558c575838f0,0x558c575839d0,0x558c57583ab0,0x558c57583b90,0x558c57583c70,0x558c57583d50,0x558c57583e30,0x558c57583f10 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c54d96c80 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head#8 = 0 2019-12-07T19:34:19.630+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fd800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fd800 onodes shared_blobs 0x558c54d3c540,0x558c55278a80,0x558c57538700,0x558c57538850,0x558c57538930,0x558c57538a10,0x558c575831f0,0x558c575832d0,0x558c57583650,0x558c57583730,0x558c57583810,0x558c575838f0,0x558c575839d0,0x558c57583ab0,0x558c57583b90,0x558c57583c70,0x558c57583d50,0x558c57583e30,0x558c57583f10 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28dc is 11 SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d1 is 11 SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d2 is 11 SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d3 is 11 SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d4 is 11 SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d5 is 11 SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d7 is 11 SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d6 is 11 SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d8 is 11 SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d9 is 11 SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28da is 11 SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28db is 11 SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28dd is 11 SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28de is 11 SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28df is 11 SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e0 is 11 SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e1 is 11 SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e2 is 11 SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e3 is 11 SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=1))) 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fd800 allocated 0x[] released 0x[] 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 prepare 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fd800 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 io_done 2019-12-07T19:34:19.630+0000 7f8f0217d700 10 osd.7 42 dequeue_op 0x558c57eff4a0 finish 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1677 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44f00 had 1 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1700 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1700 < min_flush_size 524288 2019-12-07T19:34:19.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x192dad~6a4 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x192dad 0x192dad~6a4 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x162dad 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdad 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x451 and padding block with 0xbaf 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x193451 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127354s (0.000000074s flush + 0.000127280s kv commit) 2019-12-07T19:34:19.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fd800> 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 kv_submitted 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fd800 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fd800 finishing 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fd800 onodes 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fd800 done 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fd800 [] 2019-12-07T19:34:19.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.630+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 42/41 ECSubWriteReply(tid=55, last_complete=42'8, committed=1, applied=1)) v2 -- 0x558c5841e400 con 0x558c54c19800 2019-12-07T19:34:19.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:19.631+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 303 ==== MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=202, reqid=osd.1.0:202, at_version=42'17, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 3426+0+0 (crc 0 0 0) 0x558c54e40000 con 0x558c54c64000 2019-12-07T19:34:19.631+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd8c0 prio 127 cost 0 latency 0.000035 epoch 42 MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=202, reqid=osd.1.0:202, at_version=42'17, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.631+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=202, reqid=osd.1.0:202, at_version=42'17, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=202, reqid=osd.1.0:202, at_version=42'17, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=202, reqid=osd.1.0:202, at_version=42'17, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 42 dequeue_op 0x558c581cd8c0 prio 127 cost 0 latency 0.000071 MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=202, reqid=osd.1.0:202, at_version=42'17, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=202, reqid=osd.1.0:202, at_version=42'17, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 snap_mapper.update_snaps 1:ec1285ab:::smithi13140796-19:10 c,e was 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ec1285ab:::smithi13140796-19:10 c,e,10 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 snap_mapper.set_snaps 1:ec1285ab:::smithi13140796-19:10 c,e 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 snap_mapper.update_snaps rm SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 40'15 (0'0,40'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1,d~1]] append_log log((0'0,40'15], crt=40'15) [42'16 (40'11) modify 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:19.631489+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,42'17 (40'15) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:19.631489+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'16 (0'0,42'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=40'15 crt=40'15 lcod 40'15 active mbc={} ps=[a~1,d~1]] add_log_entry 42'16 (40'11) modify 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:19.631489+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=40'15 crt=40'15 lcod 40'15 active mbc={} ps=[a~1,d~1]] add_log_entry 42'17 (40'15) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:19.631489+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=40'15 crt=40'15 lcod 40'15 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 17 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=40'15 crt=40'15 lcod 40'15 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.631+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b94900 seq 54 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 1 keys 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 1 keys = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key (42 bytes) 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- SNA_1_0000000000000010_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000042.00000000000000000016' <- 0000000042.00000000000000000016 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000042.00000000000000000017' <- 0000000042.00000000000000000017 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94900 cost 8389 (1 ios * 4000 + 4389 bytes) 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94900 onodes 0x558c54e44000,0x558c574a0500 shared_blobs 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1578 (565 bytes onode + 2 bytes spanning blobs + 1011 bytes inline extents) 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore.extentmap(0x558c574a0608) update 4#1:ec1285ab:::smithi13140796-19:10# 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:10# is 688 (412 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 prepare 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94900 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 io_done 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=40'15 crt=40'15 lcod 40'15 active mbc={} ps=[a~1,d~1]] op_applied version 42'17 2019-12-07T19:34:19.631+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.631+0000 7f8f0117b700 10 osd.7 42 dequeue_op 0x558c581cd8c0 finish 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5109 < min_flush_size 524288 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0500 had 1 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5132 < min_flush_size 524288 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5132 < min_flush_size 524288 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x193451~140c to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x193451 0x193451~140c to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x163451 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x451 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x85d and padding block with 0x7a3 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19485d 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000090319s (0.000000085s flush + 0.000090234s kv commit) 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94900> 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 kv_submitted 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94900 2019-12-07T19:34:19.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94900 finishing 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94900 onodes 0x558c54e44000,0x558c574a0500 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94900 done 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:19.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94900 [] 2019-12-07T19:34:19.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.631+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 42/41 ECSubWriteReply(tid=202, last_complete=42'17, committed=1, applied=1)) v2 -- 0x558c5763da00 con 0x558c54c64000 2019-12-07T19:34:19.631+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 304 ==== MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=203, reqid=osd.1.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=42'17)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e0000 con 0x558c54c64000 2019-12-07T19:34:19.633+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cda20 prio 127 cost 0 latency 0.000021 epoch 42 MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=203, reqid=osd.1.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=42'17)) v2 2019-12-07T19:34:19.633+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=203, reqid=osd.1.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=42'17)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=203, reqid=osd.1.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=42'17)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=203, reqid=osd.1.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=42'17)) v2) prio 127 cost 0 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 osd.7 42 dequeue_op 0x558c581cda20 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=203, reqid=osd.1.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=42'17)) v2 pg pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1,d~1]] 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1,d~1]] _handle_message: MOSDECSubOpWrite(1.7s4 42/41 ECSubWrite(tid=203, reqid=osd.1.0:202, at_version=0'0, trim_to=0'0, roll_forward_to=42'17)) v2 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=40'15 active mbc={} ps=[a~1,d~1]] append_log log((0'0,42'17], crt=40'15) [] 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1]] rollforward: entry=42'16 (40'11) modify 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:19.631489+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1]] rollforward: entry=42'17 (40'15) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:19.631489+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1]] append_log approx pg log length = 17 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.633+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b95b00 seq 55 2019-12-07T19:34:19.633+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95b00 onodes shared_blobs 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95b00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 prepare 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95b00 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 io_done 2019-12-07T19:34:19.633+0000 7f8f0117b700 10 osd.7 42 dequeue_op 0x558c581cda20 finish 2019-12-07T19:34:19.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19485d~471 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19485d 0x19485d~471 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16485d 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x85d 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcce and padding block with 0x332 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x194cce 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000074923s (0.000000098s flush + 0.000074825s kv commit) 2019-12-07T19:34:19.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95b00> 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 kv_submitted 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95b00 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95b00 finishing 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95b00 onodes 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95b00 done 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95b00 [] 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.634+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 42/41 ECSubWriteReply(tid=203, last_complete=42'17, committed=1, applied=1)) v2 -- 0x558c575b8c00 con 0x558c54c64000 2019-12-07T19:34:19.634+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 305 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e42/42) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:19.634+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.7s4 epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.634+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) queued 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e42) pg 0x558c54d5f000 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1]] do_peering_event: epoch_sent: 42 epoch_requested: 42 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~1] 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57b95800 seq 56 2019-12-07T19:34:19.634+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95800 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95800 onodes shared_blobs 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95800 allocated 0x[] released 0x[] 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 prepare 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95800 2019-12-07T19:34:19.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 io_done 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x194cce~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x194cce 0x194cce~47d to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x164cce 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcce 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x14b and padding block with 0xeb5 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19514b 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000113375s (0.000000086s flush + 0.000113289s kv commit) 2019-12-07T19:34:19.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95800> 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 kv_submitted 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95800 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95800 finishing 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95800 onodes 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95800 done 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95800 [] 2019-12-07T19:34:19.634+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:19.639+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 306 ==== MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=204, reqid=client.4540.0:336, at_version=42'21, trim_to=0'0, roll_forward_to=42'19)) v2 ==== 130314+0+0 (crc 0 0 0) 0x558c54895800 con 0x558c54c64000 2019-12-07T19:34:19.639+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000025 epoch 42 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=204, reqid=client.4540.0:336, at_version=42'21, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.639+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=204, reqid=client.4540.0:336, at_version=42'21, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.639+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=204, reqid=client.4540.0:336, at_version=42'21, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.639+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.639+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=204, reqid=client.4540.0:336, at_version=42'21, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) pg 0x558c5500a000 2019-12-07T19:34:19.639+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=204, reqid=client.4540.0:336, at_version=42'21, trim_to=0'0, roll_forward_to=42'19)) v2 pg pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.639+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=204, reqid=client.4540.0:336, at_version=42'21, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.add_oid 1:9694a0e5:::smithi13140796-3:13 c,e,11,13 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:13 got.empty() 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.set_snaps 1:9694a0e5:::smithi13140796-3:13 c,e,11,13 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_0000000000000011_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.639+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.639+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'19 (0'0,42'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'19], crt=42'19) [42'20 (29'13) clone 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,42'21 (29'13) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:336 2019-12-07T19:34:19.638034+0000 0 ObjectCleanRegions clean_offsets: [0~589824,1097728~18446744073708453887], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.639+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'20 (0'0,42'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'19 crt=42'19 lcod 42'19 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'20 (29'13) clone 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.639+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'21 (0'0,42'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'19 crt=42'19 lcod 42'19 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'21 (29'13) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:336 2019-12-07T19:34:19.638034+0000 0 ObjectCleanRegions clean_offsets: [0~589824,1097728~18446744073708453887], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.639+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'21 (0'0,42'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'19 crt=42'19 lcod 42'19 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 21 2019-12-07T19:34:19.639+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'21 (0'0,42'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'19 crt=42'19 lcod 42'19 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.639+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.639+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'20, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d58300 seq 53 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9694a0e5:::smithi13140796-3:13# key 0x8480000000000000019694a0e521736d'ithi13140796-3!='0x0000000000000013ffffffffffffffff'o' 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:13# 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1213 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.onode(0x558c54ce2a00).flush flush done 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 0x0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:13# 0x0~24000 -> 0x0~24000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x0~2000: 0x0~2000 Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10468 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54cf8850 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54cf8850 blob([0x282000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57818460 blob([0x282000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818460) get_ref 0x0~2000 Blob(0x558c57818460 blob([0x282000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818460) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57818460 blob([0x282000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10469 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c54cf89a0 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c54cf89a0 blob([0x284000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c578f0bd0 blob([0x284000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578f0bd0) get_ref 0x0~2000 Blob(0x558c578f0bd0 blob([0x284000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578f0bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c578f0bd0 blob([0x284000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10470 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c573158f0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c573158f0 blob([0x286000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57f51a40 blob([0x286000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57f51a40) get_ref 0x0~2000 Blob(0x558c57f51a40 blob([0x286000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57f51a40) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57f51a40 blob([0x286000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10471 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315810 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315810 blob([0x288000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c578f0b60 blob([0x288000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578f0b60) get_ref 0x0~2000 Blob(0x558c578f0b60 blob([0x288000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578f0b60) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c578f0b60 blob([0x288000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10472 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315730 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315730 blob([0x28a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57818310 blob([0x28a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818310) get_ref 0x0~2000 Blob(0x558c57818310 blob([0x28a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818310) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57818310 blob([0x28a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10473 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315650 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315650 blob([0x28c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57819500 blob([0x28c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57819500) get_ref 0x0~2000 Blob(0x558c57819500 blob([0x28c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57819500) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57819500 blob([0x28c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10474 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c552795e0 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c552795e0 blob([0x28e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57f50d90 blob([0x28e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57f50d90) get_ref 0x0~2000 Blob(0x558c57f50d90 blob([0x28e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57f50d90) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57f50d90 blob([0x28e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10475 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575bd960 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575bd960 blob([0x290000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57818070 blob([0x290000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818070) get_ref 0x0~2000 Blob(0x558c57818070 blob([0x290000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818070) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57818070 blob([0x290000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10476 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315570 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315570 blob([0x292000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57818000 blob([0x292000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818000) get_ref 0x0~2000 Blob(0x558c57818000 blob([0x292000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57818000) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57818000 blob([0x292000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10477 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315490 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315490 blob([0x294000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57819490 blob([0x294000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57819490) get_ref 0x0~2000 Blob(0x558c57819490 blob([0x294000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57819490) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57819490 blob([0x294000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10478 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c573153b0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c573153b0 blob([0x296000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c578f16c0 blob([0x296000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578f16c0) get_ref 0x0~2000 Blob(0x558c578f16c0 blob([0x296000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578f16c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c578f16c0 blob([0x296000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10479 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c573152d0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c573152d0 blob([0x298000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c578181c0 blob([0x298000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578181c0) get_ref 0x0~2000 Blob(0x558c578181c0 blob([0x298000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578181c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c578181c0 blob([0x298000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10480 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c573151f0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c573151f0 blob([0x29a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c578180e0 blob([0x29a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578180e0) get_ref 0x0~2000 Blob(0x558c578180e0 blob([0x29a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578180e0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c578180e0 blob([0x29a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10481 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315110 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315110 blob([0x29c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57819420 blob([0x29c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57819420) get_ref 0x0~2000 Blob(0x558c57819420 blob([0x29c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57819420) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57819420 blob([0x29c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10482 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315030 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315030 blob([0x29e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57db58f0 blob([0x29e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db58f0) get_ref 0x0~2000 Blob(0x558c57db58f0 blob([0x29e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db58f0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57db58f0 blob([0x29e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10483 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57314f50 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57314f50 blob([0x2a0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57db5880 blob([0x2a0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5880) get_ref 0x0~2000 Blob(0x558c57db5880 blob([0x2a0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5880) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c57db5880 blob([0x2a0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x20000~2000: 0x0~2000 Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10484 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315960 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315960 blob([0x2a2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57db5810 blob([0x2a2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5810) get_ref 0x0~2000 Blob(0x558c57db5810 blob([0x2a2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5810) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c57db5810 blob([0x2a2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10485 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57315a40 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57315a40 blob([0x2a4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=1)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup new Blob(0x558c57db57a0 blob([0x2a4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db57a0) get_ref 0x0~2000 Blob(0x558c57db57a0 blob([0x2a4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db57a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57db57a0 blob([0x2a4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=2)))) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.extentmap(0x558c54e45788) dirty_range mark inline shard dirty 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# -> 4#1:9694a0e5:::smithi13140796-3:13# = 0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# snapset 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# snapset = 0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys = 0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:19.639+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24000~1f000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9694a0e5:::smithi13140796-3:head# 0x24000~1f000 - have 0x24000 (147456) bytes fadvise_flags 0x20 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.639+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x24000~1f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58300 16 blobs 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.639+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1f000/1000,1f000,0 2019-12-07T19:34:19.639+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2b6000~1f000/1000,1f000,0 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2b6000~1f000] 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db56c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5730 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db56c0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5730) get_ref 0x0~2000 Blob(0x558c57db5730 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db56c0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5730) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57db5730 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db56c0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57db56d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e63180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5650 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63180 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5650) get_ref 0x0~2000 Blob(0x558c57db5650 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63180 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5650) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57db5650 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63180 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57e63198 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db52d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575bd490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db52d0 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd490 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db52d0) get_ref 0x0~2000 Blob(0x558c57db52d0 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd490 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db52d0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57db52d0 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd490 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c575bd4a8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db51f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5180 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db51f0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5180) get_ref 0x0~2000 Blob(0x558c57db5180 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db51f0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5180) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57db5180 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db51f0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57db5208 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d2a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5260 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2a80 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5260) get_ref 0x0~2000 Blob(0x558c57db5260 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2a80 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c57db5260) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57db5260 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2a80 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c574d2a98 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578df420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fc5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578df420 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc5b0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578df420) get_ref 0x0~2000 Blob(0x558c578df420 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc5b0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c578df420) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c578df420 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fc5b0 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fc5c8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fc620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fc690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fc620 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc690 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c579fc620) get_ref 0x0~2000 Blob(0x558c579fc620 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc690 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c579fc620) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c579fc620 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fc690 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fc6a8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fc700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fc770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fc700 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc770 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c579fc700) get_ref 0x0~2000 Blob(0x558c579fc700 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc770 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.blob(0x558c579fc700) get_ref init 0x2000, 1000 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c579fc700 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fc770 sbid 0x0)) 2019-12-07T19:34:19.639+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fc788 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fc7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fc850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fc7e0 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc850 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fc7e0) get_ref 0x0~2000 Blob(0x558c579fc7e0 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc850 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fc7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c579fc7e0 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fc850 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fc868 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fc8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fc930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fc8c0 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc930 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fc8c0) get_ref 0x0~2000 Blob(0x558c579fc8c0 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fc930 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fc8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c579fc8c0 blob([0x2c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fc930 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fc948 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fc9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fca10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fc9a0 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fca10 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fc9a0) get_ref 0x0~2000 Blob(0x558c579fc9a0 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fca10 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fc9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c579fc9a0 blob([0x2ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fca10 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fca28 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fca80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fcaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fca80 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcaf0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fca80) get_ref 0x0~2000 Blob(0x558c579fca80 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcaf0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fca80) get_ref init 0x2000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c579fca80 blob([0x2cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fcaf0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fcb08 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fcb60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fcbd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fcb60 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcbd0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fcb60) get_ref 0x0~2000 Blob(0x558c579fcb60 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcbd0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fcb60) get_ref init 0x2000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c579fcb60 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fcbd0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fcbe8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579fcc40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fccb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579fcc40 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fccb0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fcc40) get_ref 0x0~2000 Blob(0x558c579fcc40 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fccb0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c579fcc40) get_ref init 0x2000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c579fcc40 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fccb0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579fccc8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f50e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f518f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f50e00 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f518f0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c57f50e00) get_ref 0x0~2000 Blob(0x558c57f50e00 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f518f0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c57f50e00) get_ref init 0x2000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c57f50e00 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f518f0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f51908 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f510a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51490 blob([0x2d4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f510a0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c57f51490) get_ref 0x0~1000 Blob(0x558c57f51490 blob([0x2d4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f510a0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.blob(0x558c57f51490) get_ref init 0x1000, 1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~1000: 0x0~1000 Blob(0x558c57f51490 blob([0x2d4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57f510a0 sbid 0x0)) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f510b8 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x43000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x24000~1f000 = 0 2019-12-07T19:34:19.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_000000000000000C_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_000000000000000E_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_0000000000000011_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_0000000000000013_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000042.00000000000000000020' <- 0000000042.00000000000000000020 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000042.00000000000000000021' <- 0000000042.00000000000000000021 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58300 cost 199814 (17 ios * 4000 + 131814 bytes) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58300 onodes 0x558c54ce2a00,0x558c54e45680 shared_blobs 0x558c54cf88c0,0x558c54cf8a10,0x558c57314ee0,0x558c57314fc0,0x558c573150a0,0x558c57315180,0x558c57315260,0x558c57315340,0x558c57315420,0x558c57315500,0x558c573155e0,0x558c573156c0,0x558c573157a0,0x558c57315880,0x558c573159d0,0x558c57315ab0,0x558c574fa230,0x558c57567570 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update inline shard 790 bytes from 34 extents 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 1376 (580 bytes onode + 2 bytes spanning blobs + 794 bytes inline extents) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54e45788) update 4#1:9694a0e5:::smithi13140796-3:13# 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore.extentmap(0x558c54e45788) update inline shard 489 bytes from 18 extents 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:13# is 906 (411 bytes onode + 2 bytes spanning blobs + 493 bytes inline extents) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e4 is 11 SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e5 is 11 SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f3 is 11 SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f2 is 11 SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f1 is 11 SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f0 is 11 SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ef is 11 SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ee is 11 SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ed is 11 SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ec is 11 SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e9 is 11 SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e8 is 11 SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e7 is 11 SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e6 is 11 SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f4 is 11 SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f5 is 11 SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28eb is 11 SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ea is 11 SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=2))) 2019-12-07T19:34:19.640+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58300 allocated 0x[2b6000~1f000] released 0x[] 2019-12-07T19:34:19.640+0000 7f8f04181700 10 freelist allocate 0x2b6000~1f000 2019-12-07T19:34:19.640+0000 7f8f04181700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 prepare 2019-12-07T19:34:19.640+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58300 2019-12-07T19:34:19.640+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'21 (0'0,42'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'19 crt=42'19 lcod 42'19 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'21 2019-12-07T19:34:19.640+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.640+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c581cdb80 finish 2019-12-07T19:34:19.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.640+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 aio_wait 2019-12-07T19:34:19.640+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58300 2019-12-07T19:34:19.640+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 io_done 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5810 < min_flush_size 524288 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45680 had 1 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5833 < min_flush_size 524288 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5833 < min_flush_size 524288 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19514b~16c9 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19514b 0x19514b~16c9 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16514b 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x14b 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x814 and padding block with 0x7ec 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x196814 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107786s (0.000004097s flush + 0.000103689s kv commit) 2019-12-07T19:34:19.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58300> 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 kv_submitted 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58300 2019-12-07T19:34:19.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58300 finishing 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58300 onodes 0x558c54ce2a00,0x558c54e45680 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d2a98 in 0x558c53a59a40) _finish_write discard buffer(0x558c550e4a20 space 0x558c574d2a98 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bd4a8 in 0x558c53a59a40) _finish_write discard buffer(0x558c550e4900 space 0x558c575bd4a8 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fc5c8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f46f30 space 0x558c579fc5c8 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fc6a8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f76090 space 0x558c579fc6a8 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fc788 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f774d0 space 0x558c579fc788 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fc868 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f77560 space 0x558c579fc868 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fc948 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f777a0 space 0x558c579fc948 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=204, last_complete=42'21, committed=1, applied=1)) v2 -- 0x558c5795c800 con 0x558c54c64000 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fca28 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f775f0 space 0x558c579fca28 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fcb08 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f77680 space 0x558c579fcb08 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fcbe8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f77710 space 0x558c579fcbe8 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fccc8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f77830 space 0x558c579fccc8 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5208 in 0x558c53a59a40) _finish_write discard buffer(0x558c550e4990 space 0x558c57db5208 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db56d8 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a9ef0 space 0x558c57db56d8 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e63198 in 0x558c53a59a40) _finish_write discard buffer(0x558c550e4870 space 0x558c57e63198 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f510b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f77950 space 0x558c57f510b8 0x0~1000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51908 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f778c0 space 0x558c57f51908 0x0~2000 writing nocache) 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58300 done 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58300 [] 2019-12-07T19:34:19.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 18 ==== osd_op(client.4540.0:341 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 ==== 302+0+4 (crc 0 0 0) 0x558c57f13080 con 0x558c54c9f000 2019-12-07T19:34:19.641+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cdce0 prio 63 cost 4 latency 0.000016 epoch 42 osd_op(client.4540.0:341 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.641+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:341 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) 2019-12-07T19:34:19.641+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 19 ==== osd_op(client.4540.0:342 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 ==== 264+0+4 (crc 0 0 0) 0x558c57f13340 con 0x558c54c9f000 2019-12-07T19:34:19.641+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:341 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) queued 2019-12-07T19:34:19.641+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cde40 prio 63 cost 4 latency 0.000010 epoch 42 osd_op(client.4540.0:342 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.641+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:342 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) 2019-12-07T19:34:19.641+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.641+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:341 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) pg 0x558c57e88000 2019-12-07T19:34:19.641+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 20 ==== osd_op(client.4540.0:343 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 ==== 264+0+4 (crc 0 0 0) 0x558c57ce3080 con 0x558c54c9f000 2019-12-07T19:34:19.641+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581cdce0 prio 63 cost 4 latency 0.000043 osd_op(client.4540.0:341 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:342 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) queued 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 42 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 37 -> 42 (as per caller) 2019-12-07T19:34:19.642+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581cd8c0 prio 63 cost 4 latency 0.000017 epoch 42 osd_op(client.4540.0:343 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.642+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:343 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: osd_op(client.4540.0:341 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op: op osd_op(client.4540.0:341 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op osd_op(client.4540.0:341 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e42) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:19.642+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# snapset 2019-12-07T19:34:19.642+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# snapset = 0 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] find_object_context 1:4986b721:::smithi13140796-43:13 @13 snapset 0=[]:{} 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_object_context: obc NOT found in cache: 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# _ 2019-12-07T19:34:19.642+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# _ = 0 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] populate_obc_watchers 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:19.642+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# = 0 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_object_context: creating obc from disk: 0x558c54dd1200 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_object_context: 0x558c54dd1200 1:4986b721:::smithi13140796-43:head rwstate(none n=0 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b65a0 snapset: 0=[]:{} 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] find_object_context 1:4986b721:::smithi13140796-43:head want 13 > snapset seq 0 -- HIT 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 25 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op oi 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op obc obc(1:4986b721:::smithi13140796-43:head rwstate(read n=1 w=0)) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 0x558c57ed2000 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~720896,checksum in=4b,getxattrs] ov 21'4 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~720896,checksum in=4b,getxattrs] 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op read 0~720896 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] async_read noted for 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op checksum in=4b 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_checksum 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_checksum: async_read noted for 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 5(3) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] start_read_op: starting ReadOp(tid=28, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read_op: starting read ReadOp(tid=28, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57d9b180 con 0x558c54c64000 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57d9b400 con 0x558c54c5b400 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.2s3 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57d9b680 con 0x558c54c19c00 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57d9b900 con 0x558c53ae5400 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read_op: started ReadOp(tid=28, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=1(2),4(1),5(3),7(0)) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581cdce0 finish 2019-12-07T19:34:19.642+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c57d9b900 con 0x558c53ae5400 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.642+0000 7f8f1119b700 15 osd.7 42 enqueue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000023 epoch 42 MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:343 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) queued 2019-12-07T19:34:19.642+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:342 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) pg 0x558c57e88000 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cde40 prio 63 cost 4 latency 0.000365 osd_op(client.4540.0:342 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: osd_op(client.4540.0:342 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op: op osd_op(client.4540.0:342 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op osd_op(client.4540.0:342 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e42) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] find_object_context 1:4986b721:::smithi13140796-43:13 @13 snapset 0=[]:{} 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_object_context: found obc in cache: 0x558c54dd1200 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_object_context: 0x558c54dd1200 1:4986b721:::smithi13140796-43:head rwstate(read n=1 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b65a0 snapset: 0=[]:{} 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] find_object_context 1:4986b721:::smithi13140796-43:head want 13 > snapset seq 0 -- HIT 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f07988700 25 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op oi 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op obc obc(1:4986b721:::smithi13140796-43:head rwstate(read n=2 w=0)) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 0x558c57392a00 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~720896,checksum in=4b] ov 21'4 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~720896,checksum in=4b] 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op read 0~720896 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] async_read noted for 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op checksum in=4b 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_checksum 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_checksum: async_read noted for 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 5(3) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] start_read_op: starting ReadOp(tid=29, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read_op: starting read ReadOp(tid=29, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:19.642+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 84 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558c57d9b180 con 0x558c54c19c00 2019-12-07T19:34:19.642+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.000017 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.642+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.642+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57d9ac80 con 0x558c54c64000 2019-12-07T19:34:19.642+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57d9bb80 con 0x558c54c5b400 2019-12-07T19:34:19.642+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.2s3 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c57e8e000 con 0x558c54c19c00 2019-12-07T19:34:19.642+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c5793a000 con 0x558c53ae5400 2019-12-07T19:34:19.642+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c5793a000 con 0x558c53ae5400 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read_op: started ReadOp(tid=29, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=1(2),4(1),5(3),7(0)) 2019-12-07T19:34:19.642+0000 7f8f1119b700 15 osd.7 42 enqueue_op 0x558c57efec60 prio 127 cost 0 latency 0.000018 epoch 42 MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cde40 finish 2019-12-07T19:34:19.642+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:343 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8) prio 63 cost 4 e42) pg 0x558c57e88000 2019-12-07T19:34:19.642+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 202 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558c57d9b680 con 0x558c54c5b400 2019-12-07T19:34:19.642+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c54e79340 prio 127 cost 0 latency 0.000007 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.642+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581cd8c0 prio 63 cost 4 latency 0.000725 osd_op(client.4540.0:343 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: osd_op(client.4540.0:343 1.2s0 1.84ed6192 (undecoded) ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op: op osd_op(client.4540.0:343 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e42) v8 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op osd_op(client.4540.0:343 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e42) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] find_object_context 1:4986b721:::smithi13140796-43:13 @13 snapset 0=[]:{} 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_object_context: found obc in cache: 0x558c54dd1200 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_object_context: 0x558c54dd1200 1:4986b721:::smithi13140796-43:head rwstate(read n=2 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b65a0 snapset: 0=[]:{} 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] find_object_context 1:4986b721:::smithi13140796-43:head want 13 > snapset seq 0 -- HIT 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 25 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op oi 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_op obc obc(1:4986b721:::smithi13140796-43:head rwstate(read n=3 w=0)) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 0x558c57ed3c00 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~720896,checksum in=4b] ov 21'4 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~720896,checksum in=4b] 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op read 0~720896 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] async_read noted for 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op checksum in=4b 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_checksum 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_checksum: async_read noted for 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 5(3) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_all_avail_shards: checking acting 7(0) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] start_read_op: starting ReadOp(tid=30, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read_op: starting read ReadOp(tid=30, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c575f8000 con 0x558c54c64000 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c575f8280 con 0x558c54c5b400 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.2s3 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c575f8500 con 0x558c54c19c00 2019-12-07T19:34:19.642+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558c575f8780 con 0x558c53ae5400 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_read_op: started ReadOp(tid=30, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, []))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=1(2),4(1),5(3),7(0)) 2019-12-07T19:34:19.642+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 307 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 ==== 180389+0+0 (crc 0 0 0) 0x558c57e8e000 con 0x558c54c64000 2019-12-07T19:34:19.642+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581cd8c0 finish 2019-12-07T19:34:19.642+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558c575f8780 con 0x558c53ae5400 2019-12-07T19:34:19.642+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c54e79600 prio 127 cost 0 latency 0.000011 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.642+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.642+0000 7f8f1119b700 15 osd.7 42 enqueue_op 0x558c581cd1e0 prio 127 cost 0 latency 0.000022 epoch 42 MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.642+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.642+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.642+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000666 MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.642+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=28, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.642+0000 7f8f07988700 25 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:19.642+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~2c000 size 0x2c000 (180224) 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42bd0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c7e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c8c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c9a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ca80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.642+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cb60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecc40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecb60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580eca80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c770 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de9a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de7e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0770 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5dc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881730 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5c00 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0770 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de7e0 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de9a0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c700 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c070 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42bd0 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ce0 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5dc0 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ea0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c7e0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c8c0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c9a0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ca80 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cb60 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecc40 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea80 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c770 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580eca80 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecb60 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 203 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558c575f8500 con 0x558c54c5b400 2019-12-07T19:34:19.643+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c54e798c0 prio 127 cost 0 latency 0.000013 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:19.643+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 204 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558c575f8280 con 0x558c54c5b400 2019-12-07T19:34:19.643+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c54e79a20 prio 127 cost 0 latency 0.000010 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 308 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 ==== 180389+0+0 (crc 0 0 0) 0x558c57d9ac80 con 0x558c54c64000 2019-12-07T19:34:19.643+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000012 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57881748 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c54cdf560 space 0x558c57881748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57396ef8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c54cdf5f0 space 0x558c57396ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57db5c18 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa900 space 0x558c57db5c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0770 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c574e0788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa870 space 0x558c574e0788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de7e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578de7f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa7e0 space 0x558c578de7f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de9a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578de9b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa750 space 0x558c578de9b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5800c718 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa6c0 space 0x558c5800c718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5794c088 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa630 space 0x558c5794c088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42bd0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57c42be8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c579fa5a0 space 0x558c57c42be8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57db5cf8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57983d40 space 0x558c57db5cf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5dc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57db5dd8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57983dd0 space 0x558c57db5dd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57db5eb8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57e654d0 space 0x558c57db5eb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c7e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5800c7f8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46ea0 space 0x558c5800c7f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c8c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5800c8d8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46e10 space 0x558c5800c8d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c9a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5800c9b8 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46d80 space 0x558c5800c9b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ca80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5800ca98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46cf0 space 0x558c5800ca98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 85 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558c575f8000 con 0x558c54c19c00 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cb60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5800cb78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000012 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46c60 space 0x558c5800cb78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecc40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c580ecc58 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46bd0 space 0x558c580ecc58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c578dea98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46b40 space 0x558c578dea98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c770 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5794c788 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46ab0 space 0x558c5794c788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580eca80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c580eca98 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46a20 space 0x558c580eca98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecb60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c580ecb78 in 0x558c53a595e0) _discard 0x0~2000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 _add level 1 near 0 on buffer(0x558c57f46990 space 0x558c580ecb78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:19.643+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 = 180224 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read read request=180224 r=180224 len=180224 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.643+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: found on disk, size 180224 2019-12-07T19:34:19.643+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:34:19.643+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read: Checking hash of 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.643+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 -- 0x558c575f9180 con 0x558c53ae5400 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.643+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c575f9180 con 0x558c53ae5400 2019-12-07T19:34:19.643+0000 7f8f1119b700 15 osd.7 42 enqueue_op 0x558c581cc580 prio 127 cost 0 latency 0.000012 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.643+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.643+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.643+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.643+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.001138 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.643+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=28, attrs_read=0) 2019-12-07T19:34:19.643+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=28, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=1(2),4(1),7(0)) 2019-12-07T19:34:19.643+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581ccdc0 finish 2019-12-07T19:34:19.643+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.643+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 86 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558c57d9bb80 con 0x558c54c19c00 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.643+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000013 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c57efec60 prio 127 cost 0 latency 0.001159 MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=29, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.643+0000 7f8f07988700 25 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:19.643+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~2c000 size 0x2c000 (180224) 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42bd0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881730 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c7e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c8c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c9a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ca80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cb60 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecc40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 309 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 ==== 180389+0+0 (crc 0 0 0) 0x558c57d9b400 con 0x558c54c64000 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecb60 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c578d3ce0 prio 127 cost 0 latency 0.000012 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580eca80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de9a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de7e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.643+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 = 180224 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read read request=180224 r=180224 len=180224 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.643+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: found on disk, size 180224 2019-12-07T19:34:19.643+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:34:19.643+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read: Checking hash of 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.643+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 -- 0x558c57d9b900 con 0x558c53ae5400 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c57efec60 finish 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.643+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e79600 prio 127 cost 0 latency 0.001064 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.643+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=28, attrs_read=0) 2019-12-07T19:34:19.644+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c57d9b900 con 0x558c53ae5400 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=28, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=4(1),7(0)) 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:19.644+0000 7f8f1119b700 15 osd.7 42 enqueue_op 0x558c581cc160 prio 127 cost 0 latency 0.000050 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e79340 prio 127 cost 0 latency 0.001324 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=28, attrs_read=0) 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=28, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 4(1),180224, 5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=7(0)) 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e79340 finish 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c54e791e0 prio 127 cost 0 latency 0.000518 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=29, attrs_read=0) 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=29, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=1(2),4(1),7(0)) 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 310 ==== MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=205, reqid=client.4540.0:337, at_version=42'22, trim_to=0'0, roll_forward_to=42'19)) v2 ==== 129491+0+0 (crc 0 0 0) 0x558c575c1800 con 0x558c54c64000 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cd1e0 prio 127 cost 0 latency 0.001252 MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000027 epoch 42 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=205, reqid=client.4540.0:337, at_version=42'22, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.644+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=205, reqid=client.4540.0:337, at_version=42'22, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpRead(1.2s0 42/41 ECSubRead(tid=30, to_read={1:4986b721:::smithi13140796-43:head=0,180224,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.644+0000 7f8f07988700 25 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:19.644+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~2c000 size 0x2c000 (180224) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794db90 blob([0x913000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42bd0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d25b0 blob([0x915000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881730 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=205, reqid=client.4540.0:337, at_version=42'22, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578df500 blob([0x917000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c770 blob([0x919000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c7e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=205, reqid=client.4540.0:337, at_version=42'22, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) pg 0x558c5500a000 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c850 blob([0x91b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c8c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800c930 blob([0x91d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c9a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ca10 blob([0x91f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ca80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800caf0 blob([0x921000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cb60 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800cbd0 blob([0x923000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecc40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580ecbd0 blob([0x925000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ecb60 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=205, reqid=client.4540.0:337, at_version=42'22, trim_to=0'0, roll_forward_to=42'19)) v2 pg pg[1.9s4( v 42'21 (0'0,42'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580ecaf0 blob([0x927000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580eca80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580eca10 blob([0x929000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5794c700 blob([0x92b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809b0a0 blob([0x92d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dea80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'21 (0'0,42'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=205, reqid=client.4540.0:337, at_version=42'22, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578dec40 blob([0x92f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de9a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de8c0 blob([0x931000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de7e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578de700 blob([0x933000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e0bd0 blob([0x935000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fa3f0 blob([0x937000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5c70 blob([0x939000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'21 (0'0,42'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'21], crt=42'19) [42'22 (42'21) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:337 2019-12-07T19:34:19.641244+0000 0 ObjectCleanRegions clean_offsets: [0~1097728,1605632~18446744073707945983], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5d50 blob([0x93b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57db5e30 blob([0x93d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:19.644+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 = 180224 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'22 (0'0,42'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'21 crt=42'19 lcod 42'21 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'22 (42'21) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:337 2019-12-07T19:34:19.641244+0000 0 ObjectCleanRegions clean_offsets: [0~1097728,1605632~18446744073707945983], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read read request=180224 r=180224 len=180224 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'22 (0'0,42'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'21 crt=42'19 lcod 42'21 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 22 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'22 (0'0,42'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'21 crt=42'19 lcod 42'21 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.644+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.644+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'22, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.644+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] get_hash_info: found on disk, size 180224 2019-12-07T19:34:19.644+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:34:19.644+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read: Checking hash of 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740a000 seq 54 2019-12-07T19:34:19.644+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 -- 0x558c57d9b680 con 0x558c53ae5400 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cd1e0 finish 2019-12-07T19:34:19.644+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f1119b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.7 v2:172.21.15.109:6826/36939 0 ==== MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558c57d9b680 con 0x558c53ae5400 2019-12-07T19:34:19.644+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:19.644+0000 7f8f1119b700 15 osd.7 42 enqueue_op 0x558c581cc2c0 prio 127 cost 0 latency 0.000013 epoch 42 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x43000~1f000 2019-12-07T19:34:19.644+0000 7f8f1119b700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9694a0e5:::smithi13140796-3:head# 0x43000~1f000 - have 0x43000 (274432) bytes fadvise_flags 0x20 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.000912 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x43000~1f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=29, attrs_read=0) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5740a000 16 blobs 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=29, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=4(1),7(0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e798c0 prio 127 cost 0 latency 0.001225 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=29, attrs_read=0) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.644+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1f000/1000,1f000,0 2019-12-07T19:34:19.644+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2d5000~d000/1000,1f000,0 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=29, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 4(1),180224, 5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=7(0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x309000~12000/1000,1f000,0 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e798c0 finish 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2d5000~d000,0x309000~12000] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ec150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578195e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ec150 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578195e0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c580ec150) get_ref 0x0~2000 Blob(0x558c580ec150 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578195e0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c580ec150) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c580ec150 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578195e0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578195f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578183f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c54e786e0 prio 127 cost 0 latency 0.000664 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819570 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578183f0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57819570) get_ref 0x0~2000 Blob(0x558c57819570 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578183f0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57819570) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c57819570 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578183f0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57818408 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=30, attrs_read=0) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809afc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809afc0 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec0e0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c5809afc0) get_ref 0x0~2000 Blob(0x558c5809afc0 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec0e0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c5809afc0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c5809afc0 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec0e0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c580ec0f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57897e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57897e30 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec310 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57897e30) get_ref 0x0~2000 Blob(0x558c57897e30 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec310 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57897e30) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=30, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=1(2),4(1),7(0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c57897e30 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec310 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c54e786e0 finish 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c580ec328 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2d20 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42cb0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c574d2d20) get_ref 0x0~2000 Blob(0x558c574d2d20 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42cb0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c574d2d20) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558c574d2d20 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42cb0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57c42cc8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.644+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cc580 prio 127 cost 0 latency 0.000827 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42d90 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96070 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57c42d90) get_ref 0x0~2000 Blob(0x558c57c42d90 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96070 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57c42d90) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=28, attrs_read=0)) v2 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c57c42d90 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96070 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96088 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b960e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b961c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=28, attrs_read=0) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b960e0 blob([0x2e1000~1000,0x309000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b961c0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b960e0) get_ref 0x0~2000 Blob(0x558c57b960e0 blob([0x2e1000~1000,0x309000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b961c0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b960e0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558c57b960e0 blob([0x2e1000~1000,0x309000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b961c0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b961d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b96230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b962a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b96230 blob([0x30a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b962a0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b96230) get_ref 0x0~2000 Blob(0x558c57b96230 blob([0x30a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b962a0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b96230) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x51000~2000: 0x0~2000 Blob(0x558c57b96230 blob([0x30a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b962a0 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b962b8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b96310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b96310 blob([0x30c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96380 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b96310) get_ref 0x0~2000 Blob(0x558c57b96310 blob([0x30c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96380 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b96310) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x53000~2000: 0x0~2000 Blob(0x558c57b96310 blob([0x30c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96380 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96398 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b963f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b963f0 blob([0x30e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96460 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply Complete: ReadOp(tid=28, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 4(1),180224, 5(3),180224, 7(0),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b963f0) get_ref 0x0~2000 Blob(0x558c57b963f0 blob([0x30e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96460 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b963f0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x55000~2000: 0x0~2000 Blob(0x558c57b963f0 blob([0x30e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96460 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96478 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b964d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b964d0 blob([0x310000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96540 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b964d0) get_ref 0x0~2000 Blob(0x558c57b964d0 blob([0x310000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96540 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b964d0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x57000~2000: 0x0~2000 Blob(0x558c57b964d0 blob([0x310000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96540 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96558 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b965b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b965b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96620 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b965b0) get_ref 0x0~2000 Blob(0x558c57b965b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96620 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b965b0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x59000~2000: 0x0~2000 Blob(0x558c57b965b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96620 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96638 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b96690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b96690 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96700 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b96690) get_ref 0x0~2000 Blob(0x558c57b96690 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96700 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b96690) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5b000~2000: 0x0~2000 Blob(0x558c57b96690 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96700 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96718 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d756c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d756c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96770 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c54d756c0) get_ref 0x0~2000 Blob(0x558c54d756c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96770 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c54d756c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5d000~2000: 0x0~2000 Blob(0x558c54d756c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96770 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96788 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b967e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b967e0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96850 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b967e0) get_ref 0x0~2000 Blob(0x558c57b967e0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96850 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b967e0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5f000~2000: 0x0~2000 Blob(0x558c57b967e0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96850 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96868 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b968c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b968c0 blob([!~1000,0x31a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96930 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b968c0) get_ref 0x1000~1000 Blob(0x558c57b968c0 blob([!~1000,0x31a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96930 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.blob(0x558c57b968c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x61000~1000: 0x1000~1000 Blob(0x558c57b968c0 blob([!~1000,0x31a000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57b96930 sbid 0x0)) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57b96948 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x62000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] objects_read_async_cb: got: {1:4986b721:::smithi13140796-43:head=0,{0~720896(720896)}} 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x43000~1f000 = 0 2019-12-07T19:34:19.644+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.644+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000042.00000000000000000022' <- 0000000042.00000000000000000022 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740a000 cost 201142 (18 ios * 4000 + 129142 bytes) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740a000 onodes 0x558c54ce2a00 shared_blobs 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2b08) update inline shard 1113 bytes from 50 extents 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 1699 (580 bytes onode + 2 bytes spanning blobs + 1117 bytes inline extents) 2019-12-07T19:34:19.644+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740a000 allocated 0x[2d5000~d000,309000~12000] released 0x[] 2019-12-07T19:34:19.644+0000 7f8f08189700 10 freelist allocate 0x2d5000~d000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:19.644+0000 7f8f08189700 10 freelist allocate 0x309000~12000 2019-12-07T19:34:19.644+0000 7f8f08189700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 prepare 2019-12-07T19:34:19.644+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5740a000 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] finish_checksum 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 0x558c57ed2000 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~720896 out=720896b,checksum in=4b out=8b,getxattrs] ov 21'4 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'22 (0'0,42'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'21 crt=42'19 lcod 42'21 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'22 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~720896 out=720896b,checksum in=4b out=8b,getxattrs] 2019-12-07T19:34:19.644+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op read 0~720896 out=720896b 2019-12-07T19:34:19.644+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:19.644+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op getxattrs 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx alloc reply 0x558c575f9b80 result 0 2019-12-07T19:34:19.644+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:19.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 aio_wait 2019-12-07T19:34:19.644+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740a000 2019-12-07T19:34:19.644+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] log_op_stats osd_op(client.4540.0:341 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896 out=720896b,checksum in=4b out=8b,getxattrs out=75b] snapc 0=[] ondisk+read+known_if_redirected e42) v8 inb 0 outb 720979 lat 0.003037 2019-12-07T19:34:19.644+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 io_done 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.644+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:19.645+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:113 2019-12-07T19:34:19.645+0000 7f8f07988700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(341 smithi13140796-43 [read 0~720896 out=720896b,checksum out=8b,getxattrs out=75b] v0'0 uv4 ondisk = 0) v8 -- 0x558c575f9b80 con 0x558c54c9f000 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2961 < min_flush_size 524288 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cc580 finish 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2984 < min_flush_size 524288 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2984 < min_flush_size 524288 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x196814~ba8 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x196814 0x196814~ba8 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x166814 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x814 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3bc and padding block with 0xc44 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c578d3ce0 prio 127 cost 0 latency 0.001186 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=30, attrs_read=0) 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=30, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=4(1),7(0)) 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c578d3ce0 finish 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1973bc 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131789s (0.000004787s flush + 0.000127002s kv commit) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740a000> 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 kv_submitted 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740a000 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740a000 finishing 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740a000 onodes 0x558c54ce2a00 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818408 in 0x558c53a59a40) _finish_write discard buffer(0x558c583ca750 space 0x558c57818408 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578195f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f76a20 space 0x558c578195f8 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96088 in 0x558c53a59a40) _finish_write discard buffer(0x558c583ca990 space 0x558c57b96088 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b961d8 in 0x558c53a59a40) _finish_write discard buffer(0x558c583caa20 space 0x558c57b961d8 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b962b8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e29170 space 0x558c57b962b8 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96398 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28cf0 space 0x558c57b96398 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96478 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28d80 space 0x558c57b96478 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96558 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28e10 space 0x558c57b96558 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96638 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28ea0 space 0x558c57b96638 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96718 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28f30 space 0x558c57b96718 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96788 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e28fc0 space 0x558c57b96788 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96868 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e29050 space 0x558c57b96868 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e79a20 prio 127 cost 0 latency 0.001765 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96948 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e290e0 space 0x558c57b96948 0x1000~1000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42cc8 in 0x558c53a59a40) _finish_write discard buffer(0x558c583ca900 space 0x558c57c42cc8 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec0f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c583ca7e0 space 0x558c580ec0f8 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec328 in 0x558c53a59a40) _finish_write discard buffer(0x558c583ca870 space 0x558c580ec328 0x0~2000 writing nocache) 2019-12-07T19:34:19.645+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=205, last_complete=42'22, committed=1, applied=1)) v2 -- 0x558c5795ca00 con 0x558c54c64000 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740a000 done 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740a000 [] 2019-12-07T19:34:19.645+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.645+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=30, attrs_read=0) 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply readop not complete: ReadOp(tid=30, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 4(1),180224, 5(3),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=7(0)) 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581cc160 prio 127 cost 0 latency 0.001258 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=29, attrs_read=0)) v2 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=29, attrs_read=0) 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply Complete: ReadOp(tid=29, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 4(1),180224, 5(3),180224, 7(0),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=) 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] objects_read_async_cb: got: {1:4986b721:::smithi13140796-43:head=0,{0~720896(720896)}} 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] finish_checksum 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 0x558c57392a00 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~720896 out=720896b,checksum in=4b out=8b] ov 21'4 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~720896 out=720896b,checksum in=4b out=8b] 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op read 0~720896 out=720896b 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx alloc reply 0x558c57d9b400 result 0 2019-12-07T19:34:19.645+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:19.645+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] log_op_stats osd_op(client.4540.0:342 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896 out=720896b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e42) v8 inb 0 outb 720904 lat 0.003568 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.645+0000 7f8f03980700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:114 2019-12-07T19:34:19.645+0000 7f8f03980700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(342 smithi13140796-43 [read 0~720896 out=720896b,checksum out=8b] v0'0 uv4 ondisk = 0) v8 -- 0x558c57d9b400 con 0x558c54c9f000 2019-12-07T19:34:19.645+0000 7f8f03980700 10 osd.7 42 dequeue_op 0x558c581cc160 finish 2019-12-07T19:34:19.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2) prio 127 cost 0 e42) pg 0x558c57e88000 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cc2c0 prio 127 cost 0 latency 0.001264 MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 42/41 ECSubReadReply(tid=30, attrs_read=0)) v2 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply: reply ECSubReadReply(tid=30, attrs_read=0) 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=3 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_sub_read_reply Complete: ReadOp(tid=30, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,720896,0], need={1(2)=[0,8],4(1)=[0,8],5(3)=[0,8],7(0)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 720896, [1(2),180224, 4(1),180224, 5(3),180224, 7(0),180224]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=1(2),4(1),5(3),7(0)}, source_to_obj={1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(3)=1:4986b721:::smithi13140796-43:head,7(0)=1:4986b721:::smithi13140796-43:head}, in_progress=) 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] objects_read_async_cb: got: {1:4986b721:::smithi13140796-43:head=0,{0~720896(720896)}} 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] finish_checksum 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 0x558c57ed3c00 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~720896 out=720896b,checksum in=4b out=8b] ov 21'4 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~720896 out=720896b,checksum in=4b out=8b] 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op read 0~720896 out=720896b 2019-12-07T19:34:19.645+0000 7f8f07988700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] execute_ctx alloc reply 0x558c575f8280 result 0 2019-12-07T19:34:19.645+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:19.645+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] log_op_stats osd_op(client.4540.0:343 1.2s0 1:4986b721:::smithi13140796-43:13 [read 0~720896 out=720896b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e42) v8 inb 0 outb 720904 lat 0.003962 2019-12-07T19:34:19.645+0000 7f8f07988700 20 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:19.645+0000 7f8f07988700 15 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:115 2019-12-07T19:34:19.645+0000 7f8f07988700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(343 smithi13140796-43 [read 0~720896 out=720896b,checksum out=8b] v0'0 uv4 ondisk = 0) v8 -- 0x558c575f8280 con 0x558c54c9f000 2019-12-07T19:34:19.646+0000 7f8f07988700 10 osd.7 42 dequeue_op 0x558c581cc2c0 finish 2019-12-07T19:34:19.646+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:19.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 311 ==== MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=206, reqid=client.4540.0:338, at_version=42'23, trim_to=0'0, roll_forward_to=42'19)) v2 ==== 68051+0+0 (crc 0 0 0) 0x558c575f5800 con 0x558c54c64000 2019-12-07T19:34:19.646+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c578d3760 prio 127 cost 0 latency 0.000026 epoch 42 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=206, reqid=client.4540.0:338, at_version=42'23, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.646+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=206, reqid=client.4540.0:338, at_version=42'23, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.646+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=206, reqid=client.4540.0:338, at_version=42'23, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.646+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.646+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=206, reqid=client.4540.0:338, at_version=42'23, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) pg 0x558c5500a000 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c578d3760 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=206, reqid=client.4540.0:338, at_version=42'23, trim_to=0'0, roll_forward_to=42'19)) v2 pg pg[1.9s4( v 42'22 (0'0,42'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'22 (0'0,42'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=206, reqid=client.4540.0:338, at_version=42'23, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'22 (0'0,42'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'22], crt=42'19) [42'23 (42'22) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:338 2019-12-07T19:34:19.642974+0000 0 ObjectCleanRegions clean_offsets: [0~1605632,1867776~18446744073707683839], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'23 (0'0,42'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'22 crt=42'19 lcod 42'22 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'23 (42'22) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:338 2019-12-07T19:34:19.642974+0000 0 ObjectCleanRegions clean_offsets: [0~1605632,1867776~18446744073707683839], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'23 (0'0,42'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'22 crt=42'19 lcod 42'22 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 23 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'23 (0'0,42'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'22 crt=42'19 lcod 42'22 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.646+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.646+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'23, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54d59200 seq 55 2019-12-07T19:34:19.646+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:19.646+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:19.646+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x62000~10000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9694a0e5:::smithi13140796-3:head# 0x62000~10000 - have 0x62000 (401408) bytes fadvise_flags 0x20 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x62000~10000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d59200 8 blobs 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.646+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0x10000/1000,10000,0 2019-12-07T19:34:19.646+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x31b000~10000/1000,10000,0 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x31b000~10000] 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f679d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818380 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f679d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57818380) get_ref 0x0~2000 Blob(0x558c57818380 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f679d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57818380) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x62000~2000: 0x0~2000 Blob(0x558c57818380 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f679d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f679e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f519d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51960 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f519d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57f51960) get_ref 0x0~2000 Blob(0x558c57f51960 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f519d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57f51960) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x64000~2000: 0x0~2000 Blob(0x558c57f51960 blob([0x31d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f519d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f519e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e63c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578184d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e63c70 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578184d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57e63c70) get_ref 0x0~2000 Blob(0x558c57e63c70 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578184d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57e63c70) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x66000~2000: 0x0~2000 Blob(0x558c57e63c70 blob([0x31f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578184d0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578184e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819650 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57819650) get_ref 0x0~2000 Blob(0x558c57819650 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57819650) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x68000~2000: 0x0~2000 Blob(0x558c57819650 blob([0x321000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818620 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57818638 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578185b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578196c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578185b0 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578196c0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c578185b0) get_ref 0x0~2000 Blob(0x558c578185b0 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578196c0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c578185b0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6a000~2000: 0x0~2000 Blob(0x558c578185b0 blob([0x323000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578196c0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578196d8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818700 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818690 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57818700) get_ref 0x0~2000 Blob(0x558c57818700 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818690 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57818700) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6c000~2000: 0x0~2000 Blob(0x558c57818700 blob([0x325000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818690 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578186a8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578187e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819730 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57819730) get_ref 0x0~2000 Blob(0x558c57819730 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57819730) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6e000~2000: 0x0~2000 Blob(0x558c57819730 blob([0x327000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578187e0 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578187f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818770 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1650 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57818770) get_ref 0x0~2000 Blob(0x558c57818770 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1650 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.blob(0x558c57818770) get_ref init 0x2000, 1000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x70000~2000: 0x0~2000 Blob(0x558c57818770 blob([0x329000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1650 sbid 0x0)) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c578f1668 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x72000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 0x62000~10000 = 0 2019-12-07T19:34:19.646+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.646+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000042.00000000000000000023' <- 0000000042.00000000000000000023 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59200 cost 103702 (9 ios * 4000 + 67702 bytes) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59200 onodes 0x558c54ce2a00 shared_blobs 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update inline shard 1266 bytes from 58 extents 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore.extentmap(0x558c54ce2b08) reshard 0x[0,ffffffff) of 0 shards on 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard extent_avg 21, target 500, slop 100 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard new shard 0x0 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard new shard 0x2e000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard new shard 0x5b000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard new [0x0(0x0 bytes),0x2e000(0x0 bytes),0x5b000(0x0 bytes)] 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard old [] 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard fin [0x0(0x0 bytes),0x2e000(0x0 bytes),0x5b000(0x0 bytes)] 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# force 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update shard 0x0 is 584 bytes (was 0) from 23 extents 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update shard 0x2e000 is 445 bytes (was 0) from 23 extents 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update shard 0x5b000 is 247 bytes (was 0) from 12 extents 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 595 (593 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:19.646+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59200 allocated 0x[31b000~10000] released 0x[] 2019-12-07T19:34:19.646+0000 7f8f04181700 10 freelist allocate 0x31b000~10000 2019-12-07T19:34:19.646+0000 7f8f04181700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 prepare 2019-12-07T19:34:19.646+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d59200 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'23 (0'0,42'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'22 crt=42'19 lcod 42'22 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'23 2019-12-07T19:34:19.646+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c578d3760 finish 2019-12-07T19:34:19.646+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.646+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 aio_wait 2019-12-07T19:34:19.646+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59200 2019-12-07T19:34:19.646+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 io_done 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3140 < min_flush_size 524288 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3295 < min_flush_size 524288 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3318 < min_flush_size 524288 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3318 < min_flush_size 524288 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1973bc~cf6 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1973bc 0x1973bc~cf6 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x1673bc 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3bc 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb2 and padding block with 0xf4e 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1980b2 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118736s (0.000004510s flush + 0.000114226s kv commit) 2019-12-07T19:34:19.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d59200> 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 kv_submitted 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59200 2019-12-07T19:34:19.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59200 finishing 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59200 onodes 0x558c54ce2a00 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578184e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f77a70 space 0x558c578184e8 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818638 in 0x558c53a59a40) _finish_write discard buffer(0x558c573d8510 space 0x558c57818638 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578186a8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57a08b40 space 0x558c578186a8 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578187f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c53a69e60 space 0x558c578187f8 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578196d8 in 0x558c53a59a40) _finish_write discard buffer(0x558c551a8c60 space 0x558c578196d8 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1668 in 0x558c53a59a40) _finish_write discard buffer(0x558c550e46c0 space 0x558c578f1668 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f519e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57e282d0 space 0x558c57f519e8 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f679e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c550e4750 space 0x558c57f679e8 0x0~2000 writing nocache) 2019-12-07T19:34:19.646+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=206, last_complete=42'23, committed=1, applied=1)) v2 -- 0x558c5763da00 con 0x558c54c64000 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59200 done 2019-12-07T19:34:19.646+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59200 [] 2019-12-07T19:34:19.646+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.646+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.647+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 312 ==== MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=207, reqid=client.4540.0:339, at_version=42'24, trim_to=0'0, roll_forward_to=42'19)) v2 ==== 2481+0+0 (crc 0 0 0) 0x558c579e1800 con 0x558c54c64000 2019-12-07T19:34:19.647+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c578d38c0 prio 127 cost 0 latency 0.000026 epoch 42 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=207, reqid=client.4540.0:339, at_version=42'24, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.647+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=207, reqid=client.4540.0:339, at_version=42'24, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.647+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.647+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=207, reqid=client.4540.0:339, at_version=42'24, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.647+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.647+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=207, reqid=client.4540.0:339, at_version=42'24, trim_to=0'0, roll_forward_to=42'19)) v2) prio 127 cost 0 e42) pg 0x558c5500a000 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c578d38c0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=207, reqid=client.4540.0:339, at_version=42'24, trim_to=0'0, roll_forward_to=42'19)) v2 pg pg[1.9s4( v 42'23 (0'0,42'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'23 (0'0,42'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=207, reqid=client.4540.0:339, at_version=42'24, trim_to=0'0, roll_forward_to=42'19)) v2 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'23 (0'0,42'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'23], crt=42'19) [42'24 (42'23) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:339 2019-12-07T19:34:19.643008+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'23 crt=42'19 lcod 42'23 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'24 (42'23) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:339 2019-12-07T19:34:19.643008+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'23 crt=42'19 lcod 42'23 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 24 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'23 crt=42'19 lcod 42'23 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.647+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.647+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'24, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.647+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5740b800 seq 56 2019-12-07T19:34:19.647+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 3 keys 2019-12-07T19:34:19.647+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 3 keys = 0 2019-12-07T19:34:19.647+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.647+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.647+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000042.00000000000000000024' <- 0000000042.00000000000000000024 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.647+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.647+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b800 cost 6132 (1 ios * 4000 + 2132 bytes) 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b800 onodes 0x558c54ce2a00 shared_blobs 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 595 (593 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b800 allocated 0x[] released 0x[] 2019-12-07T19:34:19.647+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 prepare 2019-12-07T19:34:19.647+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b800 2019-12-07T19:34:19.647+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 io_done 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'23 crt=42'19 lcod 42'23 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'24 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1788 < min_flush_size 524288 2019-12-07T19:34:19.647+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c578d38c0 finish 2019-12-07T19:34:19.647+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1811 < min_flush_size 524288 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1811 < min_flush_size 524288 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1980b2~713 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1980b2 0x1980b2~713 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x1680b2 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb2 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7c5 and padding block with 0x83b 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1987c5 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000135997s (0.000000105s flush + 0.000135892s kv commit) 2019-12-07T19:34:19.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b800> 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 kv_submitted 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b800 2019-12-07T19:34:19.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b800 finishing 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b800 onodes 0x558c54ce2a00 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b800 done 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b800 [] 2019-12-07T19:34:19.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.647+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.647+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=207, last_complete=42'24, committed=1, applied=1)) v2 -- 0x558c5795ca00 con 0x558c54c64000 2019-12-07T19:34:19.647+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.652+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 175 ==== MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=116, reqid=client.4540.0:344, at_version=42'39, trim_to=0'0, roll_forward_to=41'37)) v2 ==== 134418+0+0 (unknown 0 0 0) 0x558c574a5800 con 0x558c54c5bc00 2019-12-07T19:34:19.652+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000031 epoch 42 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=116, reqid=client.4540.0:344, at_version=42'39, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.652+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=116, reqid=client.4540.0:344, at_version=42'39, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=116, reqid=client.4540.0:344, at_version=42'39, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.652+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=116, reqid=client.4540.0:344, at_version=42'39, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) pg 0x558c54d5c000 2019-12-07T19:34:19.652+0000 7f8f05984700 10 osd.7 42 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000080 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=116, reqid=client.4540.0:344, at_version=42'39, trim_to=0'0, roll_forward_to=41'37)) v2 pg pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.652+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=116, reqid=client.4540.0:344, at_version=42'39, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.add_oid 1:6ad15c4c:::smithi13140796-31:13 c,e,11,13 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.get_snaps 1:6ad15c4c:::smithi13140796-31:13 got.empty() 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.set_snaps 1:6ad15c4c:::smithi13140796-31:13 c,e,11,13 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.652+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.652+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 41'37 (0'0,41'37] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,41'37], crt=41'37) [42'38 (21'8) clone 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,42'39 (21'8) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:344 2019-12-07T19:34:19.649239+0000 0 ObjectCleanRegions clean_offsets: [0~507904,1032192~18446744073708519423], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.652+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'38 (0'0,42'38] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=41'37 crt=41'37 lcod 41'37 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'38 (21'8) clone 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.652+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'39 (0'0,42'39] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=41'37 crt=41'37 lcod 41'37 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'39 (21'8) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:344 2019-12-07T19:34:19.649239+0000 0 ObjectCleanRegions clean_offsets: [0~507904,1032192~18446744073708519423], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.652+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'39 (0'0,42'39] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=41'37 crt=41'37 lcod 41'37 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 39 2019-12-07T19:34:19.652+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'39 (0'0,42'39] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=41'37 crt=41'37 lcod 41'37 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.652+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.652+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'38, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ddf200 seq 69 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ad15c4c:::smithi13140796-31:13# key 0x8280000000000000016ad15c'L!smithi13140796-31!='0x0000000000000013ffffffffffffffff'o' 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# -> 2#1:6ad15c4c:::smithi13140796-31:13# 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1214 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.onode(0x558c54ce2280).flush flush done 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 0x0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# -> 2#1:6ad15c4c:::smithi13140796-31:13# 0x0~1f000 -> 0x0~1f000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x0~2000: 0x0~2000 Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10486 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57538b60 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57538b60 blob([0x11c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c54d3df10 blob([0x11c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c54d3df10) get_ref 0x0~2000 Blob(0x558c54d3df10 blob([0x11c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c54d3df10) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x0~2000: 0x0~2000 Blob(0x558c54d3df10 blob([0x11c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10487 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57538bd0 blob([0x11e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57538bd0 blob([0x11e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c57a07dc0 blob([0x11e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c57a07dc0) get_ref 0x0~2000 Blob(0x558c57a07dc0 blob([0x11e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c57a07dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57a07dc0 blob([0x11e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10488 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57538d20 blob([0x120000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57538d20 blob([0x120000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c57e63e30 blob([0x120000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c57e63e30) get_ref 0x0~2000 Blob(0x558c57e63e30 blob([0x120000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c57e63e30) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57e63e30 blob([0x120000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10489 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57538e00 blob([0x122000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57538e00 blob([0x122000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c57c42930 blob([0x122000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c57c42930) get_ref 0x0~2000 Blob(0x558c57c42930 blob([0x122000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c57c42930) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57c42930 blob([0x122000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10490 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57538ee0 blob([0x124000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57538ee0 blob([0x124000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c547402a0 blob([0x124000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c547402a0) get_ref 0x0~2000 Blob(0x558c547402a0 blob([0x124000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c547402a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c547402a0 blob([0x124000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10491 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57538fc0 blob([0x126000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57538fc0 blob([0x126000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c55278f50 blob([0x126000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c55278f50) get_ref 0x0~2000 Blob(0x558c55278f50 blob([0x126000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c55278f50) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c55278f50 blob([0x126000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0xc000~2000: 0x0~2000 Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10492 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575390a0 blob([0x128000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575390a0 blob([0x128000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c55279260 blob([0x128000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c55279260) get_ref 0x0~2000 Blob(0x558c55279260 blob([0x128000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c55279260) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c55279260 blob([0x128000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10493 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57539180 blob([0x12a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57539180 blob([0x12a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c54677880 blob([0x12a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c54677880) get_ref 0x0~2000 Blob(0x558c54677880 blob([0x12a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c54677880) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c54677880 blob([0x12a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10494 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57539260 blob([0x12c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57539260 blob([0x12c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c5523a1c0 blob([0x12c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c5523a1c0) get_ref 0x0~2000 Blob(0x558c5523a1c0 blob([0x12c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c5523a1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5523a1c0 blob([0x12c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10495 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57539340 blob([0x12e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57539340 blob([0x12e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c55264070 blob([0x12e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c55264070) get_ref 0x0~2000 Blob(0x558c55264070 blob([0x12e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c55264070) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c55264070 blob([0x12e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10496 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57539420 blob([0x130000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57539420 blob([0x130000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c575f6070 blob([0x130000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f6070) get_ref 0x0~2000 Blob(0x558c575f6070 blob([0x130000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f6070) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c575f6070 blob([0x130000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10497 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57539500 blob([0x132000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57539500 blob([0x132000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c575f6000 blob([0x132000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f6000) get_ref 0x0~2000 Blob(0x558c575f6000 blob([0x132000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f6000) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c575f6000 blob([0x132000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x18000~2000: 0x0~2000 Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10498 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575395e0 blob([0x134000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575395e0 blob([0x134000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c575f6380 blob([0x134000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f6380) get_ref 0x0~2000 Blob(0x558c575f6380 blob([0x134000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f6380) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c575f6380 blob([0x134000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10499 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575396c0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575396c0 blob([0x136000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c575f7c00 blob([0x136000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7c00) get_ref 0x0~2000 Blob(0x558c575f7c00 blob([0x136000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7c00) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c575f7c00 blob([0x136000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10500 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c575397a0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c575397a0 blob([0x138000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c575f7c70 blob([0x138000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7c70) get_ref 0x0~2000 Blob(0x558c575f7c70 blob([0x138000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7c70) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c575f7c70 blob([0x138000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup src 0x1e000~1000: 0x0~1000 Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10501 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57539880 blob([0x13a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57539880 blob([0x13a000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=1)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup new Blob(0x558c575f7ce0 blob([0x13a000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7ce0) get_ref 0x0~1000 Blob(0x558c575f7ce0 blob([0x13a000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7ce0) get_ref init 0x1000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dup dst 0x1e000~1000: 0x0~1000 Blob(0x558c575f7ce0 blob([0x13a000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=2)))) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dirty_range mark inline shard dirty 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.extentmap(0x558c57e8fa08) dirty_range mark inline shard dirty 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# -> 2#1:6ad15c4c:::smithi13140796-31:13# = 0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# snapset 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# snapset = 0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys = 0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:19.652+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x1f000~20000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6ad15c4c:::smithi13140796-31:head# 0x1f000~20000 - have 0x1f000 (126976) bytes fadvise_flags 0x0 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.652+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x1f000~20000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ddf200 16 blobs 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.652+0000 7f8f05984700 10 fbmap_alloc 0x558c53a72e00 allocate 0x20000/1000,20000,0 2019-12-07T19:34:19.652+0000 7f8f05984700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x32b000~c000/1000,20000,0 2019-12-07T19:34:19.652+0000 7f8f05984700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3c7000~14000/1000,20000,0 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x32b000~c000,0x3c7000~14000] 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7d50 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7dc0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7d50) get_ref 0x0~2000 Blob(0x558c575f7d50 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7dc0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7d50) get_ref init 0x2000, 1000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558c575f7d50 blob([0x32b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7dc0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575f7dd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7e30 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ea0 sbid 0x0)) 2019-12-07T19:34:19.652+0000 7f8f05984700 20 bluestore.blob(0x558c575f7e30) get_ref 0x0~2000 Blob(0x558c575f7e30 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ea0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c575f7e30) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558c575f7e30 blob([0x32d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7ea0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575f7eb8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7f10 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7f80 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c575f7f10) get_ref 0x0~2000 Blob(0x558c575f7f10 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7f80 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c575f7f10) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c575f7f10 blob([0x32f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7f80 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c575f7f98 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5742bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5742bce0 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51f80 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c5742bce0) get_ref 0x0~2000 Blob(0x558c5742bce0 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51f80 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c5742bce0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c5742bce0 blob([0x331000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51f80 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57f51f98 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51f10 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ea0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51f10) get_ref 0x0~2000 Blob(0x558c57f51f10 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ea0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51f10) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c57f51f10 blob([0x333000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51ea0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57f51eb8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51e30 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51dc0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51e30) get_ref 0x0~2000 Blob(0x558c57f51e30 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51dc0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51e30) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c57f51e30 blob([0x335000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51dc0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57f51dd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51d50 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ce0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51d50) get_ref 0x0~2000 Blob(0x558c57f51d50 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ce0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51d50) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c57f51d50 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51ce0 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57f51cf8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51c70 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51c00 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51c70) get_ref 0x0~2000 Blob(0x558c57f51c70 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51c00 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51c70) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c57f51c70 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51c00 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57f51c18 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51b90 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51b20 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51b90) get_ref 0x0~2000 Blob(0x558c57f51b90 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51b20 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f51b90) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c57f51b90 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51b20 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57f51b38 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f517a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f517a0 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f517a0) get_ref 0x0~2000 Blob(0x558c57f517a0 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57f517a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c57f517a0 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819340 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57819358 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f07e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f07e0 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1490 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c578f07e0) get_ref 0x0~2000 Blob(0x558c578f07e0 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1490 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c578f07e0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c578f07e0 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1490 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c578f14a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0700 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060150 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c578f0700) get_ref 0x0~2000 Blob(0x558c578f0700 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060150 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c578f0700) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c578f0700 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060150 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c55060168 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57654070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654000 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654070 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57654000) get_ref 0x0~2000 Blob(0x558c57654000 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654070 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c57654000) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c57654000 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57654070 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57654088 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576540e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57654150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576540e0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654150 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c576540e0) get_ref 0x0~2000 Blob(0x558c576540e0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654150 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c576540e0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c576540e0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57654150 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57654168 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576541c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57654230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576541c0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654230 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c576541c0) get_ref 0x0~2000 Blob(0x558c576541c0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654230 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c576541c0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c576541c0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57654230 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57654248 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576542a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57654310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576542a0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654310 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c576542a0) get_ref 0x0~2000 Blob(0x558c576542a0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654310 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.blob(0x558c576542a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c576542a0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57654310 sbid 0x0)) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57654328 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3f000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) dirty_range mark inline shard dirty 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x1f000~20000 = 0 2019-12-07T19:34:19.653+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.653+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.653+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_000000000000000C_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_000000000000000E_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_0000000000000013_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:19.653+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000042.00000000000000000038' <- 0000000042.00000000000000000038 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000042.00000000000000000039' <- 0000000042.00000000000000000039 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddf200 cost 203945 (17 ios * 4000 + 135945 bytes) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddf200 onodes 0x558c54ce2280,0x558c57e8f900 shared_blobs 0x558c54e43ea0,0x558c57538c40,0x558c57538d90,0x558c57538e70,0x558c57538f50,0x558c57539030,0x558c57539110,0x558c575391f0,0x558c575392d0,0x558c575393b0,0x558c57539490,0x558c57539570,0x558c57539650,0x558c57539730,0x558c57539810,0x558c575398f0 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) update inline shard 737 bytes from 32 extents 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 1324 (581 bytes onode + 2 bytes spanning blobs + 741 bytes inline extents) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.extentmap(0x558c57e8fa08) update 2#1:6ad15c4c:::smithi13140796-31:13# 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore.extentmap(0x558c57e8fa08) update inline shard 432 bytes from 16 extents 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:13# is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f6 is 11 SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f7 is 11 SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f8 is 11 SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f9 is 11 SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fa is 11 SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fb is 11 SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fc is 11 SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fd is 11 SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fe is 11 SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ff is 11 SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2900 is 11 SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2901 is 11 SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2902 is 11 SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2903 is 11 SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2904 is 11 SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2905 is 11 SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=2))) 2019-12-07T19:34:19.653+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddf200 allocated 0x[32b000~c000,3c7000~14000] released 0x[] 2019-12-07T19:34:19.653+0000 7f8f05984700 10 freelist allocate 0x32b000~c000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:19.653+0000 7f8f05984700 10 freelist allocate 0x3c7000~14000 2019-12-07T19:34:19.653+0000 7f8f05984700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 prepare 2019-12-07T19:34:19.653+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ddf200 2019-12-07T19:34:19.653+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.653+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'39 (0'0,42'39] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=41'37 crt=41'37 lcod 41'37 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'39 2019-12-07T19:34:19.653+0000 7f8f05984700 10 osd.7 42 dequeue_op 0x558c54e78840 finish 2019-12-07T19:34:19.653+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 aio_wait 2019-12-07T19:34:19.653+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddf200 2019-12-07T19:34:19.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 io_done 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5722 < min_flush_size 524288 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f900 had 1 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5745 < min_flush_size 524288 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5745 < min_flush_size 524288 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1987c5~1671 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1987c5 0x1987c5~1671 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x1687c5 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7c5 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe36 and padding block with 0x1ca 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x199e36 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000110230s (0.000004490s flush + 0.000105740s kv commit) 2019-12-07T19:34:19.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddf200> 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 kv_submitted 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddf200 2019-12-07T19:34:19.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf200 finishing 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddf200 onodes 0x558c54ce2280,0x558c57e8f900 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060168 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e293b0 space 0x558c55060168 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7dd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b440 space 0x558c575f7dd8 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7eb8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ee2d0 space 0x558c575f7eb8 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7f98 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ee090 space 0x558c575f7f98 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57654088 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e29320 space 0x558c57654088 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57654168 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e29290 space 0x558c57654168 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57654248 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e29200 space 0x558c57654248 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57654328 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f76900 space 0x558c57654328 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819358 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e294d0 space 0x558c57819358 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f14a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e29440 space 0x558c578f14a8 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51b38 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e29560 space 0x558c57f51b38 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51c18 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e28c60 space 0x558c57f51c18 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51cf8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e28bd0 space 0x558c57f51cf8 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51dd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e28b40 space 0x558c57f51dd8 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51eb8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57e28ab0 space 0x558c57f51eb8 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51f98 in 0x558c53a58fc0) _finish_write discard buffer(0x558c580cd710 space 0x558c57f51f98 0x0~2000 writing nocache) 2019-12-07T19:34:19.653+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 42/28 ECSubWriteReply(tid=116, last_complete=42'39, committed=1, applied=1)) v2 -- 0x558c5841f600 con 0x558c54c5bc00 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddf200 done 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:19.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddf200 [] 2019-12-07T19:34:19.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.653+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 313 ==== MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=208, reqid=client.4540.0:339, at_version=0'0, trim_to=0'0, roll_forward_to=42'24)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57a42800 con 0x558c54c64000 2019-12-07T19:34:19.654+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c57f3f080 prio 127 cost 0 latency 0.000017 epoch 42 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=208, reqid=client.4540.0:339, at_version=0'0, trim_to=0'0, roll_forward_to=42'24)) v2 2019-12-07T19:34:19.654+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=208, reqid=client.4540.0:339, at_version=0'0, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=208, reqid=client.4540.0:339, at_version=0'0, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=208, reqid=client.4540.0:339, at_version=0'0, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e42) pg 0x558c5500a000 2019-12-07T19:34:19.654+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c57f3f080 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=208, reqid=client.4540.0:339, at_version=0'0, trim_to=0'0, roll_forward_to=42'24)) v2 pg pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.654+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 42/41 ECSubWrite(tid=208, reqid=client.4540.0:339, at_version=0'0, trim_to=0'0, roll_forward_to=42'24)) v2 2019-12-07T19:34:19.654+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'19 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'24], crt=42'19) [] 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'20 (29'13) clone 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'21 (29'13) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:336 2019-12-07T19:34:19.638034+0000 0 ObjectCleanRegions clean_offsets: [0~589824,1097728~18446744073708453887], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'22 (42'21) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:337 2019-12-07T19:34:19.641244+0000 0 ObjectCleanRegions clean_offsets: [0~1097728,1605632~18446744073707945983], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'23 (42'22) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:338 2019-12-07T19:34:19.642974+0000 0 ObjectCleanRegions clean_offsets: [0~1605632,1867776~18446744073707683839], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'24 (42'23) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:339 2019-12-07T19:34:19.643008+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.654+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 24 2019-12-07T19:34:19.654+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.654+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.654+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.654+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:19.654+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57db3500 seq 57 2019-12-07T19:34:19.654+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:19.654+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:19.654+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.654+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.654+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:19.654+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db3500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:19.654+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db3500 onodes shared_blobs 2019-12-07T19:34:19.654+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db3500 allocated 0x[] released 0x[] 2019-12-07T19:34:19.654+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3500 prepare 2019-12-07T19:34:19.654+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db3500 2019-12-07T19:34:19.654+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3500 io_done 2019-12-07T19:34:19.654+0000 7f8f04181700 10 osd.7 42 dequeue_op 0x558c57f3f080 finish 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x199e36~471 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x199e36 0x199e36~471 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x169e36 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe36 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2a7 and padding block with 0xd59 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19a2a7 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000159007s (0.000000116s flush + 0.000158891s kv commit) 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57db3500> 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3500 kv_submitted 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db3500 2019-12-07T19:34:19.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3500 finishing 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db3500 onodes 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db3500 done 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:19.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db3500 [] 2019-12-07T19:34:19.654+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.654+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.654+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=208, last_complete=42'24, committed=1, applied=1)) v2 -- 0x558c5763da00 con 0x558c54c64000 2019-12-07T19:34:19.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:19.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 176 ==== MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=117, reqid=client.4540.0:345, at_version=42'40, trim_to=0'0, roll_forward_to=41'37)) v2 ==== 133592+0+0 (unknown 0 0 0) 0x558c579e3000 con 0x558c54c5bc00 2019-12-07T19:34:19.656+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000036 epoch 42 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=117, reqid=client.4540.0:345, at_version=42'40, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.656+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=117, reqid=client.4540.0:345, at_version=42'40, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=117, reqid=client.4540.0:345, at_version=42'40, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=117, reqid=client.4540.0:345, at_version=42'40, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) pg 0x558c54d5c000 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000429 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=117, reqid=client.4540.0:345, at_version=42'40, trim_to=0'0, roll_forward_to=41'37)) v2 pg pg[1.6s2( v 42'39 (0'0,42'39] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'39 (0'0,42'39] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=117, reqid=client.4540.0:345, at_version=42'40, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'39 (0'0,42'39] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'39], crt=41'37) [42'40 (42'39) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:345 2019-12-07T19:34:19.652404+0000 0 ObjectCleanRegions clean_offsets: [0~1032192,1556480~18446744073707995135], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'40 (0'0,42'40] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'39 crt=41'37 lcod 42'39 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'40 (42'39) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:345 2019-12-07T19:34:19.652404+0000 0 ObjectCleanRegions clean_offsets: [0~1032192,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'40 (0'0,42'40] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'39 crt=41'37 lcod 42'39 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 40 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'40 (0'0,42'40] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'39 crt=41'37 lcod 42'39 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.656+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'40, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54dde600 seq 70 2019-12-07T19:34:19.656+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:34:19.656+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:19.656+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x3f000~20000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6ad15c4c:::smithi13140796-31:head# 0x3f000~20000 - have 0x3f000 (258048) bytes fadvise_flags 0x0 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x3f000~20000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54dde600 16 blobs 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x20000/1000,20000,0 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3db000~20000/1000,20000,0 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3db000~20000] 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578961c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060310 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578961c0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c55060310) get_ref 0x0~2000 Blob(0x558c55060310 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578961c0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c55060310) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c55060310 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578961c0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c578961d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f90b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550602a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f90b60 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550602a0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90b60) get_ref 0x0~2000 Blob(0x558c57f90b60 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550602a0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90b60) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c57f90b60 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550602a0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c550602b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42a80 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42230 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42a80) get_ref 0x0~2000 Blob(0x558c57c42a80 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42230 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42a80) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c57c42a80 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42230 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57c42248 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42a10 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42380 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42a10) get_ref 0x0~2000 Blob(0x558c57c42a10 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42380 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42a10) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c57c42a10 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42380 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57c42398 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f90af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f90af0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90bd0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90af0) get_ref 0x0~2000 Blob(0x558c57f90af0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90bd0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90af0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c57f90af0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f90bd0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57f90be8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f90c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f90c40 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90cb0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90c40) get_ref 0x0~2000 Blob(0x558c57f90c40 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90cb0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90c40) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c57f90c40 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f90cb0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57f90cc8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f90d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f90d20 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90d90 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90d20) get_ref 0x0~2000 Blob(0x558c57f90d20 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90d90 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57f90d20) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558c57f90d20 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f90d90 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57f90da8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57880b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578191f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57880b60 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578191f0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57880b60) get_ref 0x0~2000 Blob(0x558c57880b60 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578191f0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57880b60) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c57880b60 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578191f0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57819208 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db59d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575833b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db59d0 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575833b0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57db59d0) get_ref 0x0~2000 Blob(0x558c57db59d0 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575833b0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57db59d0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558c57db59d0 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575833b0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575833c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57583490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6230 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583490 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6230) get_ref 0x0~2000 Blob(0x558c575f6230 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57583490 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6230) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x51000~2000: 0x0~2000 Blob(0x558c575f6230 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583490 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575834a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578192d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f63f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578192d0 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f63f0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c578192d0) get_ref 0x0~2000 Blob(0x558c578192d0 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f63f0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c578192d0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x53000~2000: 0x0~2000 Blob(0x558c578192d0 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f63f0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c575f6408 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6540 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819ce0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6540) get_ref 0x0~2000 Blob(0x558c575f6540 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819ce0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6540) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x55000~2000: 0x0~2000 Blob(0x558c575f6540 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819ce0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57819cf8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0690 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ab0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c578f0690) get_ref 0x0~2000 Blob(0x558c578f0690 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51ab0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c578f0690) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x57000~2000: 0x0~2000 Blob(0x558c578f0690 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51ab0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57f51ac8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819c00 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1500 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57819c00) get_ref 0x0~2000 Blob(0x558c57819c00 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1500 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57819c00) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x59000~2000: 0x0~2000 Blob(0x558c57819c00 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1500 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c578f1518 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578190a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819030 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57819030) get_ref 0x0~2000 Blob(0x558c57819030 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57819030) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5b000~2000: 0x0~2000 Blob(0x558c57819030 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578190a0 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c578190b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57819b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819b90 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819b20 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57819b90) get_ref 0x0~2000 Blob(0x558c57819b90 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819b20 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.blob(0x558c57819b90) get_ref init 0x2000, 1000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5d000~2000: 0x0~2000 Blob(0x558c57819b90 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819b20 sbid 0x0)) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57819b38 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5f000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) dirty_range mark inline shard dirty 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x3f000~20000 = 0 2019-12-07T19:34:19.656+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.656+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000042.00000000000000000040' <- 0000000042.00000000000000000040 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54dde600 cost 201242 (17 ios * 4000 + 133242 bytes) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54dde600 onodes 0x558c54ce2280 shared_blobs 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update inline shard 1041 bytes from 48 extents 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 1628 (581 bytes onode + 2 bytes spanning blobs + 1045 bytes inline extents) 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54dde600 allocated 0x[3db000~20000] released 0x[] 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 freelist allocate 0x3db000~20000 2019-12-07T19:34:19.656+0000 7f8f0197c700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 prepare 2019-12-07T19:34:19.656+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54dde600 2019-12-07T19:34:19.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'40 (0'0,42'40] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'39 crt=41'37 lcod 42'39 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'40 2019-12-07T19:34:19.657+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:19.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.657+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 aio_wait 2019-12-07T19:34:19.657+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54dde600 2019-12-07T19:34:19.657+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 io_done 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2864 < min_flush_size 524288 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2887 < min_flush_size 524288 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2887 < min_flush_size 524288 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19a2a7~b47 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19a2a7 0x19a2a7~b47 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16a2a7 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2a7 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdee and padding block with 0x212 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19adee 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000082601s (0.000004051s flush + 0.000078550s kv commit) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54dde600> 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 kv_submitted 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54dde600 2019-12-07T19:34:19.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54dde600 finishing 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54dde600 onodes 0x558c54ce2280 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550602b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c578ee480 space 0x558c550602b8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575833c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f47830 space 0x558c575833c8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575834a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f478c0 space 0x558c575834a8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f6408 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f47950 space 0x558c575f6408 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578190b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa000 space 0x558c578190b8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819208 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f477a0 space 0x558c57819208 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819b38 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa090 space 0x558c57819b38 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819cf8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f479e0 space 0x558c57819cf8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578961d8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c580cda70 space 0x558c578961d8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1518 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f47b00 space 0x558c578f1518 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42248 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f474d0 space 0x558c57c42248 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42398 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f47560 space 0x558c57c42398 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 42/28 ECSubWriteReply(tid=117, last_complete=42'40, committed=1, applied=1)) v2 -- 0x558c5841fc00 con 0x558c54c5bc00 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51ac8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f47a70 space 0x558c57f51ac8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90be8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f475f0 space 0x558c57f90be8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90cc8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f47680 space 0x558c57f90cc8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90da8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f47710 space 0x558c57f90da8 0x0~2000 writing nocache) 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54dde600 done 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:19.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54dde600 [] 2019-12-07T19:34:19.657+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.658+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1088190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.658+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 177 ==== MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=118, reqid=client.4540.0:346, at_version=42'41, trim_to=0'0, roll_forward_to=41'37)) v2 ==== 63960+0+0 (unknown 0 0 0) 0x558c57dfe800 con 0x558c54c5bc00 2019-12-07T19:34:19.658+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000021 epoch 42 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=118, reqid=client.4540.0:346, at_version=42'41, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.658+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=118, reqid=client.4540.0:346, at_version=42'41, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=118, reqid=client.4540.0:346, at_version=42'41, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=118, reqid=client.4540.0:346, at_version=42'41, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) pg 0x558c54d5c000 2019-12-07T19:34:19.658+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=118, reqid=client.4540.0:346, at_version=42'41, trim_to=0'0, roll_forward_to=41'37)) v2 pg pg[1.6s2( v 42'40 (0'0,42'40] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'40 (0'0,42'40] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=118, reqid=client.4540.0:346, at_version=42'41, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'40 (0'0,42'40] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'40], crt=41'37) [42'41 (42'40) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:346 2019-12-07T19:34:19.654050+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1802240~18446744073707749375], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'41 (0'0,42'41] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'40 crt=41'37 lcod 42'40 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'41 (42'40) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:346 2019-12-07T19:34:19.654050+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1802240~18446744073707749375], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'41 (0'0,42'41] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'40 crt=41'37 lcod 42'40 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 41 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'41 (0'0,42'41] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'40 crt=41'37 lcod 42'40 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.659+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'41, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f7800 seq 71 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x5f000~f000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6ad15c4c:::smithi13140796-31:head# 0x5f000~f000 - have 0x5f000 (389120) bytes fadvise_flags 0x0 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x5f000~f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c548f7800 8 blobs 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0xf000/1000,f000,0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x3fb000~f000/1000,f000,0 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x3fb000~f000] 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57896230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060620 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896230 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c55060620) get_ref 0x0~2000 Blob(0x558c55060620 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896230 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c55060620) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5f000~2000: 0x0~2000 Blob(0x558c55060620 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896230 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57896248 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57654ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654e70 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654ee0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57654e70) get_ref 0x0~2000 Blob(0x558c57654e70 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654ee0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57654e70) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x61000~2000: 0x0~2000 Blob(0x558c57654e70 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57654ee0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57654ef8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654fc0 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655030 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57654fc0) get_ref 0x0~2000 Blob(0x558c57654fc0 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655030 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57654fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x63000~2000: 0x0~2000 Blob(0x558c57654fc0 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57655030 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57655048 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576550a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576550a0 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655110 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c576550a0) get_ref 0x0~2000 Blob(0x558c576550a0 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655110 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c576550a0) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x65000~2000: 0x0~2000 Blob(0x558c576550a0 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57655110 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57655128 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576551f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655180 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576551f0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655180) get_ref 0x0~2000 Blob(0x558c57655180 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576551f0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655180) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x67000~2000: 0x0~2000 Blob(0x558c57655180 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576551f0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57655208 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576552d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655260 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576552d0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655260) get_ref 0x0~2000 Blob(0x558c57655260 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576552d0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655260) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x69000~2000: 0x0~2000 Blob(0x558c57655260 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576552d0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c576552e8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576553b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655340 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576553b0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655340) get_ref 0x0~2000 Blob(0x558c57655340 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576553b0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655340) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6b000~2000: 0x0~2000 Blob(0x558c57655340 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576553b0 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c576553c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655420 blob([!~1000,0x409000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655490 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655420) get_ref 0x1000~1000 Blob(0x558c57655420 blob([!~1000,0x409000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655490 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.blob(0x558c57655420) get_ref init 0x2000, 1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6d000~1000: 0x1000~1000 Blob(0x558c57655420 blob([!~1000,0x409000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57655490 sbid 0x0)) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c576554a8 in 0x558c53a58fc0) _discard 0x1000~1000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x6e000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) dirty_range mark inline shard dirty 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x5f000~f000 = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000042.00000000000000000041' <- 0000000042.00000000000000000041 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7800 cost 99610 (9 ios * 4000 + 63610 bytes) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7800 onodes 0x558c54ce2280 shared_blobs 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update inline shard 1206 bytes from 56 extents 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore.extentmap(0x558c54ce2388) reshard 0x[0,ffffffff) of 0 shards on 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard extent_avg 21, target 500, slop 100 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard new shard 0x0 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard new shard 0x2d000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard new shard 0x5b000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard new [0x0(0x0 bytes),0x2d000(0x0 bytes),0x5b000(0x0 bytes)] 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard old [] 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard fin [0x0(0x0 bytes),0x2d000(0x0 bytes),0x5b000(0x0 bytes)] 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# force 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update shard 0x0 is 566 bytes (was 0) from 23 extents 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update shard 0x2d000 is 442 bytes (was 0) from 23 extents 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update shard 0x5b000 is 208 bytes (was 0) from 10 extents 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 596 (594 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7800 allocated 0x[3fb000~f000] released 0x[] 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 freelist allocate 0x3fb000~f000 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 freelist _xor first_key 0x380000 last_key 0x400000 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 prepare 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c548f7800 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'41 (0'0,42'41] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'40 crt=41'37 lcod 42'40 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'41 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740c2c0 finish 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.659+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 aio_wait 2019-12-07T19:34:19.659+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7800 2019-12-07T19:34:19.659+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 io_done 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3260 < min_flush_size 524288 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3283 < min_flush_size 524288 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3283 < min_flush_size 524288 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19adee~cd3 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19adee 0x19adee~cd3 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16adee 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdee 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xac1 and padding block with 0x53f 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19bac1 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.659+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000102227s (0.000004024s flush + 0.000098203s kv commit) 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7800> 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 kv_submitted 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7800 2019-12-07T19:34:19.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7800 finishing 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7800 onodes 0x558c54ce2280 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57654ef8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa1b0 space 0x558c57654ef8 0x0~2000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655048 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa240 space 0x558c57655048 0x0~2000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655128 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa2d0 space 0x558c57655128 0x0~2000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655208 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa360 space 0x558c57655208 0x0~2000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576552e8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa3f0 space 0x558c576552e8 0x0~2000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576553c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa480 space 0x558c576553c8 0x0~2000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576554a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fa510 space 0x558c576554a8 0x1000~1000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57896248 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a120 space 0x558c57896248 0x0~2000 writing nocache) 2019-12-07T19:34:19.659+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 42/28 ECSubWriteReply(tid=118, last_complete=42'41, committed=1, applied=1)) v2 -- 0x558c5841f600 con 0x558c54c5bc00 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7800 done 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:19.659+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7800 [] 2019-12-07T19:34:19.659+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.659+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.659+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.659+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 178 ==== MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=119, reqid=client.4540.0:347, at_version=42'42, trim_to=0'0, roll_forward_to=41'37)) v2 ==== 2487+0+0 (unknown 0 0 0) 0x558c57a43000 con 0x558c54c5bc00 2019-12-07T19:34:19.659+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c5740d760 prio 127 cost 0 latency 0.000025 epoch 42 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=119, reqid=client.4540.0:347, at_version=42'42, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.659+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=119, reqid=client.4540.0:347, at_version=42'42, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=119, reqid=client.4540.0:347, at_version=42'42, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=119, reqid=client.4540.0:347, at_version=42'42, trim_to=0'0, roll_forward_to=41'37)) v2) prio 127 cost 0 e42) pg 0x558c54d5c000 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740d760 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=119, reqid=client.4540.0:347, at_version=42'42, trim_to=0'0, roll_forward_to=41'37)) v2 pg pg[1.6s2( v 42'41 (0'0,42'41] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'41 (0'0,42'41] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=119, reqid=client.4540.0:347, at_version=42'42, trim_to=0'0, roll_forward_to=41'37)) v2 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'41 (0'0,42'41] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'41], crt=41'37) [42'42 (42'41) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:347 2019-12-07T19:34:19.654079+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'41 crt=41'37 lcod 42'41 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 42'42 (42'41) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:347 2019-12-07T19:34:19.654079+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'41 crt=41'37 lcod 42'41 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 42 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'41 crt=41'37 lcod 42'41 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.659+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 42'42, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f6900 seq 72 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 3 keys 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 3 keys = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000042.00000000000000000042' <- 0000000042.00000000000000000042 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:19.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6900 cost 6137 (1 ios * 4000 + 2137 bytes) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6900 onodes 0x558c54ce2280 shared_blobs 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 597 (595 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:19.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6900 allocated 0x[] released 0x[] 2019-12-07T19:34:19.660+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 prepare 2019-12-07T19:34:19.660+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6900 2019-12-07T19:34:19.660+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 io_done 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.660+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'41 crt=41'37 lcod 42'41 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 42'42 2019-12-07T19:34:19.660+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740d760 finish 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1793 < min_flush_size 524288 2019-12-07T19:34:19.660+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1816 < min_flush_size 524288 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1816 < min_flush_size 524288 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19bac1~718 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19bac1 0x19bac1~718 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16bac1 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xac1 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1d9 and padding block with 0xe27 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19c1d9 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116875s (0.000000090s flush + 0.000116785s kv commit) 2019-12-07T19:34:19.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6900> 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 kv_submitted 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6900 2019-12-07T19:34:19.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6900 finishing 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6900 onodes 0x558c54ce2280 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6900 done 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:19.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6900 [] 2019-12-07T19:34:19.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.660+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.660+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 42/28 ECSubWriteReply(tid=119, last_complete=42'42, committed=1, applied=1)) v2 -- 0x558c5841fc00 con 0x558c54c5bc00 2019-12-07T19:34:19.660+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.665+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 179 ==== MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=120, reqid=client.4540.0:347, at_version=0'0, trim_to=0'0, roll_forward_to=42'42)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c57a43800 con 0x558c54c5bc00 2019-12-07T19:34:19.665+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c5740db80 prio 127 cost 0 latency 0.000021 epoch 42 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=120, reqid=client.4540.0:347, at_version=0'0, trim_to=0'0, roll_forward_to=42'42)) v2 2019-12-07T19:34:19.665+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=120, reqid=client.4540.0:347, at_version=0'0, trim_to=0'0, roll_forward_to=42'42)) v2) prio 127 cost 0 e42) 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=120, reqid=client.4540.0:347, at_version=0'0, trim_to=0'0, roll_forward_to=42'42)) v2) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=120, reqid=client.4540.0:347, at_version=0'0, trim_to=0'0, roll_forward_to=42'42)) v2) prio 127 cost 0 e42) pg 0x558c54d5c000 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740db80 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=120, reqid=client.4540.0:347, at_version=0'0, trim_to=0'0, roll_forward_to=42'42)) v2 pg pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.6s2 42/28 ECSubWrite(tid=120, reqid=client.4540.0:347, at_version=0'0, trim_to=0'0, roll_forward_to=42'42)) v2 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=41'37 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'42], crt=41'37) [] 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'38 (21'8) clone 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [13,11,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'39 (21'8) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:344 2019-12-07T19:34:19.649239+0000 0 ObjectCleanRegions clean_offsets: [0~507904,1032192~18446744073708519423], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'40 (42'39) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:345 2019-12-07T19:34:19.652404+0000 0 ObjectCleanRegions clean_offsets: [0~1032192,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'41 (42'40) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:346 2019-12-07T19:34:19.654050+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1802240~18446744073707749375], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=42'42 (42'41) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:347 2019-12-07T19:34:19.654079+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 42 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:19.665+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c548f6c00 seq 73 2019-12-07T19:34:19.665+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6c00 onodes shared_blobs 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 prepare 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6c00 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 io_done 2019-12-07T19:34:19.665+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740db80 finish 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19c1d9~471 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19c1d9 0x19c1d9~471 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16c1d9 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1d9 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x64a and padding block with 0x9b6 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19c64a 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:19.665+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129741s (0.000000085s flush + 0.000129656s kv commit) 2019-12-07T19:34:19.665+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6c00> 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:19.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 kv_submitted 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6c00 2019-12-07T19:34:19.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6c00 finishing 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6c00 onodes 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6c00 done 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:19.665+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6c00 [] 2019-12-07T19:34:19.665+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:19.665+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.665+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:19.665+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 42/28 ECSubWriteReply(tid=120, last_complete=42'42, committed=1, applied=1)) v2 -- 0x558c54d77000 con 0x558c54c5bc00 2019-12-07T19:34:19.665+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.666+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 180 ==== MOSDECSubOpRead(1.6s2 42/28 ECSubRead(tid=121, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c54e12f00 con 0x558c54c5bc00 2019-12-07T19:34:19.666+0000 7f8f23c56700 15 osd.7 42 enqueue_op 0x558c5740c580 prio 127 cost 0 latency 0.000016 epoch 42 MOSDECSubOpRead(1.6s2 42/28 ECSubRead(tid=121, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.666+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 42/28 ECSubRead(tid=121, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 42/28 ECSubRead(tid=121, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) queued 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 42/28 ECSubRead(tid=121, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e42) pg 0x558c54d5c000 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740c580 prio 127 cost 0 latency 0.000044 MOSDECSubOpRead(1.6s2 42/28 ECSubRead(tid=121, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpRead(1.6s2 42/28 ECSubRead(tid=121, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:19.666+0000 7f8f0197c700 25 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:19.666+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1000 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x6e000 (450560) 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538b60 blob([0x11c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 0x0~1000 = 4096 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] get_hash_info: Getting attr on 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] get_hash_info: not in cache 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] get_hash_info: found on disk, size 450560 2019-12-07T19:34:19.666+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key = 0 2019-12-07T19:34:19.666+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 42/28 ECSubReadReply(tid=121, attrs_read=0)) v2 -- 0x558c57977400 con 0x558c54c5bc00 2019-12-07T19:34:19.666+0000 7f8f0197c700 10 osd.7 42 dequeue_op 0x558c5740c580 finish 2019-12-07T19:34:19.666+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:19.708+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:19.708+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 10 osd.7 42 tick_without_osd_lock 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f6c1000/0x40000000/0x1680000000, data 0xa0e86a/0x93d000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 25 osd.7 42 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674802+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 25 osd.7 42 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674781+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 25 osd.7 42 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674735+0000 last_rx_front 2019-12-07T19:34:15.674747+0000 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 25 osd.7 42 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674761+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 25 osd.7 42 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674643+0000 last_rx_front 2019-12-07T19:34:15.674689+0000 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 20 osd.7 42 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 10 osd.7 42 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 20 osd.7 42 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:19.714+0000 7f8f1fbbe700 10 osd.7 42 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:19.743+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 12 ==== osd_ping(ping e42 up_from 13 ping_stamp 2019-12-07T19:34:19.744558+0000/43.216277278s send_stamp 43.216277278s delta_ub 0.036438250s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e400 con 0x558c552d1800 2019-12-07T19:34:19.743+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 12 ==== osd_ping(ping e42 up_from 13 ping_stamp 2019-12-07T19:34:19.744558+0000/43.216277278s send_stamp 43.216277278s delta_ub 0.036438250s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5763da00 con 0x558c552d1c00 2019-12-07T19:34:19.743+0000 7f8f24c58700 20 osd.7 42 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036038065s,0.036438250s]) 2019-12-07T19:34:19.743+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e42 up_from 13 ping_stamp 2019-12-07T19:34:19.744558+0000/43.216277278s send_stamp 43.180239213s delta_ub 18446744073.673513551s) v5 -- 0x558c5841e200 con 0x558c552d1800 2019-12-07T19:34:19.743+0000 7f8f24457700 20 osd.7 42 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.035983279s,0.036438250s]) 2019-12-07T19:34:19.743+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e42 up_from 13 ping_stamp 2019-12-07T19:34:19.744558+0000/43.216277278s send_stamp 43.180293999s delta_ub 18446744073.673568337s) v5 -- 0x558c5795c400 con 0x558c552d1c00 2019-12-07T19:34:19.758+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.808+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.858+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.909+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:19.917+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57be8400 0x558c54e35080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:19.917+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57be8400 0x558c54e35080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:19.917+0000 7f8f23c56700 10 osd.7 42 ms_handle_authentication new session 0x558c57404680 con 0x558c57be8400 entity client.admin addr 2019-12-07T19:34:19.917+0000 7f8f23c56700 10 osd.7 42 ms_handle_authentication session 0x558c57404680 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:19.917+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3810210677 conn(0x558c57be8400 0x558c54e35080 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4810 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:19.917+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4810 172.21.15.131:0/3810210677 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c581e8b40 con 0x558c57be8400 2019-12-07T19:34:19.917+0000 7f8f164bb700 20 osd.7 42 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:19.917+0000 7f8f164bb700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.917+0000 7f8f164bb700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.917+0000 7f8f164bb700 20 osd.7 42 _dispatch 0x558c581e8b40 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:19.917+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3810210677 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce0380 con 0x558c57be8400 2019-12-07T19:34:19.940+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4810 172.21.15.131:0/3810210677 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c575cc1e0 con 0x558c57be8400 2019-12-07T19:34:19.940+0000 7f8f164bb700 20 osd.7 42 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:19.940+0000 7f8f164bb700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:19.940+0000 7f8f164bb700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:19.940+0000 7f8f164bb700 20 osd.7 42 _dispatch 0x558c575cc1e0 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:19.941+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3810210677 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce0e00 con 0x558c57be8400 2019-12-07T19:34:19.943+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3810210677 conn(0x558c57be8400 msgr2=0x558c54e35080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 62 2019-12-07T19:34:19.943+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3810210677 conn(0x558c57be8400 msgr2=0x558c54e35080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:19.944+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3810210677 conn(0x558c57be8400 0x558c54e35080 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:19.944+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3810210677 conn(0x558c57be8400 0x558c54e35080 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:19.944+0000 7f8f164bb700 2 osd.7 42 ms_handle_reset con 0x558c57be8400 session 0x558c57404680 2019-12-07T19:34:19.959+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.009+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.060+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.110+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.160+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.210+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.260+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.310+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.356+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:20.360+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.410+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 tick 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 maybe_update_heartbeat_peers forcing update after 20.158424 seconds 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 reset_heartbeat_peers 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 maybe_update_heartbeat_peers updating 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 adding neighbor peer osd.1 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 adding neighbor peer osd.3 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 adding neighbor peer osd.5 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 adding random peer osd.2 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 adding random peer osd.4 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 maybe_update_heartbeat_peers 5 peers, extras 1,2,3,4,5 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:20.411+0000 7f8f21451700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:20.411+0000 7f8f21451700 20 osd.7 42 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:20.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 11 ==== osd_ping(ping e42 up_from 13 ping_stamp 2019-12-07T19:34:20.455287+0000/43.912210818s send_stamp 43.912210818s delta_ub 0.021844169s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77000 con 0x558c54c18800 2019-12-07T19:34:20.454+0000 7f8f23c56700 20 osd.7 42 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021289933s,0.021844169s]) 2019-12-07T19:34:20.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e42 up_from 13 ping_stamp 2019-12-07T19:34:20.455287+0000/43.912210818s send_stamp 43.890920885s delta_ub 18446744073.688261683s) v5 -- 0x558c5841f400 con 0x558c54c18800 2019-12-07T19:34:20.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 11 ==== osd_ping(ping e42 up_from 13 ping_stamp 2019-12-07T19:34:20.455287+0000/43.912210818s send_stamp 43.912210818s delta_ub 0.021844169s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d77000 con 0x558c54c19000 2019-12-07T19:34:20.454+0000 7f8f23c56700 20 osd.7 42 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021250603s,0.021844169s]) 2019-12-07T19:34:20.454+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e42 up_from 13 ping_stamp 2019-12-07T19:34:20.455287+0000/43.912210818s send_stamp 43.890960215s delta_ub 18446744073.688301013s) v5 -- 0x558c5841f600 con 0x558c54c19000 2019-12-07T19:34:20.460+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.510+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.522+0000 7f8f1219d700 20 osd.7 42 reports for 0 queries 2019-12-07T19:34:20.522+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c54e1f180 con 0x558c5476d800 2019-12-07T19:34:20.522+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:20.522+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x9de000/0x8c9000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:20.522+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(3 pgs tid 0 v 0) v2 -- 0x558c54e1f500 con 0x558c5476d800 2019-12-07T19:34:20.560+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.610+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1086357 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 205 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c55054a00 con 0x558c54c5b400 2019-12-07T19:34:20.614+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.bs1 epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:20.614+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e43) pg 0x558c575ef000 2019-12-07T19:34:20.614+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 43, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:20.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 206 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c55054a00 con 0x558c54c5b400 2019-12-07T19:34:20.615+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.3s4 epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:20.615+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e43) pg 0x558c5513e000 2019-12-07T19:34:20.615+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 43, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:20.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 msgr2=0x558c54c6fb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 97 2019-12-07T19:34:20.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 msgr2=0x558c54c6fb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:20.615+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=READY pgs=9 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:20.615+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=READY pgs=9 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:20.616+0000 7f8f0998c700 15 osd.7 42 enqueue_peering_evt 1.4s0 epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:20.616+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) 2019-12-07T19:34:20.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) queued 2019-12-07T19:34:20.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) pg 0x558c548ab000 2019-12-07T19:34:20.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 314 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:20.616+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:20.616+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] old_peering_msg reply_epoch 18 query_epoch 18 last_peering_reset 41 2019-12-07T19:34:20.616+0000 7f8f06986700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] discard old epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:20.616+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:20.616+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.8s5 epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:20.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:20.616+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e43) pg 0x558c57380000 2019-12-07T19:34:20.616+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 43, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 181 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c54e04700 con 0x558c54c5bc00 2019-12-07T19:34:20.616+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.1s1 epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:20.616+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 182 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c54e04700 con 0x558c54c5bc00 2019-12-07T19:34:20.616+0000 7f8f23c56700 15 osd.7 42 enqueue_peering_evt 1.fs5 epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:20.616+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e43) pg 0x558c57383000 2019-12-07T19:34:20.616+0000 7f8f05183700 20 osd.7 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 43, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e43) pg 0x558c5736a000 2019-12-07T19:34:20.616+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 43, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.617+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 315 ==== osd_map(43..43 src has 1..43) v4 ==== 661+0+0 (crc 0 0 0) 0x558c57580d80 con 0x558c54c64000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 osd.7 42 OSD::ms_dispatch: osd_map(43..43 src has 1..43) v4 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 osd.7 42 do_waiters -- start 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 osd.7 42 do_waiters -- finish 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 osd.7 42 _dispatch 0x558c57580d80 osd_map(43..43 src has 1..43) v4 2019-12-07T19:34:20.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 316 ==== MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=210, reqid=osd.1.0:210, at_version=43'26, trim_to=0'0, roll_forward_to=42'24)) v2 ==== 3449+0+0 (crc 0 0 0) 0x558c57a45800 con 0x558c54c64000 2019-12-07T19:34:20.617+0000 7f8f1199c700 3 osd.7 42 handle_osd_map epochs [43,43], i have 42, src has [1,43] 2019-12-07T19:34:20.617+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c57f3e160 prio 127 cost 0 latency 0.000031 epoch 43 MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=210, reqid=osd.1.0:210, at_version=43'26, trim_to=0'0, roll_forward_to=42'24)) v2 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 osd.7 42 handle_osd_map got inc map for epoch 43 2019-12-07T19:34:20.617+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=210, reqid=osd.1.0:210, at_version=43'26, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e43) 2019-12-07T19:34:20.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3088e826:::osdmap.42:0# 0x0~0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efcaf0 blob([0x971000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57efcb60 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=210, reqid=osd.1.0:210, at_version=43'26, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e43) queued 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efcbd0 blob([!~1000,0x972000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57efcc40 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=210, reqid=osd.1.0:210, at_version=43'26, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e43) pg 0x558c5500a000 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3088e826:::osdmap.42:0# 0x0~1448 = 5192 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 osd.7 42 add_map_bl 42 5192 bytes 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c57f3e160 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=210, reqid=osd.1.0:210, at_version=43'26, trim_to=0'0, roll_forward_to=42'24)) v2 pg pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 42 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 42 -> 43 (as per caller) 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=210, reqid=osd.1.0:210, at_version=43'26, trim_to=0'0, roll_forward_to=42'24)) v2 2019-12-07T19:34:20.617+0000 7f8f08189700 20 snap_mapper.update_snaps 1:9694a0e5:::smithi13140796-3:13 c,e,13 was 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.617+0000 7f8f08189700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:13 c,e,11,13 2019-12-07T19:34:20.617+0000 7f8f08189700 20 snap_mapper.set_snaps 1:9694a0e5:::smithi13140796-3:13 c,e,13 2019-12-07T19:34:20.617+0000 7f8f08189700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:20.617+0000 7f8f08189700 20 snap_mapper.update_snaps rm SNA_1_0000000000000011_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 osd.7 42 got_full_map 43, nothing requested 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 42'24 (0'0,42'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'24], crt=42'24) [43'25 (42'20) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:20.617350+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,43'26 (42'24) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:20.617350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 osd.7 42 handle_osd_map pg_num_history pg_num_history(e43 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {43={}} 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'25 (0'0,43'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'24 crt=42'24 lcod 42'24 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'25 (42'20) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:20.617350+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 osd.7 42 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e43 [1,43] lci=[13,43]) 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'26 (0'0,43'26] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'24 crt=42'24 lcod 42'24 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'26 (42'24) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:20.617350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c57cde000 seq 34 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'26 (0'0,43'26] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'24 crt=42'24 lcod 42'24 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 26 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'26 (0'0,43'26] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'24 crt=42'24 lcod 42'24 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c397972f:::inc_osdmap.43:0# key 0x7f7fffffffffffffffc39797'/!inc_osdmap.43!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:20.617+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:20.617+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 43'25, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c397972f:::inc_osdmap.43:0# 0x0~269 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1215 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c397972f:::inc_osdmap.43:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57cde000 1 blobs 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x40a000~1000/1000,1000,0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x40a000~1000] 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2540 blob([0x40a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38cb0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ea0900 seq 58 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c574d2540) get_ref 0x0~269 Blob(0x558c574d2540 blob([0x40a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38cb0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c574d2540) get_ref init 0x1000, 1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c574d2540 blob([0x40a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57c38cb0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c38cc8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35008) dirty_range mark inline shard dirty 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c397972f:::inc_osdmap.43:0# 0x0~269 = 0 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3b68e826:::osdmap.43:0# key 0x7f7fffffffffffffff3b68e8'&!osdmap.43!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3b68e826:::osdmap.43:0# 0x0~1464 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1216 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3b68e826:::osdmap.43:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57cde000 2 blobs 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x40b000~2000/1000,2000,0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x40b000~2000] 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c38d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c38d90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38e00 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c57c38d90) get_ref 0x0~1000 Blob(0x558c57c38d90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38e00 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c57c38d90) get_ref init 0x1000, 1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57c38d90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c38e00 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c38e18 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c38e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c38e70 blob([!~1000,0x40c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38ee0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c57c38e70) get_ref 0x1000~464 Blob(0x558c57c38e70 blob([!~1000,0x40c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57c38ee0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c57c38e70) get_ref init 0x2000, 1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c57c38e70 blob([!~1000,0x40c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57c38ee0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c38ef8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35288) dirty_range mark inline shard dirty 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3b68e826:::osdmap.43:0# 0x0~1464 = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57cde000 0 blobs 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57cde000 0 blobs 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cde000 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cde000 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34f00,0x558c57a35180 shared_blobs 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35008) update #-1:c397972f:::inc_osdmap.43:0# 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_0000000000000011_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35008) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c397972f:::inc_osdmap.43:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35288) update #-1:3b68e826:::osdmap.43:0# 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000043.00000000000000000025' <- 0000000043.00000000000000000025 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35288) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000043.00000000000000000026' <- 0000000043.00000000000000000026 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3b68e826:::osdmap.43:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cde000 allocated 0x[40a000~3000] released 0x[] 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 freelist allocate 0x40a000~3000 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:20.617+0000 7f8f1199c700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0900 cost 8418 (1 ios * 4000 + 4418 bytes) 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0900 onodes 0x558c54ce2a00,0x558c54e45680 shared_blobs 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 587 (585 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore.extentmap(0x558c54e45788) update 4#1:9694a0e5:::smithi13140796-3:13# 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde000 prepare 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:13# is 906 (411 bytes onode + 2 bytes spanning blobs + 493 bytes inline extents) 2019-12-07T19:34:20.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57cde000 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0900 allocated 0x[] released 0x[] 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 prepare 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0900 2019-12-07T19:34:20.617+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 io_done 2019-12-07T19:34:20.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'26 (0'0,43'26] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=42'24 crt=42'24 lcod 42'24 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 43'26 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4359 < min_flush_size 524288 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45680 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4382 < min_flush_size 524288 2019-12-07T19:34:20.617+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde000 aio_wait 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4382 < min_flush_size 524288 2019-12-07T19:34:20.617+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cde000 2019-12-07T19:34:20.617+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde000 io_done 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19c64a~111e to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19c64a 0x19c64a~111e to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16c64a 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x64a 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x768 and padding block with 0x898 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19d768 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124988s (0.000000167s flush + 0.000124821s kv commit) 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0900> 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 kv_submitted 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0900 2019-12-07T19:34:20.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0900 finishing 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0900 onodes 0x558c54ce2a00,0x558c54e45680 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0900 done 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:20.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0900 [] 2019-12-07T19:34:20.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a34f00 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35180 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19d768~226c to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19d768 0x19d768~226c to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~170000 x_off 0x16d768 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x768 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9d4 and padding block with 0x62c 2019-12-07T19:34:20.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 317 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:20.617+0000 7f8f24457700 15 osd.7 42 enqueue_peering_evt 1.es4 epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:20.617+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x19f9d4 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123914s (0.000003182s flush + 0.000120732s kv commit) 2019-12-07T19:34:20.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:20.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 318 ==== MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=211, reqid=osd.1.0:211, at_version=43'28, trim_to=0'0, roll_forward_to=42'24)) v2 ==== 3417+0+0 (crc 0 0 0) 0x558c575c0800 con 0x558c54c64000 2019-12-07T19:34:20.617+0000 7f8f24457700 15 osd.7 42 enqueue_op 0x558c57f3fa20 prio 127 cost 0 latency 0.000033 epoch 43 MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=211, reqid=osd.1.0:211, at_version=43'28, trim_to=0'0, roll_forward_to=42'24)) v2 2019-12-07T19:34:20.617+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=211, reqid=osd.1.0:211, at_version=43'28, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e43) 2019-12-07T19:34:20.617+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 42/41 ECSubWriteReply(tid=210, last_complete=43'26, committed=1, applied=1)) v2 -- 0x558c5795cc00 con 0x558c54c64000 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57cde000> 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde000 kv_submitted 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cde000 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c57cde000 osr 0x558c54ea4b40 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 31 0x1b000~1000 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=211, reqid=osd.1.0:211, at_version=43'28, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e43) queued 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 32 0x1b000~1000 crc 741e94e2 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 31 0x2000~1000 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 32 0x2000~1000 crc 5c7e48a2 2019-12-07T19:34:20.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=211, reqid=osd.1.0:211, at_version=43'28, trim_to=0'0, roll_forward_to=42'24)) v2) prio 127 cost 0 e43) pg 0x558c5500a000 2019-12-07T19:34:20.617+0000 7f8f141a1700 10 osd.7 42 _committed_osd_maps 43..43 2019-12-07T19:34:20.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.617+0000 7f8f141a1700 10 osd.7 42 advance to epoch 43 (<= last 43 <= newest_map 43) 2019-12-07T19:34:20.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e43) pg 0x558c54d98000 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 42 dequeue_op 0x558c57f3fa20 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=211, reqid=osd.1.0:211, at_version=43'28, trim_to=0'0, roll_forward_to=42'24)) v2 pg pg[1.9s4( v 43'26 (0'0,43'26] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:20.617+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.es4 peering, item epoch is 43, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.617+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'26 (0'0,43'26] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=211, reqid=osd.1.0:211, at_version=43'28, trim_to=0'0, roll_forward_to=42'24)) v2 2019-12-07T19:34:20.617+0000 7f8f141a1700 7 osd.7 43 consume_map version 43 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.8s5 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f08189700 20 snap_mapper.update_snaps 1:9b8adabe:::smithi13140796-6:11 c,e was 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.0s3 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.1s1 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.9s4 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.as5 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.2s0 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7:2.identify_splits_and_merges identify_splits_and_merges slot 1.2s4 has no pg and waiting_for_split 2019-12-07T19:34:20.617+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs3 has no pg and waiting_for_split 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.bs1 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.3s4 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.cs0 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.4s0 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.5s4 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.617+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.ds2 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.es4 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.6s2 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.fs5 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 43 identify_splits_and_merges 1.7s4 e42 to e43 pg_nums {18=16} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 43 prune_pg_created 2019-12-07T19:34:20.618+0000 7f8f08189700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:11 c,e,11 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:0.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 pending_peering first epoch 43 <= 43, requeueing 2019-12-07T19:34:20.618+0000 7f8f08189700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:11 c,e 2019-12-07T19:34:20.618+0000 7f8f08189700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {43=} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:1.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f08189700 20 snap_mapper.update_snaps rm SNA_1_0000000000000011_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 pending_peering first epoch 43 <= 43, requeueing 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {43=} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:2.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s4 empty, pruning 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:3.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs3 empty, pruning 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e43) pg 0x558c57380000 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 pending_peering first epoch 43 <= 43, requeueing 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {43=} 2019-12-07T19:34:20.618+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'26 (0'0,43'26] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,43'26], crt=42'24) [43'27 (42'18) modify 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:20.618370+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,43'28 (42'19) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:20.618370+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 43 <= 43, requeueing 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {43=} 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e43) pg 0x558c5736a000 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:4.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:5.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:20.618+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'27 (0'0,43'27] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'26 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'27 (42'18) modify 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:20.618370+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:6.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 pending_peering first epoch 43 <= 43, requeueing 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {43=} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7:7.consume_map 43 (was 42) 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:20.618+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'26 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'28 (42'19) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:20.618370+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 pending_peering first epoch 43 <= 43, requeueing 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {43=} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:20.618+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'26 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 28 2019-12-07T19:34:20.618+0000 7f8f141a1700 10 osd.7 43 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'26 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:20.618+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.8s5 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 43'27, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e43) pg 0x558c57383000 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.0s3 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e43) pg 0x558c5513e000 2019-12-07T19:34:20.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.1s1 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.9s4 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57cdf200 seq 59 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.as5 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# 1 keys 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# 1 keys = 0 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.2s0 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# hinfo_key (42 bytes) 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.bs1 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.3s4 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.cs0 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.4s0 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.5s4 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.8s0( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:20.618+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.ds2 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- SNA_1_0000000000000011_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:20.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c57e8b000 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000043.00000000000000000027' <- 0000000043.00000000000000000027 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000043.00000000000000000028' <- 0000000043.00000000000000000028 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cdf200 cost 7588 (1 ios * 4000 + 3588 bytes) 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cdf200 onodes 0x558c57a35400,0x558c57e8f180 shared_blobs 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57db2300 seq 24 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:20.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:20.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:20.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c57db3800 seq 14 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.3s0( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:20.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:20.618+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:20.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db2300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db2300 onodes shared_blobs 2019-12-07T19:34:20.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db2300 allocated 0x[] released 0x[] 2019-12-07T19:34:20.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c5740b200 seq 16 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db3800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db3800 onodes shared_blobs 2019-12-07T19:34:20.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db3800 allocated 0x[] released 0x[] 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 124509 (123450 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2300 prepare 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db2300 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2300 io_done 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:20.618+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3800 prepare 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c57ef9800 seq 52 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db3800 2019-12-07T19:34:20.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3800 io_done 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:20.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b200 onodes shared_blobs 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b200 allocated 0x[] released 0x[] 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c5500d000 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c5736a000 2019-12-07T19:34:20.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 prepare 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b200 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9800 onodes shared_blobs 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 io_done 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9800 allocated 0x[] released 0x[] 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 prepare 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9800 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 io_done 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c57e88000 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 42 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e43) pg 0x558c575ef000 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore.extentmap(0x558c57e8f288) update 4#1:9b8adabe:::smithi13140796-6:11# 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 42 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~1]] state: Active advmap 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] on_active_advmap new removed_snaps [11~1], snap_trimq now [11~1] 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:11# is 55375 (54316 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 42 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] state: Active: handling ActMap 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 43 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] Active: kicking snap trim 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] exit NotTrimming 2019-12-07T19:34:20.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming/WaitReservation 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c57676000 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f03980700 15 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:116 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:20.618+0000 7f8f03980700 7 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] state: handle ActMap primary 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cdf200 allocated 0x[] released 0x[] 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~1]] state: Active advmap 2019-12-07T19:34:20.618+0000 7f8f03980700 15 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:117 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf200 prepare 2019-12-07T19:34:20.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cdf200 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] on_active_advmap new removed_snaps [11~1], snap_trimq now [11~1] 2019-12-07T19:34:20.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf200 io_done 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f08189700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'26 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 43'28 2019-12-07T19:34:20.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c57522000 2019-12-07T19:34:20.618+0000 7f8f08189700 10 osd.7 43 dequeue_op 0x558c57f3fa20 finish 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c5500a000 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e43) pg 0x558c54d98000 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] state: Active: handling ActMap 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] Active: kicking snap trim 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 42 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 42 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] exit NotTrimming 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming/WaitReservation 2019-12-07T19:34:20.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 41:128 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c55141000 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 7 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] state: handle ActMap primary 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:129 2019-12-07T19:34:20.618+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 lcod 43'26 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 42 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:20.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 41 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c548ab000 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c57ff1200 seq 12 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 42 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:20.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c57380000 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1200 onodes shared_blobs 2019-12-07T19:34:20.618+0000 7f8f04982700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1200 allocated 0x[] released 0x[] 2019-12-07T19:34:20.618+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 prepare 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.es4 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1200 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 io_done 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.6s2 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c54d98000 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.fs5 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c54d5c000 2019-12-07T19:34:20.618+0000 7f8f141a1700 15 osd.7 43 enqueue_peering_evt 1.7s4 epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) 2019-12-07T19:34:20.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c57383000 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f05183700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 42 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:20.618+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f141a1700 7 osd.7 43 activate_map version 43 2019-12-07T19:34:20.618+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c54d5f000 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map: 43 2019-12-07T19:34:20.618+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 42 -> 43 2019-12-07T19:34:20.618+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 42 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~1]] state: Active advmap 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] on_active_advmap new removed_snaps [11~1], snap_trimq now [11~1] 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] state: Started advmap 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:20.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] plpg_on_pool_change 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] hit_set_clear 2019-12-07T19:34:20.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming: trimming 11 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] exit Trimming/WaitReservation 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] agent_stop 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 43 advance_pg new pool opts old pool opts 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] handle_activate_map: 43 2019-12-07T19:34:20.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 43:118 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] activate_map 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] state: Active: handling ActMap 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] Active: kicking snap trim 2019-12-07T19:34:20.618+0000 7f8f0b990700 10 osd.7 43 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] for snaptrim 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=43) prio 5 cost 1048576 e43) 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:20.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 43:119 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] exit NotTrimming 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57ef8600 seq 15 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming 2019-12-07T19:34:20.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:20.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming/WaitReservation 2019-12-07T19:34:20.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming: trimming 11 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] exit Trimming/WaitReservation 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8600 onodes shared_blobs 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8600 allocated 0x[] released 0x[] 2019-12-07T19:34:20.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:100 2019-12-07T19:34:20.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 43:130 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 prepare 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8600 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 io_done 2019-12-07T19:34:20.618+0000 7f8f0297e700 7 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] state: handle ActMap primary 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f0b990700 10 osd.7 43 queueing pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] for snaptrim 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=43) prio 5 cost 1048576 e43) 2019-12-07T19:34:20.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c575ef000 2019-12-07T19:34:20.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 42:101 2019-12-07T19:34:20.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] activate_map: Not dirtying info: last_persisted is 42 while current is 43 2019-12-07T19:34:20.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f0b990700 15 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 43:131 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 NullEvt) prio 255 cost 10 e43) pg 0x558c5513e000 2019-12-07T19:34:20.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=43) prio 5 cost 1048576 e43) queued 2019-12-07T19:34:20.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=43) prio 5 cost 1048576 e43) queued 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=43) prio 5 cost 1048576 e43) pg 0x558c57e88000 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=43) prio 5 cost 1048576 e43) pg 0x558c57676000 2019-12-07T19:34:20.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 43 epoch_requested: 43 NullEvt 2019-12-07T19:34:20.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:20.619+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] snap_trimmer posting 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] snap_trimmer posting 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 11 2019-12-07T19:34:20.619+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 11 2019-12-07T19:34:20.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 7 submitting 7 deferred done 0 stable 0 2019-12-07T19:34:20.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 319 ==== pg_lease(1.0s3 pg_lease(ru 52.075598409s ub 60.076769972s int 16s) e43/43) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5763da00 con 0x558c54c64000 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1580 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f24457700 15 osd.7 43 enqueue_peering_evt 1.0s3 epoch_sent: 43 epoch_requested: 43 MLease epoch 43 from osd.1 pg_lease(ru 52.075598409s ub 60.076769972s int 16s) 2019-12-07T19:34:20.619+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MLease epoch 43 from osd.1 pg_lease(ru 52.075598409s ub 60.076769972s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2311 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3461 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MLease epoch 43 from osd.1 pg_lease(ru 52.075598409s ub 60.076769972s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4611 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MLease epoch 43 from osd.1 pg_lease(ru 52.075598409s ub 60.076769972s int 16s)) prio 255 cost 10 e43) pg 0x558c5500d000 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:20.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MLease epoch 43 from osd.1 pg_lease(ru 52.075598409s ub 60.076769972s int 16s) 2019-12-07T19:34:20.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] proc_lease pg_lease(ru 52.075598409s ub 60.076769972s int 16s) 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 34348 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] peer_clock_delta_ub 0.021844169s -> ru 52.053754240s 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] proc_lease readable_until now 52.053754240s 2019-12-07T19:34:20.619+0000 7f8f0918b700 20 osd.7 43 agent_entry empty queue 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 67116 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] peer_clock_delta_lb 0.021250603s -> ruub 60.055519369s 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] proc_lease readable_until_ub now 60.055519369s 2019-12-07T19:34:20.619+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.0s0 pg_lease_ack(ruub 60.076769972s) e43/43) v1 -- 0x558c57f11a40 con 0x558c54c64000 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 99884 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 132652 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 165420 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 186596 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 320 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f180 had 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:20.619+0000 7f8f24457700 15 osd.7 43 enqueue_peering_evt 1.0s3 epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 60.076769972s int 16s) 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:20.619+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 60.076769972s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 187746 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 188896 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3859152896 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998355) bluestore 86 GiB free (0.959899), bluefs_ratio 0.0399413 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:20.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c550474c0) upper_bound after SNA_1_0000000000000011_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.0_0000000000000001.C' 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 188919 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 188919 < min_flush_size 524288 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c55047440) upper_bound after SNA_1_0000000000000011_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.0_0000000000000001.2' 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c550474c0) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..' 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x19f9d4~2e1f7 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c55047440) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..' 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x19f9d4 0x19f9d4~2e1f7 to file(ino 16 size 0x1a0000 mtime 2019-12-07T19:34:19.628227+0000 allocated 1a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~170000]) 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _allocate len 0x2dbcb from 0 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _allocate len 0x2dbcb from 1 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x30000/10000,30000,acd2d0000 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000011_.0_0000000000000001.C) returns 0 SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239760 in raw 0x558c55239760 len 70 nref 1) ) 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd2d0000~30000/10000,30000,acd2d0000 2019-12-07T19:34:20.619+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000011_.0_0000000000000001.2) returns 0 SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..,buffer::list(len=70, buffer::ptr(0~70 0x558c552382c0 in raw 0x558c552382c0 len 70 nref 1) ) 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x1d0000 to include allocated 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 47 (was clean) 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1a0000 x_off 0x16f9d4 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9d4 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbcb and padding block with 0x435 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x1d0000 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~1]] SnapTrimmer state: adding snap 11 to purged_snaps 2019-12-07T19:34:20.619+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~1]] SnapTrimmer state: adding snap 11 to purged_snaps 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~2]] SnapTrimmer state: purged_snaps now [a~1,d~1,10~2], snap_trimq now [] 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:20.619+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c548f6000 seq 36 2019-12-07T19:34:20.619+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:20.619+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~2]] SnapTrimmer state: purged_snaps now [a~1,d~1,10~2], snap_trimq now [] 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:20.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c54836f00 seq 20 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6000 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6000 onodes shared_blobs 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6000 allocated 0x[] released 0x[] 2019-12-07T19:34:20.619+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 prepare 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6000 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 io_done 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:20.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~2]] share_pg_info 2019-12-07T19:34:20.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54836f00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54836f00 onodes shared_blobs 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54836f00 allocated 0x[] released 0x[] 2019-12-07T19:34:20.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 prepare 2019-12-07T19:34:20.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e43/43) v1 -- 0x558c574e7800 con 0x558c54c64000 2019-12-07T19:34:20.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54836f00 2019-12-07T19:34:20.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 io_done 2019-12-07T19:34:20.619+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~2]] share_pg_info 2019-12-07T19:34:20.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e43/43) v1 -- 0x558c54e07100 con 0x558c54c19800 2019-12-07T19:34:20.619+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57ed3c00 con 0x558c54c64000 2019-12-07T19:34:20.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e43/43) v1 -- 0x558c54e04e00 con 0x558c54c5bc00 2019-12-07T19:34:20.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e43/43) v1 -- 0x558c55053c00 con 0x558c54c5b400 2019-12-07T19:34:20.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e43/43) v1 -- 0x558c55054300 con 0x558c54c19c00 2019-12-07T19:34:20.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 60.076769972s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 60.076769972s int 16s)) prio 255 cost 10 e43) pg 0x558c5500d000 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:20.619+0000 7f8f04982700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.0s0( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.075598409s ub 60.076769972s int 16s) 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim ps=[a~1,d~1,10~2]] exit Trimming 2019-12-07T19:34:20.619+0000 7f8f04982700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:20.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:20.619+0000 7f8f0297e700 15 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:132 2019-12-07T19:34:20.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] enter NotTrimming 2019-12-07T19:34:20.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c577fc300 seq 35 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:20.619+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:20.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:20.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:20.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:20.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] snap_trimmer complete 2019-12-07T19:34:20.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:20.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:20.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fc300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fc300 onodes shared_blobs 2019-12-07T19:34:20.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fc300 allocated 0x[] released 0x[] 2019-12-07T19:34:20.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 prepare 2019-12-07T19:34:20.619+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fc300 2019-12-07T19:34:20.619+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 io_done 2019-12-07T19:34:20.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.619+0000 7f8f0b990700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:20.619+0000 7f8f0b990700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: NotTrimming: trimming 11 2019-12-07T19:34:20.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[11~1] ps=[a~1,d~1,10~1]] exit Trimming/WaitReservation 2019-12-07T19:34:20.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.619+0000 7f8f0b990700 15 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 43:102 2019-12-07T19:34:20.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:20.619+0000 7f8f0b990700 10 osd.7 43 queueing pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[11~1] ps=[a~1,d~1,10~1]] for snaptrim 2019-12-07T19:34:20.619+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=43) prio 5 cost 1048576 e43) 2019-12-07T19:34:20.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~1] 2019-12-07T19:34:20.619+0000 7f8f0b990700 15 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] publish_stats_to_osd 43:103 2019-12-07T19:34:20.619+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=STANDBY pgs=9 cs=1 l=0 rx=0 tx=0).connect 2019-12-07T19:34:20.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 184 ==== MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=122, reqid=osd.3.0:122, at_version=43'44, trim_to=0'0, roll_forward_to=42'42)) v2 ==== 3458+0+0 (unknown 0 0 0) 0x558c57a44000 con 0x558c54c5bc00 2019-12-07T19:34:20.619+0000 7f8f23c56700 15 osd.7 43 enqueue_op 0x558c5740c000 prio 127 cost 0 latency 0.000035 epoch 43 MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=122, reqid=osd.3.0:122, at_version=43'44, trim_to=0'0, roll_forward_to=42'42)) v2 2019-12-07T19:34:20.619+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=122, reqid=osd.3.0:122, at_version=43'44, trim_to=0'0, roll_forward_to=42'42)) v2) prio 127 cost 0 e43) 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1cdbcb 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=43) prio 5 cost 1048576 e43) queued 2019-12-07T19:34:20.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=43) prio 5 cost 1048576 e43) pg 0x558c548ab000 2019-12-07T19:34:20.619+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] snap_trimmer posting 2019-12-07T19:34:20.619+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 11 2019-12-07T19:34:20.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.619+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:20.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:20.619+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 183 ==== osd_map(43..43 src has 1..43) v4 ==== 661+0+0 (unknown 0 0 0) 0x558c576b3200 con 0x558c54c5bc00 2019-12-07T19:34:20.619+0000 7f8f1199c700 20 osd.7 43 OSD::ms_dispatch: osd_map(43..43 src has 1..43) v4 2019-12-07T19:34:20.619+0000 7f8f1199c700 10 osd.7 43 do_waiters -- start 2019-12-07T19:34:20.619+0000 7f8f1199c700 10 osd.7 43 do_waiters -- finish 2019-12-07T19:34:20.620+0000 7f8f1199c700 20 osd.7 43 _dispatch 0x558c576b3200 osd_map(43..43 src has 1..43) v4 2019-12-07T19:34:20.619+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (47) on file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]), flushing log 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c55047540) upper_bound after SNA_1_0000000000000011_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.0_0000000000000001.4' 2019-12-07T19:34:20.620+0000 7f8f1199c700 3 osd.7 43 handle_osd_map epochs [43,43], i have 43, src has [1,43] 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f1199c700 10 osd.7 43 no new maps here, dropping 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 47 len 0x49 crc 0xa59a50a6) 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c55047540) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..' 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x2e000~1000 to file(ino 1 size 0x2e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x2e000 0x2e000~1000 to file(ino 1 size 0x2e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x2f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:20.620+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000011_.0_0000000000000001.4) returns 0 SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2e000 2019-12-07T19:34:20.620+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[11~1] ps=[a~1,d~1,10~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:20.620+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~1]] SnapTrimmer state: adding snap 11 to purged_snaps 2019-12-07T19:34:20.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:20.620+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~2]] SnapTrimmer state: purged_snaps now [a~1,d~1,10~2], snap_trimq now [] 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x2f000 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:20.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c5730a600 seq 33 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.620+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=122, reqid=osd.3.0:122, at_version=43'44, trim_to=0'0, roll_forward_to=42'42)) v2) prio 127 cost 0 e43) queued 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=122, reqid=osd.3.0:122, at_version=43'44, trim_to=0'0, roll_forward_to=42'42)) v2) prio 127 cost 0 e43) pg 0x558c54d5c000 2019-12-07T19:34:20.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:20.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a600 onodes shared_blobs 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a600 allocated 0x[] released 0x[] 2019-12-07T19:34:20.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 prepare 2019-12-07T19:34:20.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a600 2019-12-07T19:34:20.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 io_done 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 43 dequeue_op 0x558c5740c000 prio 127 cost 0 latency 0.000212 MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=122, reqid=osd.3.0:122, at_version=43'44, trim_to=0'0, roll_forward_to=42'42)) v2 pg pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 43 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 42 -> 43 (as per caller) 2019-12-07T19:34:20.620+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~2]] share_pg_info 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=122, reqid=osd.3.0:122, at_version=43'44, trim_to=0'0, roll_forward_to=42'42)) v2 2019-12-07T19:34:20.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c5737b100 con 0x558c54c64000 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6ad15c4c:::smithi13140796-31:13 c,e,13 was 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:20.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c5737aa00 con 0x558c54c19800 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6ad15c4c:::smithi13140796-31:13 c,e,11,13 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6ad15c4c:::smithi13140796-31:13 c,e,13 2019-12-07T19:34:20.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:20.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57378e00 con 0x558c54c5b400 2019-12-07T19:34:20.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57378000 con 0x558c54c19c00 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 42'42 (0'0,42'42] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'42], crt=42'42) [43'43 (42'38) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:20.620433+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,43'44 (42'42) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:20.620433+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:20.620+0000 7f8f06986700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:20.620+0000 7f8f06986700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim ps=[a~1,d~1,10~2]] exit Trimming 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000130 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'43 (0'0,43'43] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'42 crt=42'42 lcod 42'42 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'43 (42'38) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:20.620433+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.620+0000 7f8f06986700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'42 crt=42'42 lcod 42'42 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'44 (42'42) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:20.620433+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.620+0000 7f8f06986700 15 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:104 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'42 crt=42'42 lcod 42'42 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 44 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'42 crt=42'42 lcod 42'42 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:20.620+0000 7f8f06986700 20 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] enter NotTrimming 2019-12-07T19:34:20.620+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 43'43, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:20.620+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:20.620+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 47 2019-12-07T19:34:20.620+0000 7f8f06986700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] snap_trimmer complete 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 7 cleaned 0 in 0.001198431s (0.000000096s flush + 0.001198335s kv commit) 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57db2300,0x558c57db3800,0x558c5740b200,0x558c57ef9800,0x558c57cdf200,0x558c57ff1200,0x558c57ef8600> 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2300 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db2300 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2300 finishing 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db2300 onodes 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57ff0900 seq 74 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db2300 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db2300 [] 2019-12-07T19:34:20.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3800 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db3800 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys = 0 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3800 finishing 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db3800 onodes 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db3800 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db3800 [] 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2300 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b200 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b200 finishing 2019-12-07T19:34:20.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b200 onodes 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b200 done 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b200 [] 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3450 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9800 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9800 finishing 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9800 onodes 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9800 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9800 [] 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4600 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf200 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cdf200 2019-12-07T19:34:20.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf200 finishing 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:20.619+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57ed2000 con 0x558c54c19800 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4623 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4623 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cdf200 onodes 0x558c57a35400,0x558c57e8f180 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cdf200 done 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_0000000000000011_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cdf200 [] 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1cdbcb~120f to file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1cdbcb 0x1cdbcb~120f to file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1200 2019-12-07T19:34:20.620+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 43/41 ECSubWriteReply(tid=211, last_complete=43'28, committed=1, applied=1)) v2 -- 0x558c576b5600 con 0x558c54c64000 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000043.00000000000000000043' <- 0000000043.00000000000000000043 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1a0000 x_off 0x19dbcb 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1200 finishing 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbcb 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1200 onodes 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000043.00000000000000000044' <- 0000000043.00000000000000000044 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1200 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:20.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1200 [] 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdda and padding block with 0x226 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57a52000 con 0x558c54c5bc00 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8600 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:20.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8600 finishing 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8600 onodes 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8600 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8600 [] 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0900 cost 8447 (1 ios * 4000 + 4447 bytes) 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0900 onodes 0x558c54ce2280,0x558c57e8f900 shared_blobs 2019-12-07T19:34:20.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:20.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57a52700 con 0x558c54c5b400 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 589 (587 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57e8fa08) update 2#1:6ad15c4c:::smithi13140796-31:13# 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:13# is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0900 allocated 0x[] released 0x[] 2019-12-07T19:34:20.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 -- 0x558c57a52e00 con 0x558c54c19c00 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 prepare 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0900 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 io_done 2019-12-07T19:34:20.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=42'42 crt=42'42 lcod 42'42 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 43'44 2019-12-07T19:34:20.620+0000 7f8f07988700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:20.620+0000 7f8f0197c700 10 osd.7 43 dequeue_op 0x558c5740c000 finish 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f07988700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~2]] exit Trimming 2019-12-07T19:34:20.620+0000 7f8f07988700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:20.620+0000 7f8f07988700 15 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:120 2019-12-07T19:34:20.620+0000 7f8f07988700 20 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] enter NotTrimming 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1cedda 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.620+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:20.620+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000152458s (0.000000106s flush + 0.000152352s kv commit) 2019-12-07T19:34:20.620+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] snap_trimmer complete 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4338 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f6000,0x558c54836f00,0x558c577fc300,0x558c5730a600> 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6000 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6000 finishing 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6000 onodes 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6000 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6000 [] 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54836f00 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f900 had 1 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54836f00 finishing 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54836f00 onodes 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54836f00 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4361 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54836f00 [] 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4361 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fc300 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1cedda~1109 to file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fc300 finishing 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fc300 onodes 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fc300 done 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1cedda 0x1cedda~1109 to file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fc300 [] 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1a0000 x_off 0x19edda 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdda 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xee3 and padding block with 0x11d 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a600 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a600 finishing 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a600 onodes 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a600 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a600 [] 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1cfee3 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 321 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52e00 con 0x558c54c64000 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000183018s (0.000000090s flush + 0.000182928s kv commit) 2019-12-07T19:34:20.620+0000 7f8f24457700 15 osd.7 43 enqueue_peering_evt 1.7s4 epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:20.620+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e43) pg 0x558c54d5f000 2019-12-07T19:34:20.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 322 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57ed3c00 con 0x558c54c64000 2019-12-07T19:34:20.620+0000 7f8f05183700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.7s0( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:20.620+0000 7f8f24457700 15 osd.7 43 enqueue_peering_evt 1.as5 epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:20.620+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.620+0000 7f8f05183700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.620+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:20.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:20.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c574ff200 seq 57 2019-12-07T19:34:20.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.620+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:20.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e43) pg 0x558c57e8b000 2019-12-07T19:34:20.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:20.620+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=BANNER_CONNECTING pgs=9 cs=1 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0900> 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 kv_submitted 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0900 2019-12-07T19:34:20.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:20.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0900 finishing 2019-12-07T19:34:20.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff200 onodes shared_blobs 2019-12-07T19:34:20.620+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0900 onodes 0x558c54ce2280,0x558c57e8f900 2019-12-07T19:34:20.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff200 allocated 0x[] released 0x[] 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0900 done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0900 [] 2019-12-07T19:34:20.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 prepare 2019-12-07T19:34:20.620+0000 7f8f03980700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff200 2019-12-07T19:34:20.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 io_done 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:20.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.620+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 43/28 ECSubWriteReply(tid=122, last_complete=43'44, committed=1, applied=1)) v2 -- 0x558c5841fe00 con 0x558c54c5bc00 2019-12-07T19:34:20.620+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 285 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c548f7b00 seq 11 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1161 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:20.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:20.620+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:20.620+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:20.620+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1184 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1184 < min_flush_size 524288 2019-12-07T19:34:20.620+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f7b00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.620+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f7b00 onodes shared_blobs 2019-12-07T19:34:20.620+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 prepare 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1cfee3~4a0 to file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f7b00 2019-12-07T19:34:20.620+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 io_done 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1cfee3 0x1cfee3~4a0 to file(ino 16 size 0x1d0000 mtime 2019-12-07T19:34:20.620559+0000 allocated 1d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1a0000]) 2019-12-07T19:34:20.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _allocate len 0x383 from 0 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _allocate len 0x383 from 1 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd300000 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd300000~10000/10000,10000,acd300000 2019-12-07T19:34:20.620+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x1e0000 to include allocated 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 48 (was clean) 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x19fee3 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xee3 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x383 and padding block with 0xc7d 2019-12-07T19:34:20.620+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x1e0000 2019-12-07T19:34:20.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 msgr2=0x558c54c39180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 91 2019-12-07T19:34:20.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 msgr2=0x558c54c39180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d0383 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.621+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 unknown :-1 s=READY pgs=21 cs=1 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:20.621+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 unknown :-1 s=READY pgs=21 cs=1 l=0 rx=0 tx=0)._fault initiating reconnect 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (48) on file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]), flushing log 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 48 len 0x49 crc 0x873f0a70) 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x2f000~1000 to file(ino 1 size 0x2f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x2f000 0x2f000~1000 to file(ino 1 size 0x2f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x30000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2f000 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x30000 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:20.621+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 crc :-1 s=READY pgs=9 cs=1 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=f8982639b78e8be6 server_cookie=84aece1fce04073f in_seq=206 out_seq=206 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000023 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.621+0000 7f8f1199c700 10 osd.7 43 ms_handle_connect con 0x558c54c5b400 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 48 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000476746s (0.000000102s flush + 0.000476644s kv commit) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d0383~495 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d0383 0x1d0383~495 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a0383 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x383 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x818 and padding block with 0x7e8 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d0818 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ff200> 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 kv_submitted 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000069940s (0.000000072s flush + 0.000069868s kv commit) 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff200 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff200 finishing 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff200 onodes 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff200 done 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff200 [] 2019-12-07T19:34:20.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c548f7b00> 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 kv_submitted 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f7b00 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f7b00 finishing 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f7b00 onodes 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f7b00 done 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f7b00 [] 2019-12-07T19:34:20.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:20.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:20.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 125 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57ed2000 con 0x558c54c19800 2019-12-07T19:34:20.621+0000 7f8f24c58700 15 osd.7 43 enqueue_peering_evt 1.5s4 epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:20.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e43) pg 0x558c57522000 2019-12-07T19:34:20.621+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:20.621+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:20.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:20.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c57ff1800 seq 13 2019-12-07T19:34:20.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:20.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:20.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:20.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:20.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:20.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:20.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1800 onodes shared_blobs 2019-12-07T19:34:20.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1800 allocated 0x[] released 0x[] 2019-12-07T19:34:20.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 prepare 2019-12-07T19:34:20.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1800 2019-12-07T19:34:20.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 io_done 2019-12-07T19:34:20.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d0818~495 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d0818 0x1d0818~495 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a0818 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x818 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcad and padding block with 0x353 2019-12-07T19:34:20.621+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 126 ==== osd_map(43..43 src has 1..43) v4 ==== 661+0+0 (crc 0 0 0) 0x558c57570fc0 con 0x558c54c19800 2019-12-07T19:34:20.621+0000 7f8f1199c700 20 osd.7 43 OSD::ms_dispatch: osd_map(43..43 src has 1..43) v4 2019-12-07T19:34:20.621+0000 7f8f1199c700 10 osd.7 43 do_waiters -- start 2019-12-07T19:34:20.621+0000 7f8f1199c700 10 osd.7 43 do_waiters -- finish 2019-12-07T19:34:20.621+0000 7f8f1199c700 20 osd.7 43 _dispatch 0x558c57570fc0 osd_map(43..43 src has 1..43) v4 2019-12-07T19:34:20.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 127 ==== MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=56, reqid=osd.2.0:56, at_version=43'10, trim_to=0'0, roll_forward_to=42'8)) v2 ==== 3426+0+0 (crc 0 0 0) 0x558c579e3800 con 0x558c54c19800 2019-12-07T19:34:20.621+0000 7f8f1199c700 3 osd.7 43 handle_osd_map epochs [43,43], i have 43, src has [1,43] 2019-12-07T19:34:20.621+0000 7f8f1199c700 10 osd.7 43 no new maps here, dropping 2019-12-07T19:34:20.621+0000 7f8f24c58700 15 osd.7 43 enqueue_op 0x558c57eff760 prio 127 cost 0 latency 0.000021 epoch 43 MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=56, reqid=osd.2.0:56, at_version=43'10, trim_to=0'0, roll_forward_to=42'8)) v2 2019-12-07T19:34:20.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=56, reqid=osd.2.0:56, at_version=43'10, trim_to=0'0, roll_forward_to=42'8)) v2) prio 127 cost 0 e43) 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=56, reqid=osd.2.0:56, at_version=43'10, trim_to=0'0, roll_forward_to=42'8)) v2) prio 127 cost 0 e43) queued 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=56, reqid=osd.2.0:56, at_version=43'10, trim_to=0'0, roll_forward_to=42'8)) v2) prio 127 cost 0 e43) pg 0x558c55141000 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 43 dequeue_op 0x558c57eff760 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=56, reqid=osd.2.0:56, at_version=43'10, trim_to=0'0, roll_forward_to=42'8)) v2 pg pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 43 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 42 -> 43 (as per caller) 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=56, reqid=osd.2.0:56, at_version=43'10, trim_to=0'0, roll_forward_to=42'8)) v2 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 snap_mapper.update_snaps 1:bf81b261:::smithi13140796-37:13 c,e,13 was 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 snap_mapper.get_snaps 1:bf81b261:::smithi13140796-37:13 c,e,11,13 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 snap_mapper.set_snaps 1:bf81b261:::smithi13140796-37:13 c,e,13 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 snap_mapper.update_snaps rm SNA_1_0000000000000011_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 42'8 (0'0,42'8] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,42'8], crt=42'8) [43'9 (42'7) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:20.622200+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,43'10 (42'8) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:20.622200+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'9 (0'0,43'9] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=42'8 crt=42'8 lcod 42'8 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'9 (42'7) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:20.622200+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=42'8 crt=42'8 lcod 42'8 active mbc={} ps=[a~1,d~1,10~1]] add_log_entry 43'10 (42'8) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:20.622200+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=42'8 crt=42'8 lcod 42'8 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 10 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=42'8 crt=42'8 lcod 42'8 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:20.621+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 43'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:20.621+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 unknown :-1 s=BANNER_CONNECTING pgs=21 cs=2 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c577fdb00 seq 36 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000011_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_0000000000000011_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000043.00000000000000000009' <- 0000000043.00000000000000000009 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000043.00000000000000000010' <- 0000000043.00000000000000000010 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fdb00 cost 8415 (1 ios * 4000 + 4415 bytes) 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fdb00 onodes 0x558c54d96c80,0x558c57976000 shared_blobs 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96d88) update 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head# is 491 (483 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57976108) update 2#1:bf81b261:::smithi13140796-37:13# 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:13# is 931 (412 bytes onode + 2 bytes spanning blobs + 517 bytes inline extents) 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fdb00 allocated 0x[] released 0x[] 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 prepare 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fdb00 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 io_done 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=42'8 crt=42'8 lcod 42'8 active mbc={} ps=[a~1,d~1,10~1]] op_applied version 43'10 2019-12-07T19:34:20.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.621+0000 7f8f0217d700 10 osd.7 43 dequeue_op 0x558c57eff760 finish 2019-12-07T19:34:20.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.622+0000 7f8f1199c700 10 osd.7 43 ms_handle_connect con 0x558c54c5bc00 2019-12-07T19:34:20.622+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558c54c5bc00 0x558c54c39180 crc :-1 s=READY pgs=21 cs=2 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=64577724c341cd38 server_cookie=ea40093cfb40a50d in_seq=184 out_seq=177 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d0cad 2019-12-07T19:34:20.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 323 ==== MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=212, reqid=osd.1.0:211, at_version=0'0, trim_to=0'0, roll_forward_to=43'28)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c575bf800 con 0x558c54c64000 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.622+0000 7f8f24457700 15 osd.7 43 enqueue_op 0x558c57f3fe40 prio 127 cost 0 latency 0.000019 epoch 43 MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=212, reqid=osd.1.0:211, at_version=0'0, trim_to=0'0, roll_forward_to=43'28)) v2 2019-12-07T19:34:20.622+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=212, reqid=osd.1.0:211, at_version=0'0, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e43) 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001281909s (0.000000101s flush + 0.001281808s kv commit) 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff1800> 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 kv_submitted 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1800 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4305 < min_flush_size 524288 2019-12-07T19:34:20.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1800 finishing 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1800 onodes 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1800 done 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:20.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1800 [] 2019-12-07T19:34:20.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=212, reqid=osd.1.0:211, at_version=0'0, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e43) queued 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4328 < min_flush_size 524288 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4328 < min_flush_size 524288 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=212, reqid=osd.1.0:211, at_version=0'0, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e43) pg 0x558c5500a000 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d0cad~10e8 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d0cad 0x1d0cad~10e8 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a0cad 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcad 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd95 and padding block with 0x26b 2019-12-07T19:34:20.622+0000 7f8f08189700 10 osd.7 43 dequeue_op 0x558c57f3fe40 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=212, reqid=osd.1.0:211, at_version=0'0, trim_to=0'0, roll_forward_to=43'28)) v2 pg pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:20.622+0000 7f8f08189700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.9s4 43/41 ECSubWrite(tid=212, reqid=osd.1.0:211, at_version=0'0, trim_to=0'0, roll_forward_to=43'28)) v2 2019-12-07T19:34:20.622+0000 7f8f08189700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=42'24 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,43'28], crt=42'24) [] 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'25 (42'20) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:20.617350+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'26 (42'24) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:20.617350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'27 (42'18) modify 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:20.618370+0000 0 snaps [c,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.622+0000 7f8f08189700 20 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'28 (42'19) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:20.618370+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.622+0000 7f8f08189700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 28 2019-12-07T19:34:20.623+0000 7f8f08189700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:20.623+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:20.623+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d1d95 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000181560s (0.000000088s flush + 0.000181472s kv commit) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:20.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57cdf500 seq 60 2019-12-07T19:34:20.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:20.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:20.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cdf500 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cdf500 onodes shared_blobs 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cdf500 allocated 0x[] released 0x[] 2019-12-07T19:34:20.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf500 prepare 2019-12-07T19:34:20.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cdf500 2019-12-07T19:34:20.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf500 io_done 2019-12-07T19:34:20.623+0000 7f8f08189700 10 osd.7 43 dequeue_op 0x558c57f3fe40 finish 2019-12-07T19:34:20.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fdb00> 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 kv_submitted 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fdb00 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fdb00 finishing 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fdb00 onodes 0x558c54d96c80,0x558c57976000 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fdb00 done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fdb00 [] 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 185 ==== MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=123, reqid=osd.3.0:122, at_version=0'0, trim_to=0'0, roll_forward_to=43'44)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57a44800 con 0x558c54c5bc00 2019-12-07T19:34:20.623+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 43/41 ECSubWriteReply(tid=56, last_complete=43'10, committed=1, applied=1)) v2 -- 0x558c5841e600 con 0x558c54c19800 2019-12-07T19:34:20.623+0000 7f8f23c56700 15 osd.7 43 enqueue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000019 epoch 43 MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=123, reqid=osd.3.0:122, at_version=0'0, trim_to=0'0, roll_forward_to=43'44)) v2 2019-12-07T19:34:20.623+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=123, reqid=osd.3.0:122, at_version=0'0, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e43) 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=123, reqid=osd.3.0:122, at_version=0'0, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e43) queued 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=123, reqid=osd.3.0:122, at_version=0'0, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e43) pg 0x558c54d5c000 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1138 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f05984700 10 osd.7 43 dequeue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=123, reqid=osd.3.0:122, at_version=0'0, trim_to=0'0, roll_forward_to=43'44)) v2 pg pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1161 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1161 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.6s2 43/28 ECSubWrite(tid=123, reqid=osd.3.0:122, at_version=0'0, trim_to=0'0, roll_forward_to=43'44)) v2 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d1d95~489 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d1d95 0x1d1d95~489 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=42'42 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,43'44], crt=42'42) [] 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a1d95 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd95 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x21e and padding block with 0xde2 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'43 (42'38) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:20.620433+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'44 (42'42) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:20.620433+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.623+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 44 2019-12-07T19:34:20.623+0000 7f8f05984700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:20.623+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:20.623+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:20.623+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:20.623+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0300 seq 75 2019-12-07T19:34:20.623+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:20.623+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:20.623+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:20.623+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:20.623+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:20.623+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:20.623+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0300 onodes shared_blobs 2019-12-07T19:34:20.623+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0300 allocated 0x[] released 0x[] 2019-12-07T19:34:20.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 186 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52000 con 0x558c54c5bc00 2019-12-07T19:34:20.623+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 prepare 2019-12-07T19:34:20.623+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0300 2019-12-07T19:34:20.623+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 io_done 2019-12-07T19:34:20.623+0000 7f8f05984700 10 osd.7 43 dequeue_op 0x558c5740d1e0 finish 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f23c56700 15 osd.7 43 enqueue_peering_evt 1.6s2 epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.6s0( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:20.623+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.6s0( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d221e 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.6s0( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.6s0( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e43) pg 0x558c54d5c000 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000213669s (0.000000123s flush + 0.000213546s kv commit) 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57cdf500> 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf500 kv_submitted 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cdf500 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf500 finishing 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cdf500 onodes 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 3(0) info: 1.6s0( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cdf500 done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cdf500 [] 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 324 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52700 con 0x558c54c64000 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d221e~471 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d221e 0x1d221e~471 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f24457700 15 osd.7 43 enqueue_peering_evt 1.9s4 epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.9s0( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a221e 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:20.623+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.9s0( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x21e 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57b94c00 seq 76 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x68f and padding block with 0x971 2019-12-07T19:34:20.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b94c00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b94c00 onodes shared_blobs 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b94c00 allocated 0x[] released 0x[] 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 prepare 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b94c00 2019-12-07T19:34:20.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 io_done 2019-12-07T19:34:20.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.9s0( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.9s0( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e43) pg 0x558c5500a000 2019-12-07T19:34:20.623+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 1(0) info: 1.9s0( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:20.623+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.623+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:20.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:20.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57db3b00 seq 61 2019-12-07T19:34:20.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:20.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:20.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:20.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:20.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db3b00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:20.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db3b00 onodes shared_blobs 2019-12-07T19:34:20.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db3b00 allocated 0x[] released 0x[] 2019-12-07T19:34:20.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3b00 prepare 2019-12-07T19:34:20.623+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db3b00 2019-12-07T19:34:20.623+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3b00 io_done 2019-12-07T19:34:20.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d268f 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.623+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 43/41 ECSubWriteReply(tid=212, last_complete=43'28, committed=1, applied=1)) v2 -- 0x558c55271c00 con 0x558c54c64000 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000167740s (0.000000089s flush + 0.000167651s kv commit) 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0300> 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 kv_submitted 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0300 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0300 finishing 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0300 onodes 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0300 done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94c00 kv_queued 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0300 [] 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2252 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 43/28 ECSubWriteReply(tid=123, last_complete=43'44, committed=1, applied=1)) v2 -- 0x558c5795ca00 con 0x558c54c5bc00 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2275 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2275 < min_flush_size 524288 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d268f~8e3 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d268f 0x1d268f~8e3 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a268f 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x68f 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf72 and padding block with 0x8e 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d2f72 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000140438s (0.000000102s flush + 0.000140336s kv commit) 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b94c00,0x558c57db3b00> 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 kv_submitted 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b94c00 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b94c00 finishing 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b94c00 onodes 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b94c00 done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b94c00 [] 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3b00 kv_submitted 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db3b00 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3b00 finishing 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db3b00 onodes 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db3b00 done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db3b00 [] 2019-12-07T19:34:20.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 128 ==== MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=57, reqid=osd.2.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=43'10)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e0800 con 0x558c54c19800 2019-12-07T19:34:20.623+0000 7f8f24c58700 15 osd.7 43 enqueue_op 0x558c57335340 prio 127 cost 0 latency 0.000017 epoch 43 MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=57, reqid=osd.2.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=43'10)) v2 2019-12-07T19:34:20.623+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=57, reqid=osd.2.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=43'10)) v2) prio 127 cost 0 e43) 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=57, reqid=osd.2.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=43'10)) v2) prio 127 cost 0 e43) queued 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=57, reqid=osd.2.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=43'10)) v2) prio 127 cost 0 e43) pg 0x558c55141000 2019-12-07T19:34:20.623+0000 7f8f06185700 10 osd.7 43 dequeue_op 0x558c57335340 prio 127 cost 0 latency 0.000041 MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=57, reqid=osd.2.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=43'10)) v2 pg pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] 2019-12-07T19:34:20.623+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] _handle_message: MOSDECSubOpWrite(1.ds2 43/41 ECSubWrite(tid=57, reqid=osd.2.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=43'10)) v2 2019-12-07T19:34:20.623+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=42'8 active mbc={} ps=[a~1,d~1,10~1]] append_log log((0'0,43'10], crt=42'8) [] 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'9 (42'7) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:20.622200+0000 0 snaps [c,e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~1]] rollforward: entry=43'10 (42'8) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:20.622200+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:20.623+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~1]] append_log approx pg log length = 10 2019-12-07T19:34:20.623+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~1]] append_log transaction_applied = 1 2019-12-07T19:34:20.623+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:20.623+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:20.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:20.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5740af00 seq 37 2019-12-07T19:34:20.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:20.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:20.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:20.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:20.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:20.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740af00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:20.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740af00 onodes shared_blobs 2019-12-07T19:34:20.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740af00 allocated 0x[] released 0x[] 2019-12-07T19:34:20.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 prepare 2019-12-07T19:34:20.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740af00 2019-12-07T19:34:20.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 io_done 2019-12-07T19:34:20.623+0000 7f8f06185700 10 osd.7 43 dequeue_op 0x558c57335340 finish 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:20.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d2f72~471 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d2f72 0x1d2f72~471 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a2f72 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf72 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3e3 and padding block with 0xc1d 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d33e3 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122880s (0.000000083s flush + 0.000122797s kv commit) 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740af00> 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 kv_submitted 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740af00 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740af00 finishing 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740af00 onodes 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740af00 done 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740af00 [] 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.624+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 43/41 ECSubWriteReply(tid=57, last_complete=43'10, committed=1, applied=1)) v2 -- 0x558c5841e800 con 0x558c54c19800 2019-12-07T19:34:20.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 129 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e43/43) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57ed2000 con 0x558c54c19800 2019-12-07T19:34:20.624+0000 7f8f24c58700 15 osd.7 43 enqueue_peering_evt 1.ds2 epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.ds0( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:20.624+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.ds0( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e43) 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.ds0( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e43) queued 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.ds0( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e43) pg 0x558c55141000 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~1]] do_peering_event: epoch_sent: 43 epoch_requested: 43 MInfoRec from 2(0) info: 1.ds0( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~1]] proc_primary_info updating purged_snaps to [a~1,d~1,10~2] 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57a56600 seq 38 2019-12-07T19:34:20.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a56600 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a56600 onodes shared_blobs 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a56600 allocated 0x[] released 0x[] 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56600 prepare 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a56600 2019-12-07T19:34:20.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56600 io_done 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d33e3~47d to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d33e3 0x1d33e3~47d to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a33e3 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3e3 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x860 and padding block with 0x7a0 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d3860 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:20.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000106489s (0.000000070s flush + 0.000106419s kv commit) 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:20.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a56600> 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56600 kv_submitted 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a56600 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56600 finishing 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a56600 onodes 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a56600 done 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a56600 [] 2019-12-07T19:34:20.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:20.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:20.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:20.660+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 10 osd.7 43 tick_without_osd_lock 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f6be000/0x40000000/0x1680000000, data 0xa0ff37/0x940000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 25 osd.7 43 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674802+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 25 osd.7 43 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674781+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 25 osd.7 43 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674735+0000 last_rx_front 2019-12-07T19:34:15.674747+0000 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 25 osd.7 43 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674761+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 25 osd.7 43 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:15.674373+0000 last_rx_back 2019-12-07T19:34:15.674643+0000 last_rx_front 2019-12-07T19:34:15.674689+0000 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 20 osd.7 43 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 10 osd.7 43 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 20 osd.7 43 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:20.681+0000 7f8f1fbbe700 10 osd.7 43 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:20.711+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.761+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.811+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.861+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.911+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.961+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:20.974+0000 7f8f0097a700 5 osd.7 43 heartbeat osd_stat(store_statfs(0x163f6be000/0x40000000/0x1680000000, data 0xa0ff37/0x940000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5] op hist []) 2019-12-07T19:34:20.974+0000 7f8f0097a700 20 osd.7 43 check_full_status cur ratio 0.000100454, physical ratio 0.000100454, new state none 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.688301013s) v5 -- 0x558c57612a00 con 0x558c54c9e000 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.688301013s) v5 -- 0x558c57612c00 con 0x558c54c9e400 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.673568337s) v5 -- 0x558c57612e00 con 0x558c551b7800 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.673568337s) v5 -- 0x558c57613000 con 0x558c551b7c00 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.687671097s) v5 -- 0x558c57613200 con 0x558c552c2000 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.687671097s) v5 -- 0x558c57613400 con 0x558c552c2400 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.696048749s) v5 -- 0x558c57613600 con 0x558c552c2800 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.696048749s) v5 -- 0x558c57613800 con 0x558c552c2c00 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.679054303s) v5 -- 0x558c57613a00 con 0x558c54c65800 2019-12-07T19:34:20.974+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.410125924s delta_ub 18446744073.679054303s) v5 -- 0x558c57613c00 con 0x558c54c65c00 2019-12-07T19:34:20.974+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 8 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.431842309s delta_ub 0.021716385s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795ca00 con 0x558c54c9e000 2019-12-07T19:34:20.974+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 12 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.424082241s delta_ub 0.013956317s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e400 con 0x558c552c2800 2019-12-07T19:34:20.974+0000 7f8f23c56700 25 osd.7 43 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:15.674802+0000 -> 2019-12-07T19:34:20.974946+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 2019-12-07T19:34:20.974+0000 7f8f23c56700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021457104s,0.021716385s]) 2019-12-07T19:34:20.974+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 12 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.424142006s delta_ub 0.014016082s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55271c00 con 0x558c552c2c00 2019-12-07T19:34:20.974+0000 7f8f24c58700 25 osd.7 43 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:15.674761+0000 -> 2019-12-07T19:34:20.974957+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 2019-12-07T19:34:20.974+0000 7f8f24c58700 10 osd.7 43 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 42 -> 43 (as per caller) 2019-12-07T19:34:20.974+0000 7f8f24c58700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013685431s,0.013956317s]) 2019-12-07T19:34:20.974+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 12 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.446676338s delta_ub 0.036550414s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795ca00 con 0x558c551b7800 2019-12-07T19:34:20.974+0000 7f8f23c56700 25 osd.7 43 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:15.674781+0000 -> 2019-12-07T19:34:20.974976+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 2019-12-07T19:34:20.974+0000 7f8f23c56700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036260933s,0.036550414s]) 2019-12-07T19:34:20.974+0000 7f8f24457700 25 osd.7 43 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.974957+0000 last_rx_front 2019-12-07T19:34:15.674707+0000 -> 2019-12-07T19:34:20.974987+0000 2019-12-07T19:34:20.974+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 12 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.446653963s delta_ub 0.036528039s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e400 con 0x558c551b7c00 2019-12-07T19:34:20.974+0000 7f8f24457700 25 osd.7 43 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:20.974686+0000) and older pending ping(s) 2019-12-07T19:34:20.974+0000 7f8f24457700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013714831s,0.014016082s]) 2019-12-07T19:34:20.974+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 8 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.441206305s delta_ub 0.031080381s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795ca00 con 0x558c54c65800 2019-12-07T19:34:20.974+0000 7f8f23c56700 25 osd.7 43 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:15.674643+0000 -> 2019-12-07T19:34:20.975007+0000 last_rx_front 2019-12-07T19:34:15.674689+0000 2019-12-07T19:34:20.974+0000 7f8f23c56700 10 osd.7 43 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 42 -> 43 (as per caller) 2019-12-07T19:34:20.974+0000 7f8f23c56700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030759671s,0.031080381s]) 2019-12-07T19:34:20.974+0000 7f8f24c58700 25 osd.7 43 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.974976+0000 last_rx_front 2019-12-07T19:34:15.674669+0000 -> 2019-12-07T19:34:20.975018+0000 2019-12-07T19:34:20.974+0000 7f8f24c58700 25 osd.7 43 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:20.974686+0000) and older pending ping(s) 2019-12-07T19:34:20.974+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 12 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.432529959s delta_ub 0.022404035s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55271c00 con 0x558c552c2000 2019-12-07T19:34:20.974+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 8 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.441229521s delta_ub 0.031103597s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795ca00 con 0x558c54c65c00 2019-12-07T19:34:20.974+0000 7f8f24c58700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036196420s,0.036528039s]) 2019-12-07T19:34:20.974+0000 7f8f24457700 25 osd.7 43 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:15.674735+0000 -> 2019-12-07T19:34:20.975037+0000 last_rx_front 2019-12-07T19:34:15.674747+0000 2019-12-07T19:34:20.974+0000 7f8f24457700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022053700s,0.022404035s]) 2019-12-07T19:34:20.974+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 8 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.431873716s delta_ub 0.021747792s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e400 con 0x558c54c9e400 2019-12-07T19:34:20.974+0000 7f8f23c56700 25 osd.7 43 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.975007+0000 last_rx_front 2019-12-07T19:34:15.674689+0000 -> 2019-12-07T19:34:20.975049+0000 2019-12-07T19:34:20.974+0000 7f8f23c56700 25 osd.7 43 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:20.974686+0000) and older pending ping(s) 2019-12-07T19:34:20.974+0000 7f8f23c56700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030740732s,0.031103597s]) 2019-12-07T19:34:20.974+0000 7f8f24c58700 25 osd.7 43 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.974946+0000 last_rx_front 2019-12-07T19:34:15.674721+0000 -> 2019-12-07T19:34:20.975066+0000 2019-12-07T19:34:20.974+0000 7f8f24c58700 25 osd.7 43 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:20.974686+0000) and older pending ping(s) 2019-12-07T19:34:20.974+0000 7f8f24c58700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021368534s,0.021747792s]) 2019-12-07T19:34:20.974+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 12 ==== osd_ping(ping_reply e43 up_from 13 ping_stamp 2019-12-07T19:34:20.974686+0000/44.410125924s send_stamp 44.432490448s delta_ub 0.022364524s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795ca00 con 0x558c552c2400 2019-12-07T19:34:20.974+0000 7f8f23c56700 25 osd.7 43 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.975037+0000 last_rx_front 2019-12-07T19:34:15.674747+0000 -> 2019-12-07T19:34:20.975080+0000 2019-12-07T19:34:20.974+0000 7f8f23c56700 25 osd.7 43 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:20.974686+0000) and older pending ping(s) 2019-12-07T19:34:20.974+0000 7f8f23c56700 20 osd.7 43 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021970645s,0.022364524s]) 2019-12-07T19:34:21.012+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.062+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.112+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.162+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.212+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.224+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:21.262+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.312+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.362+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.364+0000 7f8f21451700 10 osd.7 43 tick 2019-12-07T19:34:21.364+0000 7f8f21451700 10 osd.7 43 do_waiters -- start 2019-12-07T19:34:21.364+0000 7f8f21451700 10 osd.7 43 do_waiters -- finish 2019-12-07T19:34:21.364+0000 7f8f21451700 20 osd.7 43 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:21.412+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.462+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.513+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.563+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.574+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:21.613+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1089561 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.618+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 207 ==== osd_map(44..44 src has 1..44) v4 ==== 633+0+0 (crc 0 0 0) 0x558c573e9b00 con 0x558c54c5b400 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 osd.7 43 OSD::ms_dispatch: osd_map(44..44 src has 1..44) v4 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 osd.7 43 do_waiters -- start 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 osd.7 43 do_waiters -- finish 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 osd.7 43 _dispatch 0x558c573e9b00 osd_map(44..44 src has 1..44) v4 2019-12-07T19:34:21.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 208 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=73, reqid=client.4540.0:354, at_version=44'14, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 3183+0+0 (crc 0 0 0) 0x558c57a45800 con 0x558c54c5b400 2019-12-07T19:34:21.618+0000 7f8f24457700 15 osd.7 43 enqueue_op 0x558c57eff1e0 prio 127 cost 0 latency 0.000040 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=73, reqid=client.4540.0:354, at_version=44'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.618+0000 7f8f1199c700 3 osd.7 43 handle_osd_map epochs [44,44], i have 43, src has [1,44] 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 osd.7 43 handle_osd_map got inc map for epoch 44 2019-12-07T19:34:21.618+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=73, reqid=client.4540.0:354, at_version=44'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3b68e826:::osdmap.43:0# 0x0~0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c38d90 blob([0x40b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c38e00 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c38e70 blob([!~1000,0x40c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57c38ee0 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3b68e826:::osdmap.43:0# 0x0~1464 = 5220 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 osd.7 43 add_map_bl 43 5220 bytes 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 osd.7 43 got_full_map 44, nothing requested 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 osd.7 43 handle_osd_map pg_num_history pg_num_history(e44 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {44={}} 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 osd.7 43 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e44 [1,44] lci=[13,44]) 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c57a56000 seq 35 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c977972f:::inc_osdmap.44:0# key 0x7f7fffffffffffffffc97797'/!inc_osdmap.44!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:21.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:21.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=73, reqid=client.4540.0:354, at_version=44'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c977972f:::inc_osdmap.44:0# 0x0~24d 2019-12-07T19:34:21.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1217 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c977972f:::inc_osdmap.44:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:21.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=73, reqid=client.4540.0:354, at_version=44'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57a56000 1 blobs 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x40d000~1000/1000,1000,0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x40d000~1000] 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f51880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f51880 blob([0x40d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d810 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c57f51880) get_ref 0x0~24d Blob(0x558c57f51880 blob([0x40d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d810 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c57f51880) get_ref init 0x1000, 1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c57f51880 blob([0x40d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c54d3d810 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54d3d828 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:21.618+0000 7f8f07187700 10 osd.7 43 dequeue_op 0x558c57eff1e0 prio 127 cost 0 latency 0.000222 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=73, reqid=client.4540.0:354, at_version=44'14, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:21.618+0000 7f8f07187700 10 osd.7 43 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 43 -> 44 (as per caller) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9a108) dirty_range mark inline shard dirty 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c977972f:::inc_osdmap.44:0# 0x0~24d = 0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3668e826:::osdmap.44:0# key 0x7f7fffffffffffffff3668e8'&!osdmap.44!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:21.618+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=73, reqid=client.4540.0:354, at_version=44'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:21.618+0000 7f8f07187700 20 snap_mapper.add_oid 1:dce63cc3:::smithi13140796-50:15 c,e,13,15 2019-12-07T19:34:21.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3668e826:::osdmap.44:0# 0x0~1448 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1218 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3668e826:::osdmap.44:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:21.618+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57a56000 2 blobs 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x40e000~2000/1000,2000,0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x40e000~2000] 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574faa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574faa10 blob([0x40e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb9d0 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c574faa10) get_ref 0x0~1000 Blob(0x558c574faa10 blob([0x40e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb9d0 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c574faa10) get_ref init 0x1000, 1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c574faa10 blob([0x40e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57dcb9d0 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57dcb9e8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55279340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac070 blob([!~1000,0x40f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279340 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c576ac070) get_ref 0x1000~448 Blob(0x558c576ac070 blob([!~1000,0x40f000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c55279340 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c576ac070) get_ref init 0x2000, 1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c576ac070 blob([!~1000,0x40f000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c55279340 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55279358 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9ab08) dirty_range mark inline shard dirty 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3668e826:::osdmap.44:0# 0x0~1448 = 0 2019-12-07T19:34:21.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57a56000 0 blobs 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:21.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:21.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:21.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:21.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57a56000 0 blobs 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a56000 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a56000 onodes 0x558c53a91900,0x558c55276c80,0x558c57d9a000,0x558c57d9aa00 shared_blobs 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9a108) update #-1:c977972f:::inc_osdmap.44:0# 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9a108) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c977972f:::inc_osdmap.44:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9ab08) update #-1:3668e826:::osdmap.44:0# 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9ab08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3668e826:::osdmap.44:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a56000 allocated 0x[40d000~3000] released 0x[] 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 freelist allocate 0x40d000~3000 2019-12-07T19:34:21.619+0000 7f8f1199c700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56000 prepare 2019-12-07T19:34:21.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57a56000 2019-12-07T19:34:21.619+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56000 aio_wait 2019-12-07T19:34:21.619+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a56000 2019-12-07T19:34:21.619+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56000 io_done 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57d9a000 had 1 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57d9aa00 had 1 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3858890752 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998288) bluestore 86 GiB free (0.959899), bluefs_ratio 0.0399387 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d3860~226c to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d3860 0x1d3860~226c to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a3860 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x860 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xacc and padding block with 0x534 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d5acc 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000164805s (0.000004925s flush + 0.000159880s kv commit) 2019-12-07T19:34:21.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a56000> 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56000 kv_submitted 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a56000 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c57a56000 osr 0x558c54ea4b40 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 32 0x1b000~1000 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7 43 _committed_osd_maps 44..44 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 33 0x1b000~1000 crc 3000d211 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7 43 advance to epoch 44 (<= last 44 <= newest_map 44) 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 32 0x2000~1000 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 33 0x2000~1000 crc 73ebbf02 2019-12-07T19:34:21.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.619+0000 7f8f141a1700 7 osd.7 44 consume_map version 44 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.8s5 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.0s3 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.1s1 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.9s4 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.as5 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.2s0 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.bs1 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.3s4 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.cs0 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.4s0 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.5s4 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.ds2 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.es4 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.6s2 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.fs5 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 identify_splits_and_merges 1.7s4 e43 to e44 pg_nums {18=16} 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 44 prune_pg_created 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:0.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:1.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:2.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:3.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:4.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:5.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:6.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7:7.consume_map 44 (was 43) 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:21.619+0000 7f8f141a1700 10 osd.7 44 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.8s5 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.0s3 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.1s1 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.9s4 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.as5 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.2s0 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.bs1 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.3s4 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.cs0 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c57380000 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.4s0 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.5s4 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.ds2 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.es4 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.6s2 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.fs5 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 15 osd.7 44 enqueue_peering_evt 1.7s4 epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:21.619+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) 2019-12-07T19:34:21.619+0000 7f8f141a1700 7 osd.7 44 activate_map version 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c57383000 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c57522000 2019-12-07T19:34:21.619+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f0918b700 20 osd.7 44 agent_entry empty queue 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f07187700 20 snap_mapper.get_snaps 1:dce63cc3:::smithi13140796-50:15 got.empty() 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f07187700 20 snap_mapper.set_snaps 1:dce63cc3:::smithi13140796-50:15 c,e,13,15 2019-12-07T19:34:21.619+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:21.619+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c5500d000 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.619+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.619+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 43 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,21'12], crt=21'12) [44'13 (21'9) clone 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:33:58.792599+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,44'14 (21'9) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:354 2019-12-07T19:34:21.618888+0000 0 ObjectCleanRegions clean_offsets: [0~835584,835584~18446744073708716031], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c54d5f000 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 44'13 (0'0,44'13] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'13 (21'9) clone 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:33:58.792599+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c55141000 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'14 (21'9) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:354 2019-12-07T19:34:21.618888+0000 0 ObjectCleanRegions clean_offsets: [0~835584,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 43 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:21.619+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 14 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:21.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 43 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c57676000 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c580d3500 seq 16 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:15# key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0x0000000000000015ffffffffffffffff'o' 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] state: Active advmap 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f06185700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:21.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# -> 1#1:dce63cc3:::smithi13140796-50:15# 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1219 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.onode(0x558c57f34c80).flush flush done 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 0x0 2019-12-07T19:34:21.619+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:21.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# -> 1#1:dce63cc3:::smithi13140796-50:15# 0x0~33000 -> 0x0~33000 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x0~2000: 0x0~2000 Blob(0x558c576ad650 blob([0x652000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10502 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c576ad650 blob([0x652000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c576ad650 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c574f1e30 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c574f1e30) get_ref 0x0~2000 Blob(0x558c574f1e30 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c574f1e30) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574f1e30 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57897c70 blob([0x654000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10503 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57897c70 blob([0x654000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57897c70 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] state: Active: handling ActMap 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] Active: kicking snap trim 2019-12-07T19:34:21.619+0000 7f8f0297e700 7 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] state: handle ActMap primary 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c574f1180 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c574f1180) get_ref 0x0~2000 Blob(0x558c574f1180 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c574f1180) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c574f1180 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c574fd960 blob([0x656000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10504 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574fd960 blob([0x656000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574fd960 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c550b2d90 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c550b2d90) get_ref 0x0~2000 Blob(0x558c550b2d90 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c550b2d90) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f0297e700 15 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:133 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c550b2d90 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c574fd570 blob([0x658000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10505 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574fd570 blob([0x658000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574fd570 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c55136c40 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c55136c40) get_ref 0x0~2000 Blob(0x558c55136c40 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c55136c40) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c55136c40 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c577f3570 blob([0x65a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10506 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c548ab000 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c577f3570 blob([0x65a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c577f3570 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3b90 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b90) get_ref 0x0~2000 Blob(0x558c579b3b90 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b90) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c579b3b90 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c578f1810 blob([0x65c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10507 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c578f1810 blob([0x65c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c578f1810 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 43 -> 44 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c57970770 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c57970770) get_ref 0x0~2000 Blob(0x558c57970770 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c57970770) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 43 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57970770 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c54677810 blob([0x65e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10508 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c54677810 blob([0x65e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] state: Active advmap 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c54677810 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c576ad8f0 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c576ad8f0) get_ref 0x0~2000 Blob(0x558c576ad8f0 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c576ad8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c576ad8f0 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([0x660000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10509 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574d2fc0 blob([0x660000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574d2fc0 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3ab0 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c579b3ab0) get_ref 0x0~2000 Blob(0x558c579b3ab0 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c579b3ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c579b3ab0 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c552790a0 blob([0x662000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10510 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c552790a0 blob([0x662000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.619+0000 7f8f0297e700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c552790a0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=1)))) 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3b20 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b20) get_ref 0x0~2000 Blob(0x558c579b3b20 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.blob(0x558c579b3b20) get_ref init 0x2000, 1000 2019-12-07T19:34:21.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] state: Active: handling ActMap 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c579b3b20 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) 2019-12-07T19:34:21.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] Active: kicking snap trim 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57567110 blob([0x664000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c57e8b000 2019-12-07T19:34:21.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10511 2019-12-07T19:34:21.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57567110 blob([0x664000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 sbid 0x0)) 2019-12-07T19:34:21.619+0000 7f8f0297e700 7 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] state: handle ActMap primary 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57567110 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c57970700 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0297e700 15 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:105 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970700) get_ref 0x0~2000 Blob(0x558c57970700 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970700) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57970700 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x14000~2000: 0x0~2000 Blob(0x558c577f3490 blob([0x666000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10512 2019-12-07T19:34:21.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c577f3490 blob([0x666000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c577f3490 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c576ad880 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c576ad880) get_ref 0x0~2000 Blob(0x558c576ad880 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c576ad880) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c54d98000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c576ad880 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57b97f80 blob([0x668000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10513 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57b97f80 blob([0x668000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57b97f80 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b39d0 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b39d0) get_ref 0x0~2000 Blob(0x558c579b39d0 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c579b39d0 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x18000~2000: 0x0~2000 Blob(0x558c581ea000 blob([0x66a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10514 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea000 blob([0x66a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea000 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3a40 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3a40) get_ref 0x0~2000 Blob(0x558c579b3a40 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3a40) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c579b3a40 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c581ea0e0 blob([0x66c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10515 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea0e0 blob([0x66c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea0e0 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c57970690 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970690) get_ref 0x0~2000 Blob(0x558c57970690 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970690) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c54d5c000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57970690 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 209 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=74, reqid=client.4540.0:355, at_version=44'15, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2140+0+0 (crc 0 0 0) 0x558c57e01800 con 0x558c54c5b400 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c581ea1c0 blob([0x66e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10516 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea1c0 blob([0x66e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea1c0 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57eff760 prio 127 cost 0 latency 0.000028 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=74, reqid=client.4540.0:355, at_version=44'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.620+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c57818c40 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=74, reqid=client.4540.0:355, at_version=44'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57818c40) get_ref 0x0~2000 Blob(0x558c57818c40 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 43 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57818c40) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57818c40 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c581ea2a0 blob([0x670000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10517 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea2a0 blob([0x670000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea2a0 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b38f0 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b38f0) get_ref 0x0~2000 Blob(0x558c579b38f0 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b38f0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c579b38f0 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x20000~2000: 0x0~2000 Blob(0x558c581ea380 blob([0x672000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10518 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea380 blob([0x672000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea380 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3960 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3960) get_ref 0x0~2000 Blob(0x558c579b3960 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3960) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c579b3960 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x22000~2000: 0x0~2000 Blob(0x558c581ea460 blob([0x674000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10519 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea460 blob([0x674000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea460 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c57970620 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970620) get_ref 0x0~2000 Blob(0x558c57970620 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970620) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c57970620 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x24000~2000: 0x0~2000 Blob(0x558c581ea540 blob([0x676000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10520 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea540 blob([0x676000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea540 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b2000 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b2000) get_ref 0x0~2000 Blob(0x558c579b2000 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b2000) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c579b2000 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x26000~2000: 0x0~2000 Blob(0x558c581ea620 blob([0x678000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10521 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea620 blob([0x678000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea620 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c57e88000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3810 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3810) get_ref 0x0~2000 Blob(0x558c579b3810 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3810) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c579b3810 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x28000~2000: 0x0~2000 Blob(0x558c581ea700 blob([0x67a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10522 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea700 blob([0x67a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea700 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s0 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3880 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3880) get_ref 0x0~2000 Blob(0x558c579b3880 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3880) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c579b3880 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 43 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c581ea7e0 blob([0x67c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10523 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea7e0 blob([0x67c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea7e0 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] state: Active advmap 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579705b0 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579705b0) get_ref 0x0~2000 Blob(0x558c579705b0 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579705b0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c579705b0 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c581ea8c0 blob([0x67e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10524 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea8c0 blob([0x67e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea8c0 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c576ad9d0 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c576ad9d0) get_ref 0x0~2000 Blob(0x558c576ad9d0 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c576ad9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c576ad9d0 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c581ea9a0 blob([0x680000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10525 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581ea9a0 blob([0x680000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581ea9a0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b3730 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3730) get_ref 0x0~2000 Blob(0x558c579b3730 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b3730) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c579b3730 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x30000~2000: 0x0~2000 Blob(0x558c581eaa80 blob([0x682000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10526 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581eaa80 blob([0x682000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581eaa80 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] state: Active: handling ActMap 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c579b37a0 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b37a0) get_ref 0x0~2000 Blob(0x558c579b37a0 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c579b37a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] Active: kicking snap trim 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c579b37a0 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup src 0x32000~1000: 0x0~1000 Blob(0x558c581eab60 blob([0x684000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10527 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c581eab60 blob([0x684000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 sbid 0x0)) 2019-12-07T19:34:21.620+0000 7f8f07988700 7 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] state: handle ActMap primary 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c581eab60 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=1)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup new Blob(0x558c57970540 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970540) get_ref 0x0~1000 Blob(0x558c57970540 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.blob(0x558c57970540) get_ref init 0x1000, 1000 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dup dst 0x32000~1000: 0x0~1000 Blob(0x558c57970540 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dirty_range mark inline shard dirty 2019-12-07T19:34:21.620+0000 7f8f07988700 15 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:121 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# -> 1#1:dce63cc3:::smithi13140796-50:15# = 0 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# snapset 2019-12-07T19:34:21.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# snapset = 0 2019-12-07T19:34:21.620+0000 7f8f07988700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys 2019-12-07T19:34:21.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_000000000000000C_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_000000000000000E_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_0000000000000013_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000013' <- 0000000044.00000000000000000013 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000014' <- 0000000044.00000000000000000014 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d3500 cost 8692 (1 ios * 4000 + 4692 bytes) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d3500 onodes 0x558c574a0a00,0x558c57f34c80 shared_blobs 0x558c55279810,0x558c5742af50,0x558c5742bb20,0x558c574fd500,0x558c574fd6c0,0x558c577f3880,0x558c57819dc0,0x558c57881810,0x558c57a073b0,0x558c57b97f10,0x558c57e63340,0x558c57efc070,0x558c581ea070,0x558c581ea150,0x558c581ea230,0x558c581ea310,0x558c581ea3f0,0x558c581ea4d0,0x558c581ea5b0,0x558c581ea690,0x558c581ea770,0x558c581ea850,0x558c581ea930,0x558c581eaa10,0x558c581eaaf0,0x558c581eabd0 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) update 1#1:dce63cc3:::smithi13140796-50:15# 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) update inline shard 702 bytes from 26 extents 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:15# is 1120 (412 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) update inline shard 702 bytes from 26 extents 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 1289 (581 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2907 is 11 SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290e is 11 SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290f is 11 SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2909 is 11 SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290d is 11 SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2910 is 11 SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2906 is 11 SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290a is 11 SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2908 is 11 SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2911 is 11 SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290c is 11 SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290b is 11 SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2912 is 11 SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2913 is 11 SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2914 is 11 SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2915 is 11 SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2916 is 11 SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2917 is 11 SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2918 is 11 SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2919 is 11 SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291a is 11 SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291b is 11 SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291c is 11 SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291d is 11 SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291e is 11 SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291f is 11 SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2))) 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d3500 allocated 0x[] released 0x[] 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3500 prepare 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d3500 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3500 io_done 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'14 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c57eff1e0 finish 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c575ef000 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c5513e000 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 43 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=74, reqid=client.4540.0:355, at_version=44'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=74, reqid=client.4540.0:355, at_version=44'15, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c57eff760 prio 127 cost 0 latency 0.000599 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=74, reqid=client.4540.0:355, at_version=44'15, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=74, reqid=client.4540.0:355, at_version=44'15, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'14 (0'0,44'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'14], crt=21'12) [44'15 (44'14) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:355 2019-12-07T19:34:21.618911+0000 0 ObjectCleanRegions clean_offsets: [835584~18446744073708716031], clean_omap: 0, new_object: 0] 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'15 (0'0,44'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'14 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'15 (44'14) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:355 2019-12-07T19:34:21.618911+0000 0 ObjectCleanRegions clean_offsets: [835584~18446744073708716031], clean_omap: 0, new_object: 0 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'15 (0'0,44'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'14 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 15 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c5736a000 2019-12-07T19:34:21.620+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'15 (0'0,44'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'14 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.620+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.620+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'15, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5730b800 seq 17 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:head#f key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffe000000000000000f'o' 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# -> 1#1:dce63cc3:::smithi13140796-50:head#f 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:21.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# -> 1#1:dce63cc3:::smithi13140796-50:head#f = 0 2019-12-07T19:34:21.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# -> 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1220 2019-12-07T19:34:21.620+0000 7f8f07187700 20 bluestore.onode(0x558c574a0a00).flush flush cnt:1 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) queued 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 44 epoch_requested: 44 NullEvt) prio 255 cost 10 e44) pg 0x558c5500a000 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 44 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 43 -> 44 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 43 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6100 < min_flush_size 524288 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:21.620+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 44 advance_pg new pool opts old pool opts 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34c80 had 1 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 44 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6123 < min_flush_size 524288 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6123 < min_flush_size 524288 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 44 2019-12-07T19:34:21.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d5acc~17eb to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d5acc 0x1d5acc~17eb to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.620+0000 7f8f04181700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 44 epoch_requested: 44 NullEvt 2019-12-07T19:34:21.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a5acc 2019-12-07T19:34:21.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xacc 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2b7 and padding block with 0xd49 2019-12-07T19:34:21.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d72b7 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000143932s (0.000000130s flush + 0.000143802s kv commit) 2019-12-07T19:34:21.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d3500> 2019-12-07T19:34:21.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3500 kv_submitted 2019-12-07T19:34:21.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d3500 2019-12-07T19:34:21.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3500 finishing 2019-12-07T19:34:21.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d3500 onodes 0x558c574a0a00,0x558c57f34c80 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d3500 done 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 prepare 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d3500 [] 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.onode(0x558c574a0a00).flush flush done 2019-12-07T19:34:21.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0 2019-12-07T19:34:21.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# -> 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 -> 0x0~33000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x0~2000: 0x0~2000 Blob(0x558c574f1e30 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c550b2f50 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c550b2f50) get_ref 0x0~2000 Blob(0x558c550b2f50 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c550b2f50) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c550b2f50 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c574f1180 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57897ea0 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57897ea0) get_ref 0x0~2000 Blob(0x558c57897ea0 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57897ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57897ea0 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c550b2d90 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57566cb0 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57566cb0) get_ref 0x0~2000 Blob(0x558c57566cb0 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57566cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57566cb0 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c55136c40 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c574f1dc0 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c574f1dc0) get_ref 0x0~2000 Blob(0x558c574f1dc0 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c574f1dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c574f1dc0 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c579b3b90 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c550b2cb0 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c550b2cb0) get_ref 0x0~2000 Blob(0x558c550b2cb0 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c550b2cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c550b2cb0 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57970770 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c550b2d20 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c550b2d20) get_ref 0x0~2000 Blob(0x558c550b2d20 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c550b2d20) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c550b2d20 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c576ad8f0 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c576ad960 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad960) get_ref 0x0~2000 Blob(0x558c576ad960 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ad960) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c576ad960 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c579b3ab0 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b3650 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3650) get_ref 0x0~2000 Blob(0x558c579b3650 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3650) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c579b3650 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c579b3b20 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b36c0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b36c0) get_ref 0x0~2000 Blob(0x558c579b36c0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b36c0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c579b36c0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57970700 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579704d0 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579704d0) get_ref 0x0~2000 Blob(0x558c579704d0 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579704d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c579704d0 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c576ad880 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57971030 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57971030) get_ref 0x0~2000 Blob(0x558c57971030 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57971030) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57971030 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c579b39d0 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b3570 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3570) get_ref 0x0~2000 Blob(0x558c579b3570 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3570) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c579b3570 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c579b3a40 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b35e0 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b35e0) get_ref 0x0~2000 Blob(0x558c579b35e0 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b35e0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c579b35e0 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57970690 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57970460 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970460) get_ref 0x0~2000 Blob(0x558c57970460 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970460) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57970460 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57818c40 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b2070 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2070) get_ref 0x0~2000 Blob(0x558c579b2070 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b2070) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c579b2070 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c579b38f0 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b3490 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3490) get_ref 0x0~2000 Blob(0x558c579b3490 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3490) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c579b3490 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x20000~2000: 0x0~2000 Blob(0x558c579b3960 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b3500 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3500) get_ref 0x0~2000 Blob(0x558c579b3500 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3500) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c579b3500 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57970620 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579703f0 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579703f0) get_ref 0x0~2000 Blob(0x558c579703f0 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579703f0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c579703f0 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x24000~2000: 0x0~2000 Blob(0x558c579b2000 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c576adab0 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adab0) get_ref 0x0~2000 Blob(0x558c576adab0 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c576adab0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c576adab0 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x26000~2000: 0x0~2000 Blob(0x558c579b3810 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b33b0 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b33b0) get_ref 0x0~2000 Blob(0x558c579b33b0 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b33b0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c579b33b0 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x28000~2000: 0x0~2000 Blob(0x558c579b3880 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b3420 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3420) get_ref 0x0~2000 Blob(0x558c579b3420 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3420) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c579b3420 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c579705b0 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57970380 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970380) get_ref 0x0~2000 Blob(0x558c57970380 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970380) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c57970380 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c576ad9d0 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c576ada40 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ada40) get_ref 0x0~2000 Blob(0x558c576ada40 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c576ada40) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c576ada40 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c579b3730 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b32d0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b32d0) get_ref 0x0~2000 Blob(0x558c579b32d0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b32d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c579b32d0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x30000~2000: 0x0~2000 Blob(0x558c579b37a0 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c579b3340 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3340) get_ref 0x0~2000 Blob(0x558c579b3340 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c579b3340) get_ref init 0x2000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c579b3340 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup src 0x32000~1000: 0x0~1000 Blob(0x558c57970540 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup new Blob(0x558c57970310 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970310) get_ref 0x0~1000 Blob(0x558c57970310 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.blob(0x558c57970310) get_ref init 0x1000, 1000 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) dup dst 0x32000~1000: 0x0~1000 Blob(0x558c57970310 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=3)))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0d88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# -> 1#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:34:21.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:34:21.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:34:21.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000015' <- 0000000044.00000000000000000015 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730b800 cost 5808 (1 ios * 4000 + 1808 bytes) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730b800 onodes 0x558c574a0c80,0x558c57f34c80 shared_blobs 0x558c55279810,0x558c5742af50,0x558c5742bb20,0x558c574fd500,0x558c574fd6c0,0x558c577f3880,0x558c57819dc0,0x558c57881810,0x558c57a073b0,0x558c57b97f10,0x558c57e63340,0x558c57efc070,0x558c581ea070,0x558c581ea150,0x558c581ea230,0x558c581ea310,0x558c581ea3f0,0x558c581ea4d0,0x558c581ea5b0,0x558c581ea690,0x558c581ea770,0x558c581ea850,0x558c581ea930,0x558c581eaa10,0x558c581eaaf0,0x558c581eabd0 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0d88) update inline shard 702 bytes from 26 extents 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 1289 (581 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) update 1#1:dce63cc3:::smithi13140796-50:head#f 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head#f is 1289 (581 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2907 is 11 SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290e is 11 SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290f is 11 SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2909 is 11 SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290d is 11 SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2910 is 11 SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2906 is 11 SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290a is 11 SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2908 is 11 SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2911 is 11 SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290c is 11 SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290b is 11 SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2912 is 11 SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2913 is 11 SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2914 is 11 SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2915 is 11 SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2916 is 11 SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2917 is 11 SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2918 is 11 SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2919 is 11 SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291a is 11 SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291b is 11 SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291c is 11 SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291d is 11 SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291e is 11 SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291f is 11 SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=3))) 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730b800 allocated 0x[] released 0x[] 2019-12-07T19:34:21.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 prepare 2019-12-07T19:34:21.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730b800 2019-12-07T19:34:21.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 io_done 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.621+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'15 (0'0,44'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'14 crt=21'12 lcod 21'12 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'15 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3401 < min_flush_size 524288 2019-12-07T19:34:21.621+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c57eff760 finish 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4038 < min_flush_size 524288 2019-12-07T19:34:21.621+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=73, last_complete=44'14, committed=1, applied=1)) v2 -- 0x558c55271c00 con 0x558c54c5b400 2019-12-07T19:34:21.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34c80 had 1 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4061 < min_flush_size 524288 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4061 < min_flush_size 524288 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d72b7~fdd to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d72b7 0x1d72b7~fdd to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a72b7 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2b7 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x294 and padding block with 0xd6c 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d8294 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138234s (0.000000082s flush + 0.000138152s kv commit) 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730b800> 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 kv_submitted 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730b800 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730b800 finishing 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730b800 onodes 0x558c574a0c80,0x558c57f34c80 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730b800 done 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730b800 [] 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.621+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=74, last_complete=44'15, committed=1, applied=1)) v2 -- 0x558c57518800 con 0x558c54c5b400 2019-12-07T19:34:21.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 210 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=75, reqid=client.4540.0:356, at_version=44'16, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2357+0+0 (crc 0 0 0) 0x558c5748d000 con 0x558c54c5b400 2019-12-07T19:34:21.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.621+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57efe000 prio 127 cost 0 latency 0.000025 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=75, reqid=client.4540.0:356, at_version=44'16, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.621+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=75, reqid=client.4540.0:356, at_version=44'16, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=75, reqid=client.4540.0:356, at_version=44'16, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=75, reqid=client.4540.0:356, at_version=44'16, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c57efe000 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=75, reqid=client.4540.0:356, at_version=44'16, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.bs1( v 44'15 (0'0,44'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'15 (0'0,44'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=75, reqid=client.4540.0:356, at_version=44'16, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'15 (0'0,44'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'15], crt=21'12) [44'16 (44'15) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:356 2019-12-07T19:34:21.618915+0000 0 ObjectCleanRegions clean_offsets: [0~835584,835584~18446744073708716031], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'15 crt=21'12 lcod 44'15 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'16 (44'15) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:356 2019-12-07T19:34:21.618915+0000 0 ObjectCleanRegions clean_offsets: [0~835584,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'15 crt=21'12 lcod 44'15 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 16 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'15 crt=21'12 lcod 44'15 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.621+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57ef9500 seq 18 2019-12-07T19:34:21.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:34:21.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000016' <- 0000000044.00000000000000000016 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9500 cost 6007 (1 ios * 4000 + 2007 bytes) 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9500 onodes 0x558c574a0c80 shared_blobs 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a0d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 1289 (581 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9500 allocated 0x[] released 0x[] 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 prepare 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9500 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 io_done 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'15 crt=21'12 lcod 44'15 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'16 2019-12-07T19:34:21.621+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c57efe000 finish 2019-12-07T19:34:21.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2428 < min_flush_size 524288 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2451 < min_flush_size 524288 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2451 < min_flush_size 524288 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d8294~993 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d8294 0x1d8294~993 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a8294 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x294 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc27 and padding block with 0x3d9 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d8c27 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118539s (0.000000091s flush + 0.000118448s kv commit) 2019-12-07T19:34:21.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9500> 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 kv_submitted 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9500 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9500 finishing 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9500 onodes 0x558c574a0c80 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9500 done 2019-12-07T19:34:21.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9500 [] 2019-12-07T19:34:21.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.622+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=75, last_complete=44'16, committed=1, applied=1)) v2 -- 0x558c57519800 con 0x558c54c5b400 2019-12-07T19:34:21.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 211 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=76, reqid=client.4540.0:356, at_version=0'0, trim_to=0'0, roll_forward_to=44'16)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c575be000 con 0x558c54c5b400 2019-12-07T19:34:21.624+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000022 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=76, reqid=client.4540.0:356, at_version=0'0, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.624+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=76, reqid=client.4540.0:356, at_version=0'0, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=76, reqid=client.4540.0:356, at_version=0'0, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=76, reqid=client.4540.0:356, at_version=0'0, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.624+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=76, reqid=client.4540.0:356, at_version=0'0, trim_to=0'0, roll_forward_to=44'16)) v2 pg pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=76, reqid=client.4540.0:356, at_version=0'0, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=21'12 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'16], crt=21'12) [] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'13 (21'9) clone 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:33:58.792599+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'14 (21'9) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:354 2019-12-07T19:34:21.618888+0000 0 ObjectCleanRegions clean_offsets: [0~835584,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'15 (44'14) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:355 2019-12-07T19:34:21.618911+0000 0 ObjectCleanRegions clean_offsets: [835584~18446744073708716031], clean_omap: 0, new_object: 0 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'16 (44'15) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:356 2019-12-07T19:34:21.618915+0000 0 ObjectCleanRegions clean_offsets: [0~835584,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 16 2019-12-07T19:34:21.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.624+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.624+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c580d3800 seq 19 2019-12-07T19:34:21.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head#f onode 0x558c57f34c80 txc 0x558c580d3800 2019-12-07T19:34:21.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head#f 0x0 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c576ad650) put_ref 0x0~2000 Blob(0x558c576ad650 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c57897c70) put_ref 0x0~2000 Blob(0x558c57897c70 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c574fd960) put_ref 0x0~2000 Blob(0x558c574fd960 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c574fd570) put_ref 0x0~2000 Blob(0x558c574fd570 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c577f3570) put_ref 0x0~2000 Blob(0x558c577f3570 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c578f1810) put_ref 0x0~2000 Blob(0x558c578f1810 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c54677810) put_ref 0x0~2000 Blob(0x558c54677810 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c574d2fc0) put_ref 0x0~2000 Blob(0x558c574d2fc0 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c552790a0) put_ref 0x0~2000 Blob(0x558c552790a0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c57567110) put_ref 0x0~2000 Blob(0x558c57567110 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c577f3490) put_ref 0x0~2000 Blob(0x558c577f3490 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c57b97f80) put_ref 0x0~2000 Blob(0x558c57b97f80 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea000) put_ref 0x0~2000 Blob(0x558c581ea000 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea0e0) put_ref 0x0~2000 Blob(0x558c581ea0e0 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea1c0) put_ref 0x0~2000 Blob(0x558c581ea1c0 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea2a0) put_ref 0x0~2000 Blob(0x558c581ea2a0 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea380) put_ref 0x0~2000 Blob(0x558c581ea380 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea460) put_ref 0x0~2000 Blob(0x558c581ea460 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea540) put_ref 0x0~2000 Blob(0x558c581ea540 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea620) put_ref 0x0~2000 Blob(0x558c581ea620 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea700) put_ref 0x0~2000 Blob(0x558c581ea700 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea7e0) put_ref 0x0~2000 Blob(0x558c581ea7e0 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea8c0) put_ref 0x0~2000 Blob(0x558c581ea8c0 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581ea9a0) put_ref 0x0~2000 Blob(0x558c581ea9a0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581eaa80) put_ref 0x0~2000 Blob(0x558c581eaa80 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.blob(0x558c581eab60) put_ref 0x0~1000 Blob(0x558c581eab60 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f34d88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c576ad650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x652000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57897c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x654000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c574fd960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x656000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574fd570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x658000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c577f3570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x65a000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c578f1810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x65c000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c54677810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x65e000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x660000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c552790a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x662000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57567110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x664000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c577f3490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x666000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57b97f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x668000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c581ea000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x66a000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c581ea0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x66c000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c581ea1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x66e000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c581ea2a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x670000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c581ea380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x672000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c581ea460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x674000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c581ea540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x676000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c581ea620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x678000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c581ea700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x67a000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c581ea7e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x67c000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c581ea8c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x67e000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c581ea9a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x680000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c581eaa80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x682000~2000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~1000: 0x0~1000 Blob(0x558c581eab60 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=3)))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x684000~1000] 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head#f = 0 2019-12-07T19:34:21.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d3800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d3800 onodes shared_blobs 0x558c55279810,0x558c5742af50,0x558c5742bb20,0x558c574fd500,0x558c574fd6c0,0x558c577f3880,0x558c57819dc0,0x558c57881810,0x558c57a073b0,0x558c57b97f10,0x558c57e63340,0x558c57efc070,0x558c581ea070,0x558c581ea150,0x558c581ea230,0x558c581ea310,0x558c581ea3f0,0x558c581ea4d0,0x558c581ea5b0,0x558c581ea690,0x558c581ea770,0x558c581ea850,0x558c581ea930,0x558c581eaa10,0x558c581eaaf0,0x558c581eabd0 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2907 is 11 SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290e is 11 SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290f is 11 SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2909 is 11 SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290d is 11 SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2910 is 11 SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2906 is 11 SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290a is 11 SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2908 is 11 SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2911 is 11 SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290c is 11 SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290b is 11 SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2912 is 11 SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2913 is 11 SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2914 is 11 SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2915 is 11 SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2916 is 11 SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2917 is 11 SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2918 is 11 SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2919 is 11 SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291a is 11 SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291b is 11 SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291c is 11 SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291d is 11 SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291e is 11 SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291f is 11 SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2))) 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d3800 allocated 0x[] released 0x[] 2019-12-07T19:34:21.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3800 prepare 2019-12-07T19:34:21.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d3800 2019-12-07T19:34:21.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3800 io_done 2019-12-07T19:34:21.624+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1845 < min_flush_size 524288 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34c80 had 1 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1868 < min_flush_size 524288 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1868 < min_flush_size 524288 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d8c27~74c to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d8c27 0x1d8c27~74c to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a8c27 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc27 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x373 and padding block with 0xc8d 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1d9373 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000128074s (0.000000113s flush + 0.000127961s kv commit) 2019-12-07T19:34:21.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d3800> 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3800 kv_submitted 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d3800 2019-12-07T19:34:21.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3800 finishing 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d3800 onodes 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d3800 done 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d3800 [] 2019-12-07T19:34:21.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.625+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=76, last_complete=44'16, committed=1, applied=1)) v2 -- 0x558c57519600 con 0x558c54c5b400 2019-12-07T19:34:21.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 212 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=77, reqid=client.4540.0:357, at_version=44'18, trim_to=0'0, roll_forward_to=44'16)) v2 ==== 142610+0+0 (crc 0 0 0) 0x558c579e1000 con 0x558c54c5b400 2019-12-07T19:34:21.626+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57efec60 prio 127 cost 0 latency 0.000036 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=77, reqid=client.4540.0:357, at_version=44'18, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.626+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=77, reqid=client.4540.0:357, at_version=44'18, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=77, reqid=client.4540.0:357, at_version=44'18, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=77, reqid=client.4540.0:357, at_version=44'18, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c57efec60 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=77, reqid=client.4540.0:357, at_version=44'18, trim_to=0'0, roll_forward_to=44'16)) v2 pg pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=77, reqid=client.4540.0:357, at_version=44'18, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.add_oid 1:db85c185:::smithi13140796-18:15 c,e,13,15 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.get_snaps 1:db85c185:::smithi13140796-18:15 got.empty() 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.set_snaps 1:db85c185:::smithi13140796-18:15 c,e,13,15 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'16 (0'0,44'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'16], crt=44'16) [44'17 (21'2) clone 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:33:58.659098+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,44'18 (21'2) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:357 2019-12-07T19:34:21.624162+0000 0 ObjectCleanRegions clean_offsets: [0~294912,851968~18446744073708699647], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'17 (0'0,44'17] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'16 crt=44'16 lcod 44'16 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'17 (21'2) clone 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:33:58.659098+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'16 crt=44'16 lcod 44'16 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'18 (21'2) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:357 2019-12-07T19:34:21.624162+0000 0 ObjectCleanRegions clean_offsets: [0~294912,851968~18446744073708699647], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'16 crt=44'16 lcod 44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 18 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'16 crt=44'16 lcod 44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.626+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'17, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57ef9200 seq 20 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:db85c185:::smithi13140796-18:15# key 0x818000000000000001db85c18521736d'ithi13140796-18!='0x0000000000000015ffffffffffffffff'o' 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# -> 1#1:db85c185:::smithi13140796-18:15# 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1221 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.onode(0x558c57e8ec80).flush flush done 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 0x0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# -> 1#1:db85c185:::smithi13140796-18:15# 0x0~12000 -> 0x0~12000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0x0~2000: 0x0~2000 Blob(0x558c552796c0 blob([0x611000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10528 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c552796c0 blob([0x611000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c552796c0 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c55136f50 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55136f50) get_ref 0x0~2000 Blob(0x558c55136f50 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55136f50) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c55136f50 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57f49b90 blob([0x613000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10529 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f49b90 blob([0x613000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f49b90 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c574e2930 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2930) get_ref 0x0~2000 Blob(0x558c574e2930 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2930) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c574e2930 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57a06850 blob([0x615000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10530 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57a06850 blob([0x615000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57a06850 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c550b3110 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3110) get_ref 0x0~2000 Blob(0x558c550b3110 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3110) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c550b3110 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57f49ab0 blob([0x617000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10531 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f49ab0 blob([0x617000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f49ab0 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c55137030 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137030) get_ref 0x0~2000 Blob(0x558c55137030 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137030) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c55137030 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57f499d0 blob([0x619000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10532 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f499d0 blob([0x619000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f499d0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c551370a0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551370a0) get_ref 0x0~2000 Blob(0x558c551370a0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551370a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c551370a0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57f498f0 blob([0x61b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10533 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f498f0 blob([0x61b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f498f0 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c55137180 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137180) get_ref 0x0~2000 Blob(0x558c55137180 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137180) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c55137180 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c578812d0 blob([0x61d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10534 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c578812d0 blob([0x61d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c578812d0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c551371f0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551371f0) get_ref 0x0~2000 Blob(0x558c551371f0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551371f0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c551371f0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57f3bab0 blob([0x61f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10535 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f3bab0 blob([0x61f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f3bab0 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c55137260 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137260) get_ref 0x0~2000 Blob(0x558c55137260 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137260) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c55137260 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c5792c540 blob([0x621000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10536 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c5792c540 blob([0x621000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c5792c540 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=1)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup new Blob(0x558c551372d0 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551372d0) get_ref 0x0~2000 Blob(0x558c551372d0 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551372d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c551372d0 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57573788) dirty_range mark inline shard dirty 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# -> 1#1:db85c185:::smithi13140796-18:15# = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# snapset 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# snapset = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x12000~22000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:db85c185:::smithi13140796-18:head# 0x12000~22000 - have 0x12000 (73728) bytes fadvise_flags 0x0 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x12000~22000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef9200 17 blobs 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x22000/1000,22000,0 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x410000~22000/1000,22000,0 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x410000~22000] 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551373b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137340 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551373b0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137340) get_ref 0x0~2000 Blob(0x558c55137340 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551373b0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137340) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c55137340 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551373b0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551373c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137420 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137490 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137420) get_ref 0x0~2000 Blob(0x558c55137420 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137490 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137420) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c55137420 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137490 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551374a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137500 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137570 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137500) get_ref 0x0~2000 Blob(0x558c55137500 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137570 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137500) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c55137500 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137570 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137588 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c551375e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c551375e0 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137650 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551375e0) get_ref 0x0~2000 Blob(0x558c551375e0 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137650 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551375e0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c551375e0 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137650 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137668 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c551376c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c551376c0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137730 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551376c0) get_ref 0x0~2000 Blob(0x558c551376c0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137730 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551376c0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c551376c0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137730 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137748 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c551377a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c551377a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137810 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551377a0) get_ref 0x0~2000 Blob(0x558c551377a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137810 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c551377a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c551377a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137810 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551378f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137880 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551378f0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137880) get_ref 0x0~2000 Blob(0x558c55137880 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551378f0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137880) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c55137880 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551378f0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137908 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551379d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137960 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551379d0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137960) get_ref 0x0~2000 Blob(0x558c55137960 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551379d0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137960) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c55137960 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551379d0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551379e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137a40 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137ab0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137a40) get_ref 0x0~2000 Blob(0x558c55137a40 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137ab0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137a40) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c55137a40 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137ab0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137ac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55137b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137b20 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137b90 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137b20) get_ref 0x0~2000 Blob(0x558c55137b20 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55137b90 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c55137b20) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c55137b20 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137b90 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137ba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b39d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b39d0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3a40 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b39d0) get_ref 0x0~2000 Blob(0x558c550b39d0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3a40 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c550b39d0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3a40 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3ab0 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3b20 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3ab0) get_ref 0x0~2000 Blob(0x558c550b3ab0 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3b20 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c550b3ab0 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3b20 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3b90 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3c00 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3b90) get_ref 0x0~2000 Blob(0x558c550b3b90 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3c00 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3b90) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c550b3b90 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3c00 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3c70 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3ce0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3c70) get_ref 0x0~2000 Blob(0x558c550b3c70 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3ce0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3c70) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c550b3c70 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ce0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3cf8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3d50 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3dc0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3d50) get_ref 0x0~2000 Blob(0x558c550b3d50 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3dc0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3d50) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c550b3d50 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3dc0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3dd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3e30 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3ea0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3e30) get_ref 0x0~2000 Blob(0x558c550b3e30 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3ea0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3e30) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c550b3e30 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ea0 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3eb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3f10 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3f80 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3f10) get_ref 0x0~2000 Blob(0x558c550b3f10 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3f80 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3f10) get_ref init 0x2000, 1000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c550b3f10 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3f80 sbid 0x0)) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3f98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x34000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x12000~22000 = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_000000000000000C_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_000000000000000E_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000017' <- 0000000044.00000000000000000017 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000018' <- 0000000044.00000000000000000018 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9200 cost 216119 (18 ios * 4000 + 144119 bytes) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9200 onodes 0x558c57573680,0x558c57e8ec80 shared_blobs 0x558c57971ea0,0x558c57a075e0,0x558c57a07e30,0x558c57c38a10,0x558c57f49880,0x558c57f49960,0x558c57f49a40,0x558c57f49b20,0x558c5809b2d0 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57573788) update 1#1:db85c185:::smithi13140796-18:15# 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57573788) update inline shard 246 bytes from 9 extents 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:15# is 664 (412 bytes onode + 2 bytes spanning blobs + 250 bytes inline extents) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update inline shard 569 bytes from 26 extents 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1156 (581 bytes onode + 2 bytes spanning blobs + 573 bytes inline extents) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2927 is 11 SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2921 is 11 SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2920 is 11 SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2926 is 11 SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2925 is 11 SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2924 is 11 SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2923 is 11 SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2922 is 11 SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2928 is 11 SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2))) 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9200 allocated 0x[410000~22000] released 0x[] 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 freelist allocate 0x410000~22000 2019-12-07T19:34:21.626+0000 7f8f0317f700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 prepare 2019-12-07T19:34:21.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef9200 2019-12-07T19:34:21.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.627+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'16 crt=44'16 lcod 44'16 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'18 2019-12-07T19:34:21.627+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c57efec60 finish 2019-12-07T19:34:21.627+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.627+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 aio_wait 2019-12-07T19:34:21.627+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9200 2019-12-07T19:34:21.627+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 io_done 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5147 < min_flush_size 524288 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573680 had 1 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5170 < min_flush_size 524288 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5170 < min_flush_size 524288 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1d9373~1432 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1d9373 0x1d9373~1432 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1a9373 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x373 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7a5 and padding block with 0x85b 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1da7a5 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107380s (0.000003802s flush + 0.000103578s kv commit) 2019-12-07T19:34:21.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9200> 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 kv_submitted 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9200 2019-12-07T19:34:21.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9200 finishing 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9200 onodes 0x558c57573680,0x558c57e8ec80 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3a58 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a090 space 0x558c550b3a58 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3b38 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a870 space 0x558c550b3b38 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3c18 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a900 space 0x558c550b3c18 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3cf8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a990 space 0x558c550b3cf8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3dd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813aa20 space 0x558c550b3dd8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3eb8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813aab0 space 0x558c550b3eb8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3f98 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813ab40 space 0x558c550b3f98 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c551373c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c40360 space 0x558c551373c8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c551374a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c459e0 space 0x558c551374a8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137588 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45a70 space 0x558c55137588 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=77, last_complete=44'18, committed=1, applied=1)) v2 -- 0x558c575b9c00 con 0x558c54c5b400 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137668 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45b00 space 0x558c55137668 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137748 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45b90 space 0x558c55137748 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137828 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a3f0 space 0x558c55137828 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137908 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a360 space 0x558c55137908 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c551379e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a2d0 space 0x558c551379e8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137ac8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a240 space 0x558c55137ac8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55137ba8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813a1b0 space 0x558c55137ba8 0x0~2000 writing nocache) 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9200 done 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9200 [] 2019-12-07T19:34:21.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.628+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 325 ==== osd_map(44..44 src has 1..44) v4 ==== 633+0+0 (crc 0 0 0) 0x558c57351440 con 0x558c54c64000 2019-12-07T19:34:21.628+0000 7f8f1199c700 20 osd.7 44 OSD::ms_dispatch: osd_map(44..44 src has 1..44) v4 2019-12-07T19:34:21.628+0000 7f8f1199c700 10 osd.7 44 do_waiters -- start 2019-12-07T19:34:21.628+0000 7f8f1199c700 10 osd.7 44 do_waiters -- finish 2019-12-07T19:34:21.628+0000 7f8f1199c700 20 osd.7 44 _dispatch 0x558c57351440 osd_map(44..44 src has 1..44) v4 2019-12-07T19:34:21.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 326 ==== MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=216, reqid=client.4540.0:362, at_version=44'35, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 3175+0+0 (crc 0 0 0) 0x558c574a3000 con 0x558c54c64000 2019-12-07T19:34:21.628+0000 7f8f1199c700 3 osd.7 44 handle_osd_map epochs [44,44], i have 44, src has [1,44] 2019-12-07T19:34:21.628+0000 7f8f1199c700 10 osd.7 44 no new maps here, dropping 2019-12-07T19:34:21.628+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57efe420 prio 127 cost 0 latency 0.000024 epoch 44 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=216, reqid=client.4540.0:362, at_version=44'35, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.628+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=216, reqid=client.4540.0:362, at_version=44'35, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=216, reqid=client.4540.0:362, at_version=44'35, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=216, reqid=client.4540.0:362, at_version=44'35, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) pg 0x558c57380000 2019-12-07T19:34:21.628+0000 7f8f04982700 10 osd.7 44 dequeue_op 0x558c57efe420 prio 127 cost 0 latency 0.000067 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=216, reqid=client.4540.0:362, at_version=44'35, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.628+0000 7f8f04982700 10 osd.7 44 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 43 -> 44 (as per caller) 2019-12-07T19:34:21.628+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=216, reqid=client.4540.0:362, at_version=44'35, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.add_oid 1:131421e8:::smithi13140796-7:15 c,e,13,15 2019-12-07T19:34:21.628+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.628+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:21.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:15 got.empty() 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.set_snaps 1:131421e8:::smithi13140796-7:15 c,e,13,15 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.628+0000 7f8f04982700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.628+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,36'33], crt=36'33) [44'34 (29'29) clone 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,44'35 (29'29) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:362 2019-12-07T19:34:21.628705+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.628+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'34 (0'0,44'34] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=36'33 crt=36'33 lcod 36'33 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'34 (29'29) clone 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.628+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'35 (0'0,44'35] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=36'33 crt=36'33 lcod 36'33 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'35 (29'29) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:362 2019-12-07T19:34:21.628705+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.629+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'35 (0'0,44'35] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=36'33 crt=36'33 lcod 36'33 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 35 2019-12-07T19:34:21.629+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'35 (0'0,44'35] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=36'33 crt=36'33 lcod 36'33 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.629+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.629+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'34, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c577fcc00 seq 25 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:131421e8:::smithi13140796-7:15# key 0x858000000000000001131421e821736d'ithi13140796-7!='0x0000000000000015ffffffffffffffff'o' 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# -> 5#1:131421e8:::smithi13140796-7:15# 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1222 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.onode(0x558c5840cc80).flush flush done 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 0x0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# -> 5#1:131421e8:::smithi13140796-7:15# 0x0~1f000 -> 0x0~1f000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x0~2000: 0x0~2000 Blob(0x558c57c388c0 blob([0x83a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10537 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57c388c0 blob([0x83a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57c388c0 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db4fc0 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4fc0) get_ref 0x0~2000 Blob(0x558c57db4fc0 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57db4fc0 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c583ea540 blob([0x83c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10538 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ea540 blob([0x83c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ea540 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db4ee0 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4ee0) get_ref 0x0~2000 Blob(0x558c57db4ee0 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57db4ee0 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c583ea700 blob([0x83e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10539 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ea700 blob([0x83e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ea700 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c550b3340 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c550b3340) get_ref 0x0~2000 Blob(0x558c550b3340 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c550b3340) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c550b3340 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c583ea7e0 blob([0x840000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10540 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ea7e0 blob([0x840000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ea7e0 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db4e00 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4e00) get_ref 0x0~2000 Blob(0x558c57db4e00 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4e00) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57db4e00 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c583ea8c0 blob([0x842000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10541 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ea8c0 blob([0x842000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ea8c0 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db5030 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db5030) get_ref 0x0~2000 Blob(0x558c57db5030 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db5030) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57db5030 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c583ea9a0 blob([0x844000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10542 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ea9a0 blob([0x844000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ea9a0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db50a0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db50a0) get_ref 0x0~2000 Blob(0x558c57db50a0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db50a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57db50a0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c583eaa80 blob([0x846000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10543 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583eaa80 blob([0x846000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583eaa80 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db44d0 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db44d0) get_ref 0x0~2000 Blob(0x558c57db44d0 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db44d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57db44d0 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c583eab60 blob([0x848000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10544 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583eab60 blob([0x848000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583eab60 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c580ec540 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c580ec540) get_ref 0x0~2000 Blob(0x558c580ec540 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c580ec540) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c580ec540 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c583eac40 blob([0x84a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10545 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583eac40 blob([0x84a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583eac40 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57ed1b90 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57ed1b90) get_ref 0x0~2000 Blob(0x558c57ed1b90 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57ed1b90) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57ed1b90 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x12000~2000: 0x0~2000 Blob(0x558c583ead20 blob([0x84c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10546 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ead20 blob([0x84c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ead20 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c580ec5b0 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c580ec5b0) get_ref 0x0~2000 Blob(0x558c580ec5b0 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c580ec5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c580ec5b0 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x14000~2000: 0x0~2000 Blob(0x558c583eb960 blob([0x84e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10547 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583eb960 blob([0x84e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583eb960 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db43f0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db43f0) get_ref 0x0~2000 Blob(0x558c57db43f0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db43f0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57db43f0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x16000~2000: 0x0~2000 Blob(0x558c583eba40 blob([0x850000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10548 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583eba40 blob([0x850000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583eba40 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db4540 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4540) get_ref 0x0~2000 Blob(0x558c57db4540 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4540) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c57db4540 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x18000~2000: 0x0~2000 Blob(0x558c583ebb20 blob([0x852000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10549 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ebb20 blob([0x852000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ebb20 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db45b0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db45b0) get_ref 0x0~2000 Blob(0x558c57db45b0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db45b0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57db45b0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c583ebc00 blob([0x854000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10550 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ebc00 blob([0x854000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ebc00 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db4620 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4620) get_ref 0x0~2000 Blob(0x558c57db4620 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4620) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57db4620 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c583ebce0 blob([0x856000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10551 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ebce0 blob([0x856000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ebce0 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db4690 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4690) get_ref 0x0~2000 Blob(0x558c57db4690 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4690) get_ref init 0x2000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57db4690 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup src 0x1e000~1000: 0x0~1000 Blob(0x558c583ebdc0 blob([0x858000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10552 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c583ebdc0 blob([0x858000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 sbid 0x0)) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c583ebdc0 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=1)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup new Blob(0x558c57db4700 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4700) get_ref 0x0~1000 Blob(0x558c57db4700 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.blob(0x558c57db4700) get_ref init 0x1000, 1000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dup dst 0x1e000~1000: 0x0~1000 Blob(0x558c57db4700 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2)))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# -> 5#1:131421e8:::smithi13140796-7:15# = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# snapset 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# snapset = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_000000000000000C_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_000000000000000E_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_0000000000000013_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_0000000000000015_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000044.00000000000000000034' <- 0000000044.00000000000000000034 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000044.00000000000000000035' <- 0000000044.00000000000000000035 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c577fcc00 cost 8693 (1 ios * 4000 + 4693 bytes) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c577fcc00 onodes 0x558c575f9b80,0x558c5840cc80 shared_blobs 0x558c577f2000,0x558c583ea690,0x558c583ea770,0x558c583ea850,0x558c583ea930,0x558c583eaa10,0x558c583eaaf0,0x558c583eabd0,0x558c583eacb0,0x558c583ead90,0x558c583eb9d0,0x558c583ebab0,0x558c583ebb90,0x558c583ebc70,0x558c583ebd50,0x558c583ebe30 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) update 5#1:131421e8:::smithi13140796-7:15# 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) update inline shard 432 bytes from 16 extents 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:15# is 849 (411 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) update inline shard 432 bytes from 16 extents 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 1018 (580 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2929 is 11 SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292a is 11 SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292b is 11 SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292c is 11 SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292d is 11 SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292e is 11 SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292f is 11 SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2930 is 11 SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2931 is 11 SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2932 is 11 SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2933 is 11 SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2934 is 11 SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2935 is 11 SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2936 is 11 SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2937 is 11 SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2938 is 11 SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2))) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c577fcc00 allocated 0x[] released 0x[] 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 prepare 2019-12-07T19:34:21.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c577fcc00 2019-12-07T19:34:21.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 io_done 2019-12-07T19:34:21.629+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'35 (0'0,44'35] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=36'33 crt=36'33 lcod 36'33 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'35 2019-12-07T19:34:21.629+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.629+0000 7f8f04982700 10 osd.7 44 dequeue_op 0x558c57efe420 finish 2019-12-07T19:34:21.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5327 < min_flush_size 524288 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9b80 had 1 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840cc80 had 1 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5350 < min_flush_size 524288 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5350 < min_flush_size 524288 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1da7a5~14e6 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1da7a5 0x1da7a5~14e6 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1aa7a5 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7a5 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc8b and padding block with 0x375 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1dbc8b 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000158533s (0.000000102s flush + 0.000158431s kv commit) 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c577fcc00> 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 kv_submitted 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c577fcc00 2019-12-07T19:34:21.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c577fcc00 finishing 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c577fcc00 onodes 0x558c575f9b80,0x558c5840cc80 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c577fcc00 done 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:21.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c577fcc00 [] 2019-12-07T19:34:21.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.629+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 44/41 ECSubWriteReply(tid=216, last_complete=44'35, committed=1, applied=1)) v2 -- 0x558c548cc200 con 0x558c54c64000 2019-12-07T19:34:21.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.629+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 327 ==== MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=217, reqid=client.4540.0:363, at_version=44'36, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2134+0+0 (crc 0 0 0) 0x558c54d87800 con 0x558c54c64000 2019-12-07T19:34:21.629+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57efe580 prio 127 cost 0 latency 0.000051 epoch 44 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=217, reqid=client.4540.0:363, at_version=44'36, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.629+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=217, reqid=client.4540.0:363, at_version=44'36, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=217, reqid=client.4540.0:363, at_version=44'36, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=217, reqid=client.4540.0:363, at_version=44'36, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) pg 0x558c57380000 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 44 dequeue_op 0x558c57efe580 prio 127 cost 0 latency 0.000110 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=217, reqid=client.4540.0:363, at_version=44'36, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.8s5( v 44'35 (0'0,44'35] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'35 (0'0,44'35] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=217, reqid=client.4540.0:363, at_version=44'36, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'35 (0'0,44'35] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'35], crt=36'33) [44'36 (44'35) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:363 2019-12-07T19:34:21.628730+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0] 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'36 (0'0,44'36] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'35 crt=36'33 lcod 44'35 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'36 (44'35) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:363 2019-12-07T19:34:21.628730+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'36 (0'0,44'36] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'35 crt=36'33 lcod 44'35 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 36 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'36 (0'0,44'36] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'35 crt=36'33 lcod 44'35 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.630+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.630+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'36, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c574fec00 seq 26 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:131421e8:::smithi13140796-7:head#24 key 0x858000000000000001131421e821736d'ithi13140796-7!='0xfffffffffffffffe0000000000000024'o' 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:21.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# -> 5#1:131421e8:::smithi13140796-7:head#24 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# -> 5#1:131421e8:::smithi13140796-7:head#24 = 0 2019-12-07T19:34:21.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# -> 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1223 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.onode(0x558c575f9b80).flush flush done 2019-12-07T19:34:21.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0 2019-12-07T19:34:21.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# -> 5#1:131421e8:::smithi13140796-7:head# 0x0~1f000 -> 0x0~1f000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x0~2000: 0x0~2000 Blob(0x558c57db4fc0 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c550b3500 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c550b3500) get_ref 0x0~2000 Blob(0x558c550b3500 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c550b3500) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c550b3500 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57db4ee0 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c550b3260 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c550b3260) get_ref 0x0~2000 Blob(0x558c550b3260 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c550b3260) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c550b3260 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c550b3340 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c550b32d0 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c550b32d0) get_ref 0x0~2000 Blob(0x558c550b32d0 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c550b32d0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c550b32d0 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57db4e00 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db47e0 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db47e0) get_ref 0x0~2000 Blob(0x558c57db47e0 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db47e0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57db47e0 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57db5030 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4850 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4850) get_ref 0x0~2000 Blob(0x558c57db4850 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4850) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57db4850 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57db50a0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db48c0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db48c0) get_ref 0x0~2000 Blob(0x558c57db48c0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db48c0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57db48c0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57db44d0 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4930 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4930) get_ref 0x0~2000 Blob(0x558c57db4930 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4930) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57db4930 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c580ec540 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db49a0 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db49a0) get_ref 0x0~2000 Blob(0x558c57db49a0 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db49a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57db49a0 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57ed1b90 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4a10 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4a10) get_ref 0x0~2000 Blob(0x558c57db4a10 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4a10) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57db4a10 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x12000~2000: 0x0~2000 Blob(0x558c580ec5b0 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4a80 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4a80) get_ref 0x0~2000 Blob(0x558c57db4a80 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4a80) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57db4a80 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57db43f0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4af0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4af0) get_ref 0x0~2000 Blob(0x558c57db4af0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4af0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57db4af0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57db4540 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4b60 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4b60) get_ref 0x0~2000 Blob(0x558c57db4b60 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4b60) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c57db4b60 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x18000~2000: 0x0~2000 Blob(0x558c57db45b0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4bd0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4bd0) get_ref 0x0~2000 Blob(0x558c57db4bd0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57db4bd0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57db4620 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4c40 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4c40) get_ref 0x0~2000 Blob(0x558c57db4c40 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4c40) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c57db4c40 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57db4690 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4cb0 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4cb0) get_ref 0x0~2000 Blob(0x558c57db4cb0 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57db4cb0 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup src 0x1e000~1000: 0x0~1000 Blob(0x558c57db4700 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup new Blob(0x558c57db4d20 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4d20) get_ref 0x0~1000 Blob(0x558c57db4d20 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.blob(0x558c57db4d20) get_ref init 0x1000, 1000 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dup dst 0x1e000~1000: 0x0~1000 Blob(0x558c57db4d20 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=3)))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c57598108) dirty_range mark inline shard dirty 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# -> 5#1:131421e8:::smithi13140796-7:head# = 0 2019-12-07T19:34:21.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:21.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000044.00000000000000000036' <- 0000000044.00000000000000000036 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._fastinfo' <- _fastinfo 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fec00 cost 5785 (1 ios * 4000 + 1785 bytes) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fec00 onodes 0x558c57598000,0x558c5840cc80 shared_blobs 0x558c577f2000,0x558c583ea690,0x558c583ea770,0x558c583ea850,0x558c583ea930,0x558c583eaa10,0x558c583eaaf0,0x558c583eabd0,0x558c583eacb0,0x558c583ead90,0x558c583eb9d0,0x558c583ebab0,0x558c583ebb90,0x558c583ebc70,0x558c583ebd50,0x558c583ebe30 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c57598108) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c57598108) update inline shard 432 bytes from 16 extents 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 1018 (580 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840cd88) update 5#1:131421e8:::smithi13140796-7:head#24 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head#24 is 1018 (580 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2929 is 11 SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292a is 11 SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292b is 11 SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292c is 11 SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292d is 11 SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292e is 11 SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292f is 11 SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2930 is 11 SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2931 is 11 SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2932 is 11 SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2933 is 11 SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2934 is 11 SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2935 is 11 SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2936 is 11 SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2937 is 11 SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2938 is 11 SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=3))) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fec00 allocated 0x[] released 0x[] 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 prepare 2019-12-07T19:34:21.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fec00 2019-12-07T19:34:21.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 io_done 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'36 (0'0,44'36] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'35 crt=36'33 lcod 44'35 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'36 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 44 dequeue_op 0x558c57efe580 finish 2019-12-07T19:34:21.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3217 < min_flush_size 524288 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57598000 had 1 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840cc80 had 1 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3240 < min_flush_size 524288 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3240 < min_flush_size 524288 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1dbc8b~ca8 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1dbc8b 0x1dbc8b~ca8 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1abc8b 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc8b 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x933 and padding block with 0x6cd 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1dc933 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000103811s (0.000000122s flush + 0.000103689s kv commit) 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fec00> 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 kv_submitted 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fec00 2019-12-07T19:34:21.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fec00 finishing 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fec00 onodes 0x558c57598000,0x558c5840cc80 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fec00 done 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:21.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fec00 [] 2019-12-07T19:34:21.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.630+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 44/41 ECSubWriteReply(tid=217, last_complete=44'36, committed=1, applied=1)) v2 -- 0x558c57519400 con 0x558c54c64000 2019-12-07T19:34:21.630+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 328 ==== MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=218, reqid=client.4540.0:364, at_version=44'37, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2352+0+0 (crc 0 0 0) 0x558c5748d000 con 0x558c54c64000 2019-12-07T19:34:21.630+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57efe6e0 prio 127 cost 0 latency 0.000021 epoch 44 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=218, reqid=client.4540.0:364, at_version=44'37, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.630+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=218, reqid=client.4540.0:364, at_version=44'37, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=218, reqid=client.4540.0:364, at_version=44'37, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=218, reqid=client.4540.0:364, at_version=44'37, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e44) pg 0x558c57380000 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 44 dequeue_op 0x558c57efe6e0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=218, reqid=client.4540.0:364, at_version=44'37, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.8s5( v 44'36 (0'0,44'36] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'36 (0'0,44'36] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=218, reqid=client.4540.0:364, at_version=44'37, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:21.630+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'36 (0'0,44'36] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'36], crt=36'33) [44'37 (44'36) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:364 2019-12-07T19:34:21.628735+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.631+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'36 crt=36'33 lcod 44'36 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'37 (44'36) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:364 2019-12-07T19:34:21.628735+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.631+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'36 crt=36'33 lcod 44'36 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 37 2019-12-07T19:34:21.631+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'36 crt=36'33 lcod 44'36 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.631+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.631+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'37, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c574ffb00 seq 27 2019-12-07T19:34:21.631+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:21.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:21.631+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.631+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000044.00000000000000000037' <- 0000000044.00000000000000000037 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._fastinfo' <- _fastinfo 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:21.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ffb00 cost 6003 (1 ios * 4000 + 2003 bytes) 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ffb00 onodes 0x558c57598000 shared_blobs 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore.extentmap(0x558c57598108) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 1018 (580 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ffb00 allocated 0x[] released 0x[] 2019-12-07T19:34:21.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 prepare 2019-12-07T19:34:21.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ffb00 2019-12-07T19:34:21.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 io_done 2019-12-07T19:34:21.631+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'36 crt=36'33 lcod 44'36 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'37 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.631+0000 7f8f04982700 10 osd.7 44 dequeue_op 0x558c57efe6e0 finish 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2154 < min_flush_size 524288 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57598000 had 1 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2177 < min_flush_size 524288 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2177 < min_flush_size 524288 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1dc933~881 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1dc933 0x1dc933~881 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1ac933 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x933 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1b4 and padding block with 0xe4c 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1dd1b4 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000111443s (0.000000115s flush + 0.000111328s kv commit) 2019-12-07T19:34:21.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574ffb00> 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 kv_submitted 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ffb00 2019-12-07T19:34:21.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ffb00 finishing 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ffb00 onodes 0x558c57598000 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ffb00 done 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:21.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ffb00 [] 2019-12-07T19:34:21.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.631+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 44/41 ECSubWriteReply(tid=218, last_complete=44'37, committed=1, applied=1)) v2 -- 0x558c57519000 con 0x558c54c64000 2019-12-07T19:34:21.631+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 213 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=78, reqid=client.4540.0:358, at_version=44'19, trim_to=0'0, roll_forward_to=44'16)) v2 ==== 170456+0+0 (crc 0 0 0) 0x558c5748e000 con 0x558c54c5b400 2019-12-07T19:34:21.632+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c57eff340 prio 127 cost 0 latency 0.000023 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=78, reqid=client.4540.0:358, at_version=44'19, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.632+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=78, reqid=client.4540.0:358, at_version=44'19, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=78, reqid=client.4540.0:358, at_version=44'19, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=78, reqid=client.4540.0:358, at_version=44'19, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c57eff340 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=78, reqid=client.4540.0:358, at_version=44'19, trim_to=0'0, roll_forward_to=44'16)) v2 pg pg[1.bs1( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=78, reqid=client.4540.0:358, at_version=44'19, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'18], crt=44'16) [44'19 (44'18) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:358 2019-12-07T19:34:21.627982+0000 0 ObjectCleanRegions clean_offsets: [0~851968,1523712~18446744073708027903], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'18 crt=44'16 lcod 44'18 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'19 (44'18) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:358 2019-12-07T19:34:21.627982+0000 0 ObjectCleanRegions clean_offsets: [0~851968,1523712~18446744073708027903], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'18 crt=44'16 lcod 44'18 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 19 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'18 crt=44'16 lcod 44'18 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.632+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'19, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57ef9b00 seq 21 2019-12-07T19:34:21.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:34:21.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:21.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x34000~29000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:db85c185:::smithi13140796-18:head# 0x34000~29000 - have 0x34000 (212992) bytes fadvise_flags 0x0 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x34000~29000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef9b00 21 blobs 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x29000/1000,29000,0 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x432000~c000/1000,29000,0 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x973000~1d000/1000,29000,0 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x432000~c000,0x973000~1d000] 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b2fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2fc0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c55137110) get_ref 0x0~2000 Blob(0x558c55137110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2fc0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c55137110) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c55137110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2fc0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b2fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b30a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3030 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b30a0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3030) get_ref 0x0~2000 Blob(0x558c550b3030 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b30a0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3030) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c550b3030 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b30a0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b30b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d67e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d67e0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6850 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d67e0) get_ref 0x0~2000 Blob(0x558c580d67e0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6850 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d67e0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c580d67e0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6850 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d68c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d68c0 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6930 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d68c0) get_ref 0x0~2000 Blob(0x558c580d68c0 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6930 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d68c0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c580d68c0 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6930 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d69a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d69a0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6a10 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d69a0) get_ref 0x0~2000 Blob(0x558c580d69a0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6a10 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d69a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c580d69a0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6a10 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d6a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d6a80 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6af0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6a80) get_ref 0x0~2000 Blob(0x558c580d6a80 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6af0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6a80) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c580d6a80 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6af0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6b08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d6b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d6b60 blob([0x973000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6bd0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6b60) get_ref 0x0~2000 Blob(0x558c580d6b60 blob([0x973000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6bd0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6b60) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c580d6b60 blob([0x973000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6bd0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6be8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d6c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d6c40 blob([0x975000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6cb0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6c40) get_ref 0x0~2000 Blob(0x558c580d6c40 blob([0x975000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6cb0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6c40) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c580d6c40 blob([0x975000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6cb0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6cc8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d6d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d6d20 blob([0x977000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6d90 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6d20) get_ref 0x0~2000 Blob(0x558c580d6d20 blob([0x977000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6d90 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c580d6d20) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c580d6d20 blob([0x977000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6d90 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6da8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c423f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c577f2690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c423f0 blob([0x979000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2690 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c57c423f0) get_ref 0x0~2000 Blob(0x558c57c423f0 blob([0x979000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f2690 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c57c423f0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c57c423f0 blob([0x979000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2690 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c577f26a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55136fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55136ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55136fc0 blob([0x97b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136ee0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c55136fc0) get_ref 0x0~2000 Blob(0x558c55136fc0 blob([0x97b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136ee0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c55136fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c55136fc0 blob([0x97b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55136ee0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55136ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579710a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579710a0 blob([0x97d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c579710a0) get_ref 0x0~2000 Blob(0x558c579710a0 blob([0x97d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c579710a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c579710a0 blob([0x97d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3c00 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581eab60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581eab60 blob([0x97f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adea0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581eab60) get_ref 0x0~2000 Blob(0x558c581eab60 blob([0x97f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adea0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581eab60) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c581eab60 blob([0x97f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adea0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adeb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581eaa80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f11f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581eaa80 blob([0x981000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581eaa80) get_ref 0x0~2000 Blob(0x558c581eaa80 blob([0x981000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581eaa80) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c581eaa80 blob([0x981000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea9a0 blob([0x983000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1960 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea9a0) get_ref 0x0~2000 Blob(0x558c581ea9a0 blob([0x983000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1960 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c581ea9a0 blob([0x983000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1960 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574f1978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea8c0 blob([0x985000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea8c0) get_ref 0x0~2000 Blob(0x558c581ea8c0 blob([0x985000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c581ea8c0 blob([0x985000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea7e0 blob([0x987000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adf10 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea7e0) get_ref 0x0~2000 Blob(0x558c581ea7e0 blob([0x987000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adf10 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c581ea7e0 blob([0x987000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adf10 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adf28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea700 blob([0x989000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea700) get_ref 0x0~2000 Blob(0x558c581ea700 blob([0x989000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea700) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c581ea700 blob([0x989000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea620 blob([0x98b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad730 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea620) get_ref 0x0~2000 Blob(0x558c581ea620 blob([0x98b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad730 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea620) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c581ea620 blob([0x98b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad730 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad748 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea540 blob([0x98d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971c00 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea540) get_ref 0x0~2000 Blob(0x558c581ea540 blob([0x98d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971c00 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea540) get_ref init 0x2000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c581ea540 blob([0x98d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971c00 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57971c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581ea460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581ea460 blob([0x98f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970150 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea460) get_ref 0x0~1000 Blob(0x558c581ea460 blob([0x98f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970150 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.blob(0x558c581ea460) get_ref init 0x1000, 1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~1000: 0x0~1000 Blob(0x558c581ea460 blob([0x98f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57970150 sbid 0x0)) 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57970168 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5d000 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x34000~29000 = 0 2019-12-07T19:34:21.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.632+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000019' <- 0000000044.00000000000000000019 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.632+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.632+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef9b00 cost 258106 (22 ios * 4000 + 170106 bytes) 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef9b00 onodes 0x558c57e8ec80 shared_blobs 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update inline shard 965 bytes from 47 extents 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1552 (581 bytes onode + 2 bytes spanning blobs + 969 bytes inline extents) 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef9b00 allocated 0x[432000~c000,973000~1d000] released 0x[] 2019-12-07T19:34:21.633+0000 7f8f0317f700 10 freelist allocate 0x432000~c000 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:21.633+0000 7f8f0317f700 10 freelist allocate 0x973000~1d000 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 freelist _xor first_key 0x900000 last_key 0x980000 2019-12-07T19:34:21.633+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 prepare 2019-12-07T19:34:21.633+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef9b00 2019-12-07T19:34:21.633+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'18 crt=44'16 lcod 44'18 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'19 2019-12-07T19:34:21.633+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c57eff340 finish 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.633+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 aio_wait 2019-12-07T19:34:21.633+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef9b00 2019-12-07T19:34:21.633+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 io_done 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2846 < min_flush_size 524288 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2869 < min_flush_size 524288 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2869 < min_flush_size 524288 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1dd1b4~b35 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1dd1b4 0x1dd1b4~b35 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1ad1b4 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1b4 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xce9 and padding block with 0x317 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1ddce9 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000152149s (0.000043385s flush + 0.000108764s kv commit) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef9b00> 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 kv_submitted 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef9b00 2019-12-07T19:34:21.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef9b00 finishing 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef9b00 onodes 0x558c57e8ec80 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b2fd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5796eb40 space 0x558c550b2fd8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b30b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c402d0 space 0x558c550b30b8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55136ef8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e1b0 space 0x558c55136ef8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e360 space 0x558c574f1208 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1978 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e3f0 space 0x558c574f1978 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad748 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e630 space 0x558c576ad748 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e5a0 space 0x558c576ad828 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adeb8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e2d0 space 0x558c576adeb8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adf28 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e510 space 0x558c576adf28 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c577f26a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e120 space 0x558c577f26a8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970168 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e750 space 0x558c57970168 0x0~1000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=78, last_complete=44'19, committed=1, applied=1)) v2 -- 0x558c55250800 con 0x558c54c5b400 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e480 space 0x558c57971198 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971c18 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e6c0 space 0x558c57971c18 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e240 space 0x558c579b3c18 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6868 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813ac60 space 0x558c580d6868 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6948 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813acf0 space 0x558c580d6948 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6a28 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813ad80 space 0x558c580d6a28 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6b08 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813ae10 space 0x558c580d6b08 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6be8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813aea0 space 0x558c580d6be8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6cc8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e000 space 0x558c580d6cc8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6da8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e090 space 0x558c580d6da8 0x0~2000 writing nocache) 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef9b00 done 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef9b00 [] 2019-12-07T19:34:21.633+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.633+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 214 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=79, reqid=client.4540.0:359, at_version=44'20, trim_to=0'0, roll_forward_to=44'16)) v2 ==== 27096+0+0 (crc 0 0 0) 0x558c57e01800 con 0x558c54c5b400 2019-12-07T19:34:21.634+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c54e79600 prio 127 cost 0 latency 0.000028 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=79, reqid=client.4540.0:359, at_version=44'20, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.634+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=79, reqid=client.4540.0:359, at_version=44'20, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=79, reqid=client.4540.0:359, at_version=44'20, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=79, reqid=client.4540.0:359, at_version=44'20, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c54e79600 prio 127 cost 0 latency 0.000362 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=79, reqid=client.4540.0:359, at_version=44'20, trim_to=0'0, roll_forward_to=44'16)) v2 pg pg[1.bs1( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=79, reqid=client.4540.0:359, at_version=44'20, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'19], crt=44'16) [44'20 (44'19) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:359 2019-12-07T19:34:21.628639+0000 0 ObjectCleanRegions clean_offsets: [0~1523712,1622016~18446744073707929599], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'20 (0'0,44'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'19 crt=44'16 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'20 (44'19) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:359 2019-12-07T19:34:21.628639+0000 0 ObjectCleanRegions clean_offsets: [0~1523712,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'20 (0'0,44'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'19 crt=44'16 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 20 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'20 (0'0,44'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'19 crt=44'16 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.634+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'20, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57d0a000 seq 22 2019-12-07T19:34:21.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:34:21.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:21.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x5d000~6000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:db85c185:::smithi13140796-18:head# 0x5d000~6000 - have 0x5d000 (380928) bytes fadvise_flags 0x0 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x5d000~6000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57d0a000 3 blobs 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x6000/1000,6000,0 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x990000~6000/1000,6000,0 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x990000~6000] 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d6770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3180 blob([0x990000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6770 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3180) get_ref 0x0~2000 Blob(0x558c550b3180 blob([0x990000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d6770 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3180) get_ref init 0x2000, 1000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5d000~2000: 0x0~2000 Blob(0x558c550b3180 blob([0x990000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6770 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b20e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b20e0 blob([0x992000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3110 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.blob(0x558c579b20e0) get_ref 0x0~2000 Blob(0x558c579b20e0 blob([0x992000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3110 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.blob(0x558c579b20e0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5f000~2000: 0x0~2000 Blob(0x558c579b20e0 blob([0x992000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3110 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3128 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b3030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b30a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b3030 blob([0x994000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3030) get_ref 0x0~2000 Blob(0x558c579b3030 blob([0x994000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3030) get_ref init 0x2000, 1000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x61000~2000: 0x0~2000 Blob(0x558c579b3030 blob([0x994000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x63000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x5d000~6000 = 0 2019-12-07T19:34:21.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000020' <- 0000000044.00000000000000000020 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0a000 cost 42746 (4 ios * 4000 + 26746 bytes) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0a000 onodes 0x558c57e8ec80 shared_blobs 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update inline shard 1023 bytes from 50 extents 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1610 (581 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0a000 allocated 0x[990000~6000] released 0x[] 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 freelist allocate 0x990000~6000 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 freelist _xor first_key 0x980000 last_key 0x980000 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a000 prepare 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57d0a000 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'20 (0'0,44'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'19 crt=44'16 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'20 2019-12-07T19:34:21.634+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.634+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a000 aio_wait 2019-12-07T19:34:21.634+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0a000 2019-12-07T19:34:21.634+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a000 io_done 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2846 < min_flush_size 524288 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2869 < min_flush_size 524288 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2869 < min_flush_size 524288 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1ddce9~b35 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1ddce9 0x1ddce9~b35 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1adce9 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xce9 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x81e and padding block with 0x7e2 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1de81e 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124935s (0.000038999s flush + 0.000085936s kv commit) 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57d0a000> 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a000 kv_submitted 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0a000 2019-12-07T19:34:21.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a000 finishing 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0a000 onodes 0x558c57e8ec80 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e900 space 0x558c579b30b8 0x0~2000 writing nocache) 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3128 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e870 space 0x558c579b3128 0x0~2000 writing nocache) 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d6788 in 0x558c53a59c00) _finish_write discard buffer(0x558c5813abd0 space 0x558c580d6788 0x0~2000 writing nocache) 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0a000 done 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0a000 [] 2019-12-07T19:34:21.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.634+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=79, last_complete=44'20, committed=1, applied=1)) v2 -- 0x558c55250400 con 0x558c54c5b400 2019-12-07T19:34:21.634+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 215 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=80, reqid=client.4540.0:360, at_version=44'21, trim_to=0'0, roll_forward_to=44'16)) v2 ==== 2487+0+0 (crc 0 0 0) 0x558c579e3800 con 0x558c54c5b400 2019-12-07T19:34:21.635+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c54e79340 prio 127 cost 0 latency 0.000033 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=80, reqid=client.4540.0:360, at_version=44'21, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.635+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=80, reqid=client.4540.0:360, at_version=44'21, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=80, reqid=client.4540.0:360, at_version=44'21, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=80, reqid=client.4540.0:360, at_version=44'21, trim_to=0'0, roll_forward_to=44'16)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c54e79340 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=80, reqid=client.4540.0:360, at_version=44'21, trim_to=0'0, roll_forward_to=44'16)) v2 pg pg[1.bs1( v 44'20 (0'0,44'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'20 (0'0,44'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=80, reqid=client.4540.0:360, at_version=44'21, trim_to=0'0, roll_forward_to=44'16)) v2 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'20 (0'0,44'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'20], crt=44'16) [44'21 (44'20) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:360 2019-12-07T19:34:21.628654+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'20 crt=44'16 lcod 44'20 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'21 (44'20) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:360 2019-12-07T19:34:21.628654+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'20 crt=44'16 lcod 44'20 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 21 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'20 crt=44'16 lcod 44'20 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.635+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'21, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57d0a300 seq 23 2019-12-07T19:34:21.635+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 3 keys 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 3 keys = 0 2019-12-07T19:34:21.635+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.635+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000044.00000000000000000021' <- 0000000044.00000000000000000021 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0a300 cost 6137 (1 ios * 4000 + 2137 bytes) 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0a300 onodes 0x558c57e8ec80 shared_blobs 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1611 (582 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0a300 allocated 0x[] released 0x[] 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a300 prepare 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0a300 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a300 io_done 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'20 crt=44'16 lcod 44'20 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'21 2019-12-07T19:34:21.635+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c54e79340 finish 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2807 < min_flush_size 524288 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2830 < min_flush_size 524288 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2830 < min_flush_size 524288 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1de81e~b0e to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1de81e 0x1de81e~b0e to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1ae81e 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x81e 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x32c and padding block with 0xcd4 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1df32c 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107381s (0.000000094s flush + 0.000107287s kv commit) 2019-12-07T19:34:21.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57d0a300> 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a300 kv_submitted 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0a300 2019-12-07T19:34:21.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a300 finishing 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0a300 onodes 0x558c57e8ec80 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0a300 done 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0a300 [] 2019-12-07T19:34:21.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.635+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=80, last_complete=44'21, committed=1, applied=1)) v2 -- 0x558c5795c600 con 0x558c54c5b400 2019-12-07T19:34:21.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.638+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 216 ==== MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=81, reqid=client.4540.0:360, at_version=0'0, trim_to=0'0, roll_forward_to=44'21)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e0800 con 0x558c54c5b400 2019-12-07T19:34:21.638+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000023 epoch 44 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=81, reqid=client.4540.0:360, at_version=0'0, trim_to=0'0, roll_forward_to=44'21)) v2 2019-12-07T19:34:21.638+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=81, reqid=client.4540.0:360, at_version=0'0, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=81, reqid=client.4540.0:360, at_version=0'0, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=81, reqid=client.4540.0:360, at_version=0'0, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=81, reqid=client.4540.0:360, at_version=0'0, trim_to=0'0, roll_forward_to=44'21)) v2 pg pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 44/41 ECSubWrite(tid=81, reqid=client.4540.0:360, at_version=0'0, trim_to=0'0, roll_forward_to=44'21)) v2 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'16 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'21], crt=44'16) [] 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'17 (21'2) clone 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:33:58.659098+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'18 (21'2) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:357 2019-12-07T19:34:21.624162+0000 0 ObjectCleanRegions clean_offsets: [0~294912,851968~18446744073708699647], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'19 (44'18) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:358 2019-12-07T19:34:21.627982+0000 0 ObjectCleanRegions clean_offsets: [0~851968,1523712~18446744073708027903], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'20 (44'19) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:359 2019-12-07T19:34:21.628639+0000 0 ObjectCleanRegions clean_offsets: [0~1523712,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'21 (44'20) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:360 2019-12-07T19:34:21.628654+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 21 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.638+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57d0a600 seq 24 2019-12-07T19:34:21.638+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0a600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0a600 onodes shared_blobs 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0a600 allocated 0x[] released 0x[] 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a600 prepare 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0a600 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a600 io_done 2019-12-07T19:34:21.638+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1df32c~471 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1df32c 0x1df32c~471 to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1b0000 x_off 0x1af32c 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x32c 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x79d and padding block with 0x863 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1df79d 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000126342s (0.000000080s flush + 0.000126262s kv commit) 2019-12-07T19:34:21.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57d0a600> 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a600 kv_submitted 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0a600 2019-12-07T19:34:21.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a600 finishing 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0a600 onodes 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0a600 done 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:21.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0a600 [] 2019-12-07T19:34:21.638+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.638+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=81, last_complete=44'21, committed=1, applied=1)) v2 -- 0x558c5795c800 con 0x558c54c5b400 2019-12-07T19:34:21.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.638+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 217 ==== MOSDECSubOpRead(1.bs1 44/41 ECSubRead(tid=82, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57599400 con 0x558c54c5b400 2019-12-07T19:34:21.638+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c581cd340 prio 127 cost 0 latency 0.000030 epoch 44 MOSDECSubOpRead(1.bs1 44/41 ECSubRead(tid=82, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:21.638+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 44/41 ECSubRead(tid=82, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e44) 2019-12-07T19:34:21.638+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 329 ==== MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=219, reqid=client.4540.0:365, at_version=44'19, trim_to=0'0, roll_forward_to=42'17)) v2 ==== 61375+0+0 (crc 0 0 0) 0x558c579e2800 con 0x558c54c64000 2019-12-07T19:34:21.638+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c581cd760 prio 127 cost 0 latency 0.000029 epoch 44 MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=219, reqid=client.4540.0:365, at_version=44'19, trim_to=0'0, roll_forward_to=42'17)) v2 2019-12-07T19:34:21.639+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=219, reqid=client.4540.0:365, at_version=44'19, trim_to=0'0, roll_forward_to=42'17)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.639+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=219, reqid=client.4540.0:365, at_version=44'19, trim_to=0'0, roll_forward_to=42'17)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.639+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.639+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=219, reqid=client.4540.0:365, at_version=44'19, trim_to=0'0, roll_forward_to=42'17)) v2) prio 127 cost 0 e44) pg 0x558c54d5f000 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 44 dequeue_op 0x558c581cd760 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=219, reqid=client.4540.0:365, at_version=44'19, trim_to=0'0, roll_forward_to=42'17)) v2 pg pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=219, reqid=client.4540.0:365, at_version=44'19, trim_to=0'0, roll_forward_to=42'17)) v2 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.add_oid 1:ecc72bdb:::smithi13140796-10:15 c,e,13,15 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:15 got.empty() 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:15 c,e,13,15 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 42'17 (0'0,42'17] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,42'17], crt=42'17) [44'18 (21'2) clone 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:33:58.631859+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,44'19 (21'2) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:365 2019-12-07T19:34:21.638575+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'18 (0'0,44'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=42'17 crt=42'17 lcod 42'17 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'18 (21'2) clone 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:33:58.631859+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=42'17 crt=42'17 lcod 42'17 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 44'19 (21'2) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:365 2019-12-07T19:34:21.638575+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=42'17 crt=42'17 lcod 42'17 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 19 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=42'17 crt=42'17 lcod 42'17 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.639+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.639+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 44'18, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c574fe300 seq 58 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ecc72bdb:::smithi13140796-10:15# key 0x848000000000000001ecc72bdb21736d'ithi13140796-10!='0x0000000000000015ffffffffffffffff'o' 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:15# 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1224 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.onode(0x558c54d97180).flush flush done 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 0x0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:15# 0x0~a000 -> 0x0~a000 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10553 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d74cb0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d74cb0 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=1)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c583eb5e0 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c583eb5e0) get_ref 0x0~2000 Blob(0x558c583eb5e0 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c583eb5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x0~2000: 0x0~2000 Blob(0x558c583eb5e0 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10554 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d74a80 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d74a80 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=1)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c5794dc70 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c5794dc70) get_ref 0x0~2000 Blob(0x558c5794dc70 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c5794dc70) get_ref init 0x2000, 1000 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c5794dc70 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10555 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d74bd0 blob([0x42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d74bd0 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=1)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c550b3880 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c550b3880) get_ref 0x0~2000 Blob(0x558c550b3880 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c550b3880) get_ref init 0x2000, 1000 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c550b3880 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10556 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d74d20 blob([0x44000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d74d20 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=1)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c5800d490 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c5800d490) get_ref 0x0~2000 Blob(0x558c5800d490 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c5800d490) get_ref init 0x2000, 1000 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5800d490 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x8000~2000: 0x0~2000 Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10557 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54d749a0 blob([0x46000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 sbid 0x0)) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54d749a0 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=1)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c583eb650 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c583eb650) get_ref 0x0~2000 Blob(0x558c583eb650 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.blob(0x558c583eb650) get_ref init 0x2000, 1000 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c583eb650 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2)))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dirty_range mark inline shard dirty 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2388) dirty_range mark inline shard dirty 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:15# = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# snapset 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# snapset = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 22 keys 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 44/41 ECSubRead(tid=82, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 44/41 ECSubRead(tid=82, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e44) pg 0x558c575ef000 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 22 keys = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.639+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c581cd340 prio 127 cost 0 latency 0.000619 MOSDECSubOpRead(1.bs1 44/41 ECSubRead(tid=82, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_000000000000000C_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpRead(1.bs1 44/41 ECSubRead(tid=82, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_000000000000000E_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_0000000000000015_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:21.639+0000 7f8f0317f700 25 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:21.639+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~1000 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x63000 (405504) 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:21.639+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000044.00000000000000000018' <- 0000000044.00000000000000000018 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552796c0 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000044.00000000000000000019' <- 0000000044.00000000000000000019 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c552796c0 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe300 cost 66902 (1 ios * 4000 + 62902 bytes) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe300 onodes 0x558c54d97180,0x558c575a2280 shared_blobs 0x558c54d3ce70,0x558c54d74930,0x558c54d74a10,0x558c54d74af0,0x558c54d74c40 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) update inline shard 138 bytes from 5 extents 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 58072 (57928 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2388) update 4#1:ecc72bdb:::smithi13140796-10:15# 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2388) update inline shard 138 bytes from 5 extents 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:15# is 556 (412 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293c is 11 SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293d is 11 SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2939 is 11 SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293a is 11 SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293b is 11 SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2))) 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe300 allocated 0x[] released 0x[] 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 prepare 2019-12-07T19:34:21.639+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe300 2019-12-07T19:34:21.639+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 io_done 2019-12-07T19:34:21.639+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=42'17 crt=42'17 lcod 42'17 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 44'19 2019-12-07T19:34:21.639+0000 7f8f05183700 10 osd.7 44 dequeue_op 0x558c581cd760 finish 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.639+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2147 < min_flush_size 524288 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 34915 < min_flush_size 524288 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 62707 < min_flush_size 524288 2019-12-07T19:34:21.639+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 330 ==== MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=220, reqid=client.4540.0:364, at_version=0'0, trim_to=0'0, roll_forward_to=44'37)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57a44000 con 0x558c54c64000 2019-12-07T19:34:21.639+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000022 epoch 44 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=220, reqid=client.4540.0:364, at_version=0'0, trim_to=0'0, roll_forward_to=44'37)) v2 2019-12-07T19:34:21.639+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=220, reqid=client.4540.0:364, at_version=0'0, trim_to=0'0, roll_forward_to=44'37)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c552796c0 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57a07e48 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=220, reqid=client.4540.0:364, at_version=0'0, trim_to=0'0, roll_forward_to=44'37)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c5759e990 space 0x558c57a07e48 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=220, reqid=client.4540.0:364, at_version=0'0, trim_to=0'0, roll_forward_to=44'37)) v2) prio 127 cost 0 e44) pg 0x558c57380000 2019-12-07T19:34:21.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~1000 = 4096 2019-12-07T19:34:21.639+0000 7f8f0898a700 10 osd.7 44 dequeue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=220, reqid=client.4540.0:364, at_version=0'0, trim_to=0'0, roll_forward_to=44'37)) v2 pg pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.639+0000 7f8f0317f700 20 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:21.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.8s5 44/41 ECSubWrite(tid=220, reqid=client.4540.0:364, at_version=0'0, trim_to=0'0, roll_forward_to=44'37)) v2 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97180 had 1 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2280 had 1 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 62730 < min_flush_size 524288 2019-12-07T19:34:21.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=36'33 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'37], crt=36'33) [] 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 62730 < min_flush_size 524288 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1df79d~f50a to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1df79d 0x1df79d~f50a to file(ino 16 size 0x1e0000 mtime 2019-12-07T19:34:20.621949+0000 allocated 1e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1b0000]) 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _allocate len 0xeca7 from 0 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _allocate len 0xeca7 from 1 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'34 (29'29) clone 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd310000 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd310000~10000/10000,10000,acd310000 2019-12-07T19:34:21.639+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x1f0000 to include allocated 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 49 (was clean) 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1c0000 x_off 0x1af79d 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x79d 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xca7 and padding block with 0x359 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'35 (29'29) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:362 2019-12-07T19:34:21.628705+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.639+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x1f0000 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'36 (44'35) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:363 2019-12-07T19:34:21.628730+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'37 (44'36) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:364 2019-12-07T19:34:21.628735+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 37 2019-12-07T19:34:21.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.639+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.639+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.639+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:21.639+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57db2600 seq 28 2019-12-07T19:34:21.640+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:head#24 onode 0x558c5840cc80 txc 0x558c57db2600 2019-12-07T19:34:21.640+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.8s5_head 5#1:131421e8:::smithi13140796-7:head#24 0x0 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c57c388c0) put_ref 0x0~2000 Blob(0x558c57c388c0 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ea540) put_ref 0x0~2000 Blob(0x558c583ea540 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ea700) put_ref 0x0~2000 Blob(0x558c583ea700 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ea7e0) put_ref 0x0~2000 Blob(0x558c583ea7e0 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ea8c0) put_ref 0x0~2000 Blob(0x558c583ea8c0 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ea9a0) put_ref 0x0~2000 Blob(0x558c583ea9a0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583eaa80) put_ref 0x0~2000 Blob(0x558c583eaa80 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583eab60) put_ref 0x0~2000 Blob(0x558c583eab60 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583eac40) put_ref 0x0~2000 Blob(0x558c583eac40 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ead20) put_ref 0x0~2000 Blob(0x558c583ead20 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1eeca7 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583eb960) put_ref 0x0~2000 Blob(0x558c583eb960 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583eba40) put_ref 0x0~2000 Blob(0x558c583eba40 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebb20) put_ref 0x0~2000 Blob(0x558c583ebb20 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebc00) put_ref 0x0~2000 Blob(0x558c583ebc00 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebce0) put_ref 0x0~2000 Blob(0x558c583ebce0 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebdc0) put_ref 0x0~1000 Blob(0x558c583ebdc0 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840cd88) dirty_range mark inline shard dirty 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57c388c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x83a000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c583ea540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x83c000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c583ea700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x83e000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c583ea7e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x840000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c583ea8c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x842000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c583ea9a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x844000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (49) on file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]), flushing log 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c583eaa80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x846000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c583eab60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x848000~2000] 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 49 len 0x49 crc 0xa7e49fbb) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c583eac40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84a000~2000] 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x30000~1000 to file(ino 1 size 0x30000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x30000 0x30000~1000 to file(ino 1 size 0x30000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c583ead20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x31000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84c000~2000] 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x30000 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c583eb960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84e000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c583eba40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x850000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c583ebb20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x852000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c583ebc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x854000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c583ebce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x856000~2000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~1000: 0x0~1000 Blob(0x558c583ebdc0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=3)))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x858000~1000] 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2))) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x31000 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:21.640+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:head#24 = 0 2019-12-07T19:34:21.640+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.640+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:21.640+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db2600 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db2600 onodes shared_blobs 0x558c577f2000,0x558c583ea690,0x558c583ea770,0x558c583ea850,0x558c583ea930,0x558c583eaa10,0x558c583eaaf0,0x558c583eabd0,0x558c583eacb0,0x558c583ead90,0x558c583eb9d0,0x558c583ebab0,0x558c583ebb90,0x558c583ebc70,0x558c583ebd50,0x558c583ebe30 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2929 is 11 SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292a is 11 SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292b is 11 SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292c is 11 SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292d is 11 SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292e is 11 SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292f is 11 SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2930 is 11 SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2931 is 11 SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2932 is 11 SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2933 is 11 SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2934 is 11 SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2935 is 11 SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2936 is 11 SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2937 is 11 SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2938 is 11 SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2))) 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db2600 allocated 0x[] released 0x[] 2019-12-07T19:34:21.640+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2600 prepare 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db2600 2019-12-07T19:34:21.640+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2600 io_done 2019-12-07T19:34:21.640+0000 7f8f0898a700 10 osd.7 44 dequeue_op 0x558c581cdb80 finish 2019-12-07T19:34:21.640+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] get_hash_info: Getting attr on 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:21.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] get_hash_info: not in cache 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:21.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:21.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000070 2019-12-07T19:34:21.640+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key = 0 2019-12-07T19:34:21.640+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 44/41 ECSubReadReply(tid=82, attrs_read=0)) v2 -- 0x558c57599180 con 0x558c54c5b400 2019-12-07T19:34:21.640+0000 7f8f0317f700 10 osd.7 44 dequeue_op 0x558c581cd340 finish 2019-12-07T19:34:21.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 49 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001174746s (0.000000091s flush + 0.001174655s kv commit) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe300> 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 kv_submitted 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe300 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:21.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe300 finishing 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe300 onodes 0x558c54d97180,0x558c575a2280 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe300 done 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:21.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe300 [] 2019-12-07T19:34:21.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:21.640+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 44/41 ECSubWriteReply(tid=219, last_complete=44'19, committed=1, applied=1)) v2 -- 0x558c57612200 con 0x558c54c64000 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840cc80 had 1 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1627 < min_flush_size 524288 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1627 < min_flush_size 524288 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1eeca7~65b to file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1eeca7 0x1eeca7~65b to file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.640+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1c0000 x_off 0x1beca7 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xca7 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x302 and padding block with 0xcfe 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1ef302 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000103843s (0.000000083s flush + 0.000103760s kv commit) 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57db2600> 2019-12-07T19:34:21.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2600 kv_submitted 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db2600 2019-12-07T19:34:21.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2600 finishing 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db2600 onodes 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db2600 done 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:21.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db2600 [] 2019-12-07T19:34:21.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.641+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.641+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 44/41 ECSubWriteReply(tid=220, last_complete=44'37, committed=1, applied=1)) v2 -- 0x558c57518c00 con 0x558c54c64000 2019-12-07T19:34:21.641+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:21.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 331 ==== MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=221, reqid=client.4540.0:365, at_version=0'0, trim_to=0'0, roll_forward_to=44'19)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57a44800 con 0x558c54c64000 2019-12-07T19:34:21.641+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c581cd1e0 prio 127 cost 0 latency 0.000016 epoch 44 MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=221, reqid=client.4540.0:365, at_version=0'0, trim_to=0'0, roll_forward_to=44'19)) v2 2019-12-07T19:34:21.641+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=221, reqid=client.4540.0:365, at_version=0'0, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e44) 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=221, reqid=client.4540.0:365, at_version=0'0, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e44) queued 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=221, reqid=client.4540.0:365, at_version=0'0, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e44) pg 0x558c54d5f000 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 osd.7 44 dequeue_op 0x558c581cd1e0 prio 127 cost 0 latency 0.000067 MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=221, reqid=client.4540.0:365, at_version=0'0, trim_to=0'0, roll_forward_to=44'19)) v2 pg pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.7s4 44/41 ECSubWrite(tid=221, reqid=client.4540.0:365, at_version=0'0, trim_to=0'0, roll_forward_to=44'19)) v2 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=42'17 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'19], crt=42'17) [] 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'18 (21'2) clone 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:33:58.631859+0000 0 snaps [15,13,e,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=44'19 (21'2) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:365 2019-12-07T19:34:21.638575+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 19 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:21.641+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740bb00 seq 59 2019-12-07T19:34:21.641+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740bb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740bb00 onodes shared_blobs 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 prepare 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740bb00 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 io_done 2019-12-07T19:34:21.641+0000 7f8f0117b700 10 osd.7 44 dequeue_op 0x558c581cd1e0 finish 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1ef302~471 to file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1ef302 0x1ef302~471 to file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1c0000 x_off 0x1bf302 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x302 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x773 and padding block with 0x88d 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1ef773 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:21.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000134249s (0.000000119s flush + 0.000134130s kv commit) 2019-12-07T19:34:21.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740bb00> 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:21.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 kv_submitted 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740bb00 2019-12-07T19:34:21.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740bb00 finishing 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740bb00 onodes 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740bb00 done 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:21.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740bb00 [] 2019-12-07T19:34:21.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:21.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:21.641+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 44/41 ECSubWriteReply(tid=221, last_complete=44'19, committed=1, applied=1)) v2 -- 0x558c5501ce00 con 0x558c54c64000 2019-12-07T19:34:21.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0259201 2019-12-07T19:34:21.662+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:21.662+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:21.662+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 33 0x2000~1000 2019-12-07T19:34:21.662+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 73ebbf02 2019-12-07T19:34:21.663+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 33 0x1b000~1000 2019-12-07T19:34:21.663+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 3000d211 2019-12-07T19:34:21.663+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.663+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:21.663+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:21.713+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.717+0000 7f8f1fbbe700 10 osd.7 44 tick_without_osd_lock 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f66a000/0x40000000/0x1680000000, data 0xad05cc/0x994000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 25 osd.7 44 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.974946+0000 last_rx_front 2019-12-07T19:34:20.975066+0000 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 25 osd.7 44 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.974976+0000 last_rx_front 2019-12-07T19:34:20.975018+0000 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 25 osd.7 44 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.975037+0000 last_rx_front 2019-12-07T19:34:20.975080+0000 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 25 osd.7 44 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.974957+0000 last_rx_front 2019-12-07T19:34:20.974987+0000 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 25 osd.7 44 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:20.974686+0000 last_rx_back 2019-12-07T19:34:20.975007+0000 last_rx_front 2019-12-07T19:34:20.975049+0000 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 20 osd.7 44 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 10 osd.7 44 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 20 osd.7 44 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:21.718+0000 7f8f1fbbe700 10 osd.7 44 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:21.763+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.813+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.844+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 10 ==== osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:21.845336+0000/45.311659633s send_stamp 45.311659633s delta_ub 0.031103597s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795ca00 con 0x558c54c5ac00 2019-12-07T19:34:21.844+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 10 ==== osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:21.845336+0000/45.311659633s send_stamp 45.311659633s delta_ub 0.031103597s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e400 con 0x558c54c5a400 2019-12-07T19:34:21.844+0000 7f8f23c56700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030703992s,0.031103597s]) 2019-12-07T19:34:21.844+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:21.845336+0000/45.311659633s send_stamp 45.280955641s delta_ub 18446744073.678847624s) v5 -- 0x558c57613c00 con 0x558c54c5ac00 2019-12-07T19:34:21.844+0000 7f8f23c56700 10 osd.7 44 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 43 -> 44 (as per caller) 2019-12-07T19:34:21.844+0000 7f8f24c58700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030678160s,0.031103597s]) 2019-12-07T19:34:21.844+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:21.845336+0000/45.311659633s send_stamp 45.280981473s delta_ub 18446744073.678873456s) v5 -- 0x558c57613600 con 0x558c54c5a400 2019-12-07T19:34:21.863+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.913+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:21.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 14 ==== osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:21.954414+0000/45.411926747s send_stamp 45.411926747s delta_ub 0.022404035s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841fe00 con 0x558c54c18000 2019-12-07T19:34:21.953+0000 7f8f23c56700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021976190s,0.022404035s]) 2019-12-07T19:34:21.953+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 14 ==== osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:21.954414+0000/45.411926747s send_stamp 45.411926747s delta_ub 0.022404035s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5501ce00 con 0x558c552d1400 2019-12-07T19:34:21.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:21.954414+0000/45.411926747s send_stamp 45.389950557s delta_ub 18446744073.687575426s) v5 -- 0x558c57613c00 con 0x558c54c18000 2019-12-07T19:34:21.953+0000 7f8f23c56700 10 osd.7 44 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 43 -> 44 (as per caller) 2019-12-07T19:34:21.953+0000 7f8f24457700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021949571s,0.022404035s]) 2019-12-07T19:34:21.953+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:21.954414+0000/45.411926747s send_stamp 45.389977176s delta_ub 18446744073.687602045s) v5 -- 0x558c57518c00 con 0x558c552d1400 2019-12-07T19:34:21.963+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.013+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.064+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.074+0000 7f8f0097a700 5 osd.7 44 heartbeat osd_stat(store_statfs(0x163f66a000/0x40000000/0x1680000000, data 0xad05cc/0x994000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5] op hist []) 2019-12-07T19:34:22.074+0000 7f8f0097a700 20 osd.7 44 check_full_status cur ratio 0.000104014, physical ratio 0.000104014, new state none 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.688183082s) v5 -- 0x558c57613e00 con 0x558c54c9e000 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.688183082s) v5 -- 0x558c58332000 con 0x558c54c9e400 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.673355196s) v5 -- 0x558c58332200 con 0x558c551b7800 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.673355196s) v5 -- 0x558c58332400 con 0x558c551b7c00 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.687602045s) v5 -- 0x558c58332600 con 0x558c552c2000 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.687602045s) v5 -- 0x558c58332800 con 0x558c552c2400 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.695836785s) v5 -- 0x558c58332a00 con 0x558c552c2800 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.695836785s) v5 -- 0x558c58332c00 con 0x558c552c2c00 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.678873456s) v5 -- 0x558c58332e00 con 0x558c54c65800 2019-12-07T19:34:22.074+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.510456485s delta_ub 18446744073.678873456s) v5 -- 0x558c58333000 con 0x558c54c65c00 2019-12-07T19:34:22.074+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 13 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.524411269s delta_ub 0.013954784s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e800 con 0x558c552c2800 2019-12-07T19:34:22.074+0000 7f8f24c58700 25 osd.7 44 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:20.974957+0000 -> 2019-12-07T19:34:22.075252+0000 last_rx_front 2019-12-07T19:34:20.974987+0000 2019-12-07T19:34:22.074+0000 7f8f24c58700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013719165s,0.013954784s]) 2019-12-07T19:34:22.074+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 9 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.541512912s delta_ub 0.031056427s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841fe00 con 0x558c54c65800 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:20.975007+0000 -> 2019-12-07T19:34:22.075263+0000 last_rx_front 2019-12-07T19:34:20.975049+0000 2019-12-07T19:34:22.074+0000 7f8f23c56700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030810649s,0.031056427s]) 2019-12-07T19:34:22.074+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 13 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.524472780s delta_ub 0.014016295s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5501ce00 con 0x558c552c2c00 2019-12-07T19:34:22.074+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 9 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.541539374s delta_ub 0.031082889s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841fe00 con 0x558c54c65c00 2019-12-07T19:34:22.074+0000 7f8f24457700 25 osd.7 44 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075252+0000 last_rx_front 2019-12-07T19:34:20.974987+0000 -> 2019-12-07T19:34:22.075295+0000 2019-12-07T19:34:22.074+0000 7f8f24457700 25 osd.7 44 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:22.075017+0000) and older pending ping(s) 2019-12-07T19:34:22.074+0000 7f8f24457700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013738042s,0.014016295s]) 2019-12-07T19:34:22.074+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 9 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.532230738s delta_ub 0.021774253s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e800 con 0x558c54c9e400 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075263+0000 last_rx_front 2019-12-07T19:34:20.975049+0000 -> 2019-12-07T19:34:22.075319+0000 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:22.075017+0000) and older pending ping(s) 2019-12-07T19:34:22.074+0000 7f8f23c56700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030781162s,0.031082889s]) 2019-12-07T19:34:22.074+0000 7f8f24c58700 25 osd.7 44 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:20.974946+0000 last_rx_front 2019-12-07T19:34:20.975066+0000 -> 2019-12-07T19:34:22.075339+0000 2019-12-07T19:34:22.074+0000 7f8f24c58700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021451670s,0.021774253s]) 2019-12-07T19:34:22.074+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 13 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.532873213s delta_ub 0.022416728s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5501ce00 con 0x558c552c2000 2019-12-07T19:34:22.074+0000 7f8f24457700 25 osd.7 44 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:20.975037+0000 -> 2019-12-07T19:34:22.075351+0000 last_rx_front 2019-12-07T19:34:20.975080+0000 2019-12-07T19:34:22.074+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 13 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.532817650s delta_ub 0.022361165s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841fe00 con 0x558c552c2400 2019-12-07T19:34:22.074+0000 7f8f24457700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022082372s,0.022416728s]) 2019-12-07T19:34:22.074+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 13 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.547024406s delta_ub 0.036567921s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e800 con 0x558c551b7c00 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075351+0000 last_rx_front 2019-12-07T19:34:20.975080+0000 -> 2019-12-07T19:34:22.075363+0000 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:22.075017+0000) and older pending ping(s) 2019-12-07T19:34:22.074+0000 7f8f23c56700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022015059s,0.022361165s]) 2019-12-07T19:34:22.074+0000 7f8f24c58700 25 osd.7 44 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:20.974976+0000 last_rx_front 2019-12-07T19:34:20.975018+0000 -> 2019-12-07T19:34:22.075379+0000 2019-12-07T19:34:22.074+0000 7f8f24c58700 10 osd.7 44 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 43 -> 44 (as per caller) 2019-12-07T19:34:22.074+0000 7f8f24c58700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036205461s,0.036567921s]) 2019-12-07T19:34:22.074+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 9 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.532252757s delta_ub 0.021796272s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841fe00 con 0x558c54c9e000 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:20.974946+0000 -> 2019-12-07T19:34:22.075394+0000 last_rx_front 2019-12-07T19:34:22.075339+0000 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:22.075017+0000) and older pending ping(s) 2019-12-07T19:34:22.074+0000 7f8f23c56700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021418989s,0.021796272s]) 2019-12-07T19:34:22.074+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 13 ==== osd_ping(ping_reply e44 up_from 13 ping_stamp 2019-12-07T19:34:22.075017+0000/45.510456485s send_stamp 45.547071640s delta_ub 0.036615155s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841fe00 con 0x558c551b7800 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:20.974976+0000 -> 2019-12-07T19:34:22.075420+0000 last_rx_front 2019-12-07T19:34:22.075379+0000 2019-12-07T19:34:22.074+0000 7f8f23c56700 25 osd.7 44 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:22.075017+0000) and older pending ping(s) 2019-12-07T19:34:22.074+0000 7f8f23c56700 20 osd.7 44 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036212356s,0.036615155s]) 2019-12-07T19:34:22.113+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.164+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.201+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:22.210+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c5751d000 0x558c54d05180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:22.210+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c5751d000 0x558c54d05180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:22.210+0000 7f8f23c56700 10 osd.7 44 ms_handle_authentication new session 0x558c57405380 con 0x558c5751d000 entity client.admin addr 2019-12-07T19:34:22.210+0000 7f8f23c56700 10 osd.7 44 ms_handle_authentication session 0x558c57405380 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:22.210+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/563364556 conn(0x558c5751d000 0x558c54d05180 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4834 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:22.211+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4834 172.21.15.131:0/563364556 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c57edf860 con 0x558c5751d000 2019-12-07T19:34:22.211+0000 7f8f164bb700 20 osd.7 44 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:22.211+0000 7f8f164bb700 10 osd.7 44 do_waiters -- start 2019-12-07T19:34:22.211+0000 7f8f164bb700 10 osd.7 44 do_waiters -- finish 2019-12-07T19:34:22.211+0000 7f8f164bb700 20 osd.7 44 _dispatch 0x558c57edf860 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:22.211+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/563364556 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce16c0 con 0x558c5751d000 2019-12-07T19:34:22.214+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.234+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4834 172.21.15.131:0/563364556 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c57edf680 con 0x558c5751d000 2019-12-07T19:34:22.234+0000 7f8f164bb700 20 osd.7 44 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:22.234+0000 7f8f164bb700 10 osd.7 44 do_waiters -- start 2019-12-07T19:34:22.234+0000 7f8f164bb700 10 osd.7 44 do_waiters -- finish 2019-12-07T19:34:22.234+0000 7f8f164bb700 20 osd.7 44 _dispatch 0x558c57edf680 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:22.235+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/563364556 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce1880 con 0x558c5751d000 2019-12-07T19:34:22.238+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/563364556 conn(0x558c5751d000 msgr2=0x558c54d05180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 69 2019-12-07T19:34:22.238+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/563364556 conn(0x558c5751d000 msgr2=0x558c54d05180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:22.238+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/563364556 conn(0x558c5751d000 0x558c54d05180 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:22.238+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/563364556 conn(0x558c5751d000 0x558c54d05180 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:22.238+0000 7f8f164bb700 2 osd.7 44 ms_handle_reset con 0x558c5751d000 session 0x558c57405380 2019-12-07T19:34:22.264+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.314+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.353+0000 7f8f21451700 10 osd.7 44 tick 2019-12-07T19:34:22.353+0000 7f8f21451700 10 osd.7 44 do_waiters -- start 2019-12-07T19:34:22.353+0000 7f8f21451700 10 osd.7 44 do_waiters -- finish 2019-12-07T19:34:22.353+0000 7f8f21451700 20 osd.7 44 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:22.364+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.414+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.464+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.514+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.564+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.614+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1192162 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 219 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=83, reqid=osd.4.0:83, at_version=45'23, trim_to=0'0, roll_forward_to=44'21)) v2 ==== 3458+0+0 (crc 0 0 0) 0x558c575bf800 con 0x558c54c5b400 2019-12-07T19:34:22.623+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c581cc6e0 prio 127 cost 0 latency 0.000042 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=83, reqid=osd.4.0:83, at_version=45'23, trim_to=0'0, roll_forward_to=44'21)) v2 2019-12-07T19:34:22.623+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=83, reqid=osd.4.0:83, at_version=45'23, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.623+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 218 ==== osd_map(45..45 src has 1..45) v4 ==== 661+0+0 (crc 0 0 0) 0x558c57351680 con 0x558c54c5b400 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 osd.7 44 OSD::ms_dispatch: osd_map(45..45 src has 1..45) v4 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 osd.7 44 do_waiters -- start 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 osd.7 44 do_waiters -- finish 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 osd.7 44 _dispatch 0x558c57351680 osd_map(45..45 src has 1..45) v4 2019-12-07T19:34:22.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 220 ==== MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=84, reqid=osd.4.0:84, at_version=45'17, trim_to=0'0, roll_forward_to=42'15)) v2 ==== 3362+0+0 (crc 0 0 0) 0x558c5831d000 con 0x558c54c5b400 2019-12-07T19:34:22.623+0000 7f8f1199c700 3 osd.7 44 handle_osd_map epochs [45,45], i have 44, src has [1,45] 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 osd.7 44 handle_osd_map got inc map for epoch 45 2019-12-07T19:34:22.623+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c581cc840 prio 127 cost 0 latency 0.000026 epoch 45 MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=84, reqid=osd.4.0:84, at_version=45'17, trim_to=0'0, roll_forward_to=42'15)) v2 2019-12-07T19:34:22.623+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=84, reqid=osd.4.0:84, at_version=45'17, trim_to=0'0, roll_forward_to=42'15)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.623+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3668e826:::osdmap.44:0# 0x0~0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574faa10 blob([0x40e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57dcb9d0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ac070 blob([!~1000,0x40f000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c55279340 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3668e826:::osdmap.44:0# 0x0~1448 = 5192 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 osd.7 44 add_map_bl 44 5192 bytes 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=83, reqid=osd.4.0:83, at_version=45'23, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=83, reqid=osd.4.0:83, at_version=45'23, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c581cc6e0 prio 127 cost 0 latency 0.000132 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=83, reqid=osd.4.0:83, at_version=45'23, trim_to=0'0, roll_forward_to=44'21)) v2 pg pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 44 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 44 -> 45 (as per caller) 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=83, reqid=osd.4.0:83, at_version=45'23, trim_to=0'0, roll_forward_to=44'21)) v2 2019-12-07T19:34:22.623+0000 7f8f07187700 20 snap_mapper.update_snaps 1:dce63cc3:::smithi13140796-50:15 e,13,15 was 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.623+0000 7f8f07187700 20 snap_mapper.get_snaps 1:dce63cc3:::smithi13140796-50:15 c,e,13,15 2019-12-07T19:34:22.623+0000 7f8f07187700 20 snap_mapper.set_snaps 1:dce63cc3:::smithi13140796-50:15 e,13,15 2019-12-07T19:34:22.623+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:22.623+0000 7f8f07187700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 44'21 (0'0,44'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'21], crt=44'21) [45'22 (44'13) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:22.623726+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'23 (44'16) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:22.623726+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'22 (0'0,45'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'21 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'22 (44'13) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:22.623726+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 osd.7 44 got_full_map 45, nothing requested 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'21 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'23 (44'16) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:22.623726+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'21 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 23 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'21 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.623+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.623+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'22, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 osd.7 44 handle_osd_map pg_num_history pg_num_history(e45 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {45={}} 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 osd.7 44 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e45 [1,45] lci=[13,45]) 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c580d2300 seq 36 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c477972f:::inc_osdmap.45:0# key 0x7f7fffffffffffffffc47797'/!inc_osdmap.45!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c580d3b00 seq 25 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys 2019-12-07T19:34:22.623+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c477972f:::inc_osdmap.45:0# 0x0~269 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1225 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c477972f:::inc_osdmap.45:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2300 1 blobs 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x996000~1000/1000,1000,0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x996000~1000] 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060850 blob([0x996000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91260 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c55060850) get_ref 0x0~269 Blob(0x558c55060850 blob([0x996000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91260 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c55060850) get_ref init 0x1000, 1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c55060850 blob([0x996000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91260 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57f91278 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9a888) dirty_range mark inline shard dirty 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c477972f:::inc_osdmap.45:0# 0x0~269 = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3ce8e826:::osdmap.45:0# key 0x7f7fffffffffffffff3ce8e8'&!osdmap.45!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ce8e826:::osdmap.45:0# 0x0~1464 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1226 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3ce8e826:::osdmap.45:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2300 2 blobs 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x997000~2000/1000,2000,0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x997000~2000] 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39110 blob([0x997000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39180 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c57c39110) get_ref 0x0~1000 Blob(0x558c57c39110 blob([0x997000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39180 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c57c39110) get_ref init 0x1000, 1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57c39110 blob([0x997000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c39180 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c39198 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c391f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c391f0 blob([!~1000,0x998000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39260 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c57c391f0) get_ref 0x1000~464 Blob(0x558c57c391f0 blob([!~1000,0x998000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57c39260 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c57c391f0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c57c391f0 blob([!~1000,0x998000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57c39260 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c39278 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c574a0888) dirty_range mark inline shard dirty 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ce8e826:::osdmap.45:0# 0x0~1464 = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2300 0 blobs 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2300 0 blobs 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d2300 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d2300 onodes 0x558c53a91900,0x558c55276c80,0x558c574a0780,0x558c57d9a780 shared_blobs 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c574a0888) update #-1:3ce8e826:::osdmap.45:0# 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c574a0888) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3ce8e826:::osdmap.45:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9a888) update #-1:c477972f:::inc_osdmap.45:0# 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57d9a888) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c477972f:::inc_osdmap.45:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_000000000000000C_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d2300 allocated 0x[996000~3000] released 0x[] 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000022' <- 0000000045.00000000000000000022 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 freelist allocate 0x996000~3000 2019-12-07T19:34:22.623+0000 7f8f1199c700 20 freelist _xor first_key 0x980000 last_key 0x980000 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000023' <- 0000000045.00000000000000000023 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2300 prepare 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:22.623+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c580d2300 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d3b00 cost 8429 (1 ios * 4000 + 4429 bytes) 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d3b00 onodes 0x558c574a0a00,0x558c574a0c80 shared_blobs 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) update 1#1:dce63cc3:::smithi13140796-50:15# 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:15# is 1120 (412 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 1281 (573 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d3b00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3b00 prepare 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d3b00 2019-12-07T19:34:22.623+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3b00 io_done 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=44'21 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'23 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c581cc6e0 finish 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=84, reqid=osd.4.0:84, at_version=45'17, trim_to=0'0, roll_forward_to=42'15)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.623+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=84, reqid=osd.4.0:84, at_version=45'17, trim_to=0'0, roll_forward_to=42'15)) v2) prio 127 cost 0 e45) pg 0x558c5513e000 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c581cc840 prio 127 cost 0 latency 0.000548 MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=84, reqid=osd.4.0:84, at_version=45'17, trim_to=0'0, roll_forward_to=42'15)) v2 pg pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.623+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=84, reqid=osd.4.0:84, at_version=45'17, trim_to=0'0, roll_forward_to=42'15)) v2 2019-12-07T19:34:22.623+0000 7f8f07187700 20 snap_mapper.update_snaps 1:c8ce3553:::smithi13140796-29:e e was 2019-12-07T19:34:22.623+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.623+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:22.623+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2300 aio_wait 2019-12-07T19:34:22.623+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d2300 2019-12-07T19:34:22.623+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2300 io_done 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2189 < min_flush_size 524288 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5287 < min_flush_size 524288 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3858825216 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998271) bluestore 86 GiB free (0.959895), bluefs_ratio 0.0399382 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5310 < min_flush_size 524288 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5310 < min_flush_size 524288 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1ef773~14be to file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1ef773 0x1ef773~14be to file(ino 16 size 0x1f0000 mtime 2019-12-07T19:34:21.641103+0000 allocated 1f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1c0000]) 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _allocate len 0xc31 from 0 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _allocate len 0xc31 from 1 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd320000 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd320000~10000/10000,10000,acd320000 2019-12-07T19:34:22.623+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x200000 to include allocated 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 50 (was clean) 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1bf773 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x773 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc31 and padding block with 0x3cf 2019-12-07T19:34:22.623+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x200000 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.get_snaps 1:c8ce3553:::smithi13140796-29:e c,e 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.set_snaps 1:c8ce3553:::smithi13140796-29:e e 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 42'15 (0'0,42'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,42'15], crt=42'15) [45'16 (42'14) modify 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:22.623849+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'17 (42'15) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:22.623849+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 45'16 (0'0,45'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=42'15 crt=42'15 lcod 42'15 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'16 (42'14) modify 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:22.623849+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=42'15 crt=42'15 lcod 42'15 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'17 (42'15) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:22.623849+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=42'15 crt=42'15 lcod 42'15 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 17 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=42'15 crt=42'15 lcod 42'15 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.624+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.624+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c583c8000 seq 53 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 1 keys 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# 1 keys = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000C_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000045.00000000000000000016' <- 0000000045.00000000000000000016 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000045.00000000000000000017' <- 0000000045.00000000000000000017 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c8000 cost 8333 (1 ios * 4000 + 4333 bytes) 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c8000 onodes 0x558c57976500,0x558c57976780 shared_blobs 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976608) update 4#1:c8ce3553:::smithi13140796-29:e# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:e# is 823 (412 bytes onode + 2 bytes spanning blobs + 409 bytes inline extents) 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore.extentmap(0x558c57976888) update 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:head# is 911 (541 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c8000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8000 prepare 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c8000 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8000 io_done 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=42'15 crt=42'15 lcod 42'15 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'17 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c581cc840 finish 2019-12-07T19:34:22.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f0c31 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (50) on file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]), flushing log 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 50 len 0x4a crc 0xf1804b5e) 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x31000~1000 to file(ino 1 size 0x31000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x31000 0x31000~1000 to file(ino 1 size 0x31000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x32000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x31000 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x32000 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:22.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 221 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=85, reqid=osd.4.0:85, at_version=45'25, trim_to=0'0, roll_forward_to=44'21)) v2 ==== 3458+0+0 (crc 0 0 0) 0x558c5831d800 con 0x558c54c5b400 2019-12-07T19:34:22.624+0000 7f8f24457700 15 osd.7 44 enqueue_op 0x558c581cc9a0 prio 127 cost 0 latency 0.000030 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=85, reqid=osd.4.0:85, at_version=45'25, trim_to=0'0, roll_forward_to=44'21)) v2 2019-12-07T19:34:22.624+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=85, reqid=osd.4.0:85, at_version=45'25, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000095 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=85, reqid=osd.4.0:85, at_version=45'25, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=85, reqid=osd.4.0:85, at_version=45'25, trim_to=0'0, roll_forward_to=44'21)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c581cc9a0 prio 127 cost 0 latency 0.000117 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=85, reqid=osd.4.0:85, at_version=45'25, trim_to=0'0, roll_forward_to=44'21)) v2 pg pg[1.bs1( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=85, reqid=osd.4.0:85, at_version=45'25, trim_to=0'0, roll_forward_to=44'21)) v2 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.update_snaps 1:db85c185:::smithi13140796-18:15 e,13,15 was 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 50 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000795581s (0.000000144s flush + 0.000795437s kv commit) 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0780 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57d9a780 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d3b00> 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13340 < min_flush_size 524288 2019-12-07T19:34:22.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3b00 kv_submitted 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d3b00 2019-12-07T19:34:22.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3b00 finishing 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d3b00 onodes 0x558c574a0a00,0x558c574a0c80 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d3b00 done 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:22.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d3b00 [] 2019-12-07T19:34:22.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976500 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976780 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13399 < min_flush_size 524288 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13399 < min_flush_size 524288 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f0c31~3457 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f0c31 0x1f0c31~3457 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c0c31 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc31 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x88 and padding block with 0xf78 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.get_snaps 1:db85c185:::smithi13140796-18:15 c,e,13,15 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.set_snaps 1:db85c185:::smithi13140796-18:15 e,13,15 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:22.624+0000 7f8f07187700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'23], crt=44'21) [45'24 (44'17) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:22.624660+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'25 (44'21) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:22.624660+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'24 (0'0,45'24] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'23 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'24 (44'17) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:22.624660+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'23 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'25 (44'21) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:22.624660+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'23 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 25 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'23 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.624+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.624+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'24, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c583c8300 seq 26 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_000000000000000C_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000024' <- 0000000045.00000000000000000024 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000025' <- 0000000045.00000000000000000025 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c8300 cost 7639 (1 ios * 4000 + 3639 bytes) 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c8300 onodes 0x558c57573680,0x558c57e8ec80 shared_blobs 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore.extentmap(0x558c57573788) update 1#1:db85c185:::smithi13140796-18:15# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:15# is 664 (412 bytes onode + 2 bytes spanning blobs + 250 bytes inline extents) 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1603 (574 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c8300 allocated 0x[] released 0x[] 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8300 prepare 2019-12-07T19:34:22.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c8300 2019-12-07T19:34:22.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8300 io_done 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'23 crt=44'21 lcod 44'21 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'25 2019-12-07T19:34:22.624+0000 7f8f07187700 10 osd.7 44 dequeue_op 0x558c581cc9a0 finish 2019-12-07T19:34:22.624+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 44/41 ECSubWriteReply(tid=83, last_complete=45'23, committed=1, applied=1)) v2 -- 0x558c5795dc00 con 0x558c54c5b400 2019-12-07T19:34:22.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f4088 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 1 in 0.000444562s (0.000005147s flush + 0.000439415s kv commit) 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4352 < min_flush_size 524288 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573680 had 1 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4375 < min_flush_size 524288 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4375 < min_flush_size 524288 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f4088~1117 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f4088 0x1f4088~1117 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c4088 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x88 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x19f and padding block with 0xe61 2019-12-07T19:34:22.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 187 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52000 con 0x558c54c5bc00 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d2300,0x558c583c8000> 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c58141400> 2019-12-07T19:34:22.625+0000 7f8f23c56700 15 osd.7 44 enqueue_peering_evt 1.1s1 epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2300 kv_submitted 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d2300 2019-12-07T19:34:22.625+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c580d2300 osr 0x558c54ea4b40 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 34 0x1b000~1000 crc 2adf7eec 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 34 0x2000~1000 crc c8d0ab3f 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8000 kv_submitted 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c8000 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8000 finishing 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c8000 onodes 0x558c57976500,0x558c57976780 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8000 done 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c8000 [] 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7 44 _committed_osd_maps 45..45 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58600 deferred_cleanup 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7 44 advance to epoch 45 (<= last 45 <= newest_map 45) 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58600 onodes 0x558c53a91900,0x558c55276c80,0x558c5738c780,0x558c5738cf00 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efca28 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4f710 space 0x558c57efca28 0x0~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efcb78 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4f7a0 space 0x558c57efcb78 0x0~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efcc58 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4f830 space 0x558c57efcc58 0x1000~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f141a1700 7 osd.7 45 consume_map version 45 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58600 done 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cde000 deferred_cleanup 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58600 [] 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.8s5 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.0s3 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.1s1 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde000 deferred_cleanup 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.9s4 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cde000 onodes 0x558c53a91900,0x558c55276c80,0x558c57a34f00,0x558c57a35180 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.as5 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38cc8 in 0x558c53a59340) _finish_write discard buffer(0x558c580cd3b0 space 0x558c57c38cc8 0x0~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.2s0 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.bs1 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38e18 in 0x558c53a59340) _finish_write discard buffer(0x558c580cd320 space 0x558c57c38e18 0x0~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.3s4 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.cs0 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38ef8 in 0x558c53a59340) _finish_write discard buffer(0x558c580cd290 space 0x558c57c38ef8 0x1000~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.4s0 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cde000 done 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a56000 deferred_cleanup 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.5s4 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cde000 [] 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.ds2 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.es4 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.6s2 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.fs5 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56000 deferred_cleanup 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 identify_splits_and_merges 1.7s4 e44 to e45 pg_nums {18=16} 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a56000 onodes 0x558c53a91900,0x558c55276c80,0x558c57d9a000,0x558c57d9aa00 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 45 prune_pg_created 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3d828 in 0x558c53a59340) _finish_write discard buffer(0x558c580cca20 space 0x558c54d3d828 0x0~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:0.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55279358 in 0x558c53a59340) _finish_write discard buffer(0x558c551a9e60 space 0x558c55279358 0x1000~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:1.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57dcb9e8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a94d0 space 0x558c57dcb9e8 0x0~1000 writing nocache) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:2.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a56000 done 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2300 deferred_queued 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a56000 [] 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:3.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:4.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:5.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:6.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7:7.consume_map 45 (was 44) 2019-12-07T19:34:22.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:22.625+0000 7f8f141a1700 10 osd.7 45 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.8s5 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e45) pg 0x558c5736a000 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.0s3 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.1s1 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.9s4 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f519f 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000271875s (0.000000071s flush + 0.000271804s kv commit) 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.625+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 44/41 ECSubWriteReply(tid=84, last_complete=45'17, committed=1, applied=1)) v2 -- 0x558c5795d000 con 0x558c54c5b400 2019-12-07T19:34:22.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 44 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.as5 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.2s0 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.bs1 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.3s4 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.cs0 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.4s0 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.5s4 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.ds2 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.es4 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.6s2 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.fs5 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f141a1700 15 osd.7 45 enqueue_peering_evt 1.7s4 epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) 2019-12-07T19:34:22.625+0000 7f8f141a1700 7 osd.7 45 activate_map version 45 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c57e8b000 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c57676000 2019-12-07T19:34:22.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:22.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c57cdf800 seq 15 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:22.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 44 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:22.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:22.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:22.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cdf800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cdf800 onodes shared_blobs 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cdf800 allocated 0x[] released 0x[] 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 44 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:22.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf800 prepare 2019-12-07T19:34:22.625+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:22.625+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cdf800 2019-12-07T19:34:22.625+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf800 io_done 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean ps=[a~1,d~1,10~2]] state: Active advmap 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] on_active_advmap new removed_snaps [c~1], snap_trimq now [c~1] 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c5500a000 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c5736a000 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 44 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c57e88000 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] state: Active: handling ActMap 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] Active: kicking snap trim 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 44 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,d~1,10~2]] state: Active advmap 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] on_active_advmap new removed_snaps [c~1], snap_trimq now [c~1] 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] exit NotTrimming 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming 2019-12-07T19:34:22.625+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/WaitReservation 2019-12-07T19:34:22.625+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c54d98000 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c57380000 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f06986700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:134 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f06986700 7 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] state: handle ActMap primary 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 44 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] state: Active: handling ActMap 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f06986700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 44:135 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] Active: kicking snap trim 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] exit NotTrimming 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c548ab000 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c575ef000 2019-12-07T19:34:22.625+0000 7f8f0b990700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/WaitReservation 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f0b990700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming: trimming c 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/WaitReservation 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f03980700 15 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:122 2019-12-07T19:34:22.625+0000 7f8f0b990700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:136 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:22.625+0000 7f8f03980700 7 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] state: handle ActMap primary 2019-12-07T19:34:22.625+0000 7f8f0b990700 10 osd.7 45 queueing pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] for snaptrim 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 44 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 44 while current is 45 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=45) prio 5 cost 1048576 e45) 2019-12-07T19:34:22.625+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f03980700 15 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 44:123 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean ps=[a~1,d~1,10~2]] state: Active advmap 2019-12-07T19:34:22.625+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=45) prio 5 cost 1048576 e45) queued 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f0b990700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:137 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] on_active_advmap new removed_snaps [c~1], snap_trimq now [c~1] 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c5500d000 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=45) prio 5 cost 1048576 e45) pg 0x558c57676000 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] snap_trimmer posting 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0b990700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c57522000 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap c 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 44 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:22.625+0000 7f8f0b990700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming: trimming c 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/WaitReservation 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 44 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:22.625+0000 7f8f0b990700 15 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:124 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 44 while current is 45 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 lcod 45'23 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f0b990700 10 osd.7 45 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] for snaptrim 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=45) prio 5 cost 1048576 e45) 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f900) upper_bound after SNA_1_000000000000000C_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C' 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] state: Active: handling ActMap 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c5513e000 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f900) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..' 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] Active: kicking snap trim 2019-12-07T19:34:22.625+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:22.625+0000 7f8f0b990700 15 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:125 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000C_.0_0000000000000001.C) returns 0 SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239760 in raw 0x558c55239760 len 69 nref 1) ) 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.625+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.625+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=45) prio 5 cost 1048576 e45) queued 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 44 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:22.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] exit NotTrimming 2019-12-07T19:34:22.625+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=45) prio 5 cost 1048576 e45) pg 0x558c57e88000 2019-12-07T19:34:22.625+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f900) upper_bound after SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1.. key 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..' 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] snap_trimmer posting 2019-12-07T19:34:22.625+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:22.625+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap c 2019-12-07T19:34:22.625+0000 7f8f0918b700 20 osd.7 45 agent_entry empty queue 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.625+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.625+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57d1f000) upper_bound after SNA_1_000000000000000C_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.2' 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57d1f000) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f519f~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000C_.0_0000000000000001.2) returns 0 SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..,buffer::list(len=70, buffer::ptr(0~70 0x558c552382c0 in raw 0x558c552382c0 len 70 nref 1) ) 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f519f 0x1f519f~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c519f 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x19f 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 44 while current is 45 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x634 and padding block with 0x9cc 2019-12-07T19:34:22.625+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.625+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,d~1,10~2]] SnapTrimmer state: adding snap c to purged_snaps 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.625+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f900) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..) returns 0 SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239760 in raw 0x558c55239760 len 70 nref 1) ) 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.625+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:22.625+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/WaitReservation 2019-12-07T19:34:22.625+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,c~2,10~2]] SnapTrimmer state: purged_snaps now [a~1,c~2,10~2], snap_trimq now [] 2019-12-07T19:34:22.625+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c54ea1800 seq 21 2019-12-07T19:34:22.625+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c55141000 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork react trimming 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.625+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:22.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: found obc in cache: 0x558c5738e6c0 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:22.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.625+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:22.625+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c57383000 2019-12-07T19:34:22.625+0000 7f8f06986700 15 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 43:106 2019-12-07T19:34:22.625+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1800 onodes shared_blobs 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1800 allocated 0x[] released 0x[] 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: 0x558c5738e6c0 1:3c88320e:::smithi13140796-1:c rwstate(none n=0 w=0) oi: 1:3c88320e:::smithi13140796-1:c(35'19 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6480 snapset: c=[]:{c=[c]} 2019-12-07T19:34:22.625+0000 7f8f06986700 7 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] state: handle ActMap primary 2019-12-07T19:34:22.625+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 prepare 2019-12-07T19:34:22.625+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1800 2019-12-07T19:34:22.625+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 io_done 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 44 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 44 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: found obc in cache: 0x558c5738e240 2019-12-07T19:34:22.625+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.625+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,c~2,10~2]] share_pg_info 2019-12-07T19:34:22.625+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.625+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 44 -> 45 2019-12-07T19:34:22.625+0000 7f8f06986700 15 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 44:107 2019-12-07T19:34:22.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.626+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: 0x558c5738e240 1:3c88320e:::smithi13140796-1:head rwstate(none n=0 w=0) oi: 1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6480 snapset: c=[]:{c=[c]} 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:22.626+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c57a53500 con 0x558c54c64000 2019-12-07T19:34:22.626+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.626+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583c8300> 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8300 kv_submitted 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3c88320e:::smithi13140796-1:c old_snaps c old snapset c=[]:{c=[c]} 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c8300 2019-12-07T19:34:22.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8300 finishing 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.626+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c8300 onodes 0x558c57573680,0x558c57e8ec80 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8300 done 2019-12-07T19:34:22.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c8300 [] 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] simple_opc_create 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.626+0000 7f8f0217d700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.626+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:22.626+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c57a53c00 con 0x558c54c19800 2019-12-07T19:34:22.626+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3c88320e:::smithi13140796-1:c snaps c -> ... deleting 2019-12-07T19:34:22.626+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c57a54300 con 0x558c54c5bc00 2019-12-07T19:34:22.626+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.626+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3c88320e:::smithi13140796-1:c new snapset c=[]:{} on 1:3c88320e:::smithi13140796-1:head(35'22 client.4540.0:279 dirty|data_digest s 1212416 uv 22 dd f05001eb alloc_hint [0 0 0]) 2019-12-07T19:34:22.626+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.626+0000 7f8f05984700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.626+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c57a54a00 con 0x558c54c5b400 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f5634 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3c88320e:::smithi13140796-1:c writing updated snapset on 1:3c88320e:::smithi13140796-1:head, snapset is c=[]:{} 2019-12-07T19:34:22.626+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.626+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000201395s (0.000000089s flush + 0.000201306s kv commit) 2019-12-07T19:34:22.626+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c576df500 con 0x558c54c19c00 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.626+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.626+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] new_repop rep_tid 31 (no op) 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:22.626+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.626+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f07988700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,c~2,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:22.626+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] new_repop: repgather(0x558c576b6c60 0'0 rep_tid=31 committed?=0 r=0) 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57cdf800> 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.626+0000 7f8f07988700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[a~1,c~2,10~2]] exit Trimming 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf800 kv_submitted 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] simple_opc_submit 0x558c576b6c60 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cdf800 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f5634~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.626+0000 7f8f0297e700 7 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] issue_repop rep_tid 31 o 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.626+0000 7f8f07988700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,c~2,10~2] 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdf800 finishing 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cdf800 onodes 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f5634 0x1f5634~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cdf800 done 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cdf800 [] 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.626+0000 7f8f07988700 15 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] publish_stats_to_osd 45:126 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c5634 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x634 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xac9 and padding block with 0x537 2019-12-07T19:34:22.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.626+0000 7f8f07988700 20 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] enter NotTrimming 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] submit_transaction: op Op(1:3c88320e:::smithi13140796-1:c v=45'28 tt=0'0 tid=31 reqid=osd.7.0:31 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:22.626+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:22.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.626+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:22.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.626+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# 2019-12-07T19:34:22.626+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] snap_trimmer complete 2019-12-07T19:34:22.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: found on disk, size 8192 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# hinfo_key 2019-12-07T19:34:22.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.626+0000 7f8f0b990700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:22.626+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# hinfo_key = 0 2019-12-07T19:34:22.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0b990700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: NotTrimming: trimming c 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): delete, setting projected size to 0 2019-12-07T19:34:22.626+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim_wait trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/WaitReservation 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3c88320e:::smithi13140796-1:c projected size 0 2019-12-07T19:34:22.626+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.626+0000 7f8f0b990700 15 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:108 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:22.626+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:22.626+0000 7f8f0b990700 10 osd.7 45 queueing pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean trimq=[c~1] ps=[a~1,d~1,10~2]] for snaptrim 2019-12-07T19:34:22.626+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=45) prio 5 cost 1048576 e45) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: found on disk, size 303104 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key = 0 2019-12-07T19:34:22.626+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 45/41 ECSubWriteReply(tid=85, last_complete=45'25, committed=1, applied=1)) v2 -- 0x558c5795da00 con 0x558c54c5b400 2019-12-07T19:34:22.626+0000 7f8f0b990700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3c88320e:::smithi13140796-1:head projected size 1212416 2019-12-07T19:34:22.626+0000 7f8f0b990700 15 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:109 2019-12-07T19:34:22.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] start_rmw: Op(1:3c88320e:::smithi13140796-1:c v=45'28 tt=0'0 tid=31 reqid=osd.7.0:31 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_state_to_reads: Op(1:3c88320e:::smithi13140796-1:c v=45'28 tt=0'0 tid=31 reqid=osd.7.0:31 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: starting commit on Op(1:3c88320e:::smithi13140796-1:c v=45'28 tt=0'0 tid=31 reqid=osd.7.0:31 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f5ac9 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] generate_transactions: 1:3c88320e:::smithi13140796-1:c, current size is 32768 buffers are {} 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000182637s (0.000000081s flush + 0.000182556s kv commit) 2019-12-07T19:34:22.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): new_size start 0 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=45) prio 5 cost 1048576 e45) queued 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_overwrite: {} 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=45) prio 5 cost 1048576 e45) pg 0x558c548ab000 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_append: {} 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 43 while current is 45 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3c88320e:::smithi13140796-1:c resetting hinfo to logical size 0 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] snap_trimmer posting 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap c 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] generate_transactions: 1:3c88320e:::smithi13140796-1:head, current size is 1212416 buffers are {} 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 NullEvt) prio 255 cost 10 e45) pg 0x558c54d5f000 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.626+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1800> 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 kv_submitted 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1800 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1800 finishing 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1800 onodes 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1800 done 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1800 [] 2019-12-07T19:34:22.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map: 45 2019-12-07T19:34:22.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 44 -> 45 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): new_size start 1212416 2019-12-07T19:34:22.626+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57d1fbc0) upper_bound after SNA_1_000000000000000C_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.4' 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 44 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_overwrite: {} 2019-12-07T19:34:22.626+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57d1fbc0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_append: {} 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] state: Started advmap 2019-12-07T19:34:22.626+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000C_.0_0000000000000001.4) returns 0 SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:22.626+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3c88320e:::smithi13140796-1:head resetting hinfo to logical size 1212416 2019-12-07T19:34:22.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.626+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:22.626+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:22.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] plpg_on_pool_change 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.626+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57d1fbc0) upper_bound after SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. key 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] hit_set_clear 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written: {1:3c88320e:::smithi13140796-1:c={},1:3c88320e:::smithi13140796-1:head={}} 2019-12-07T19:34:22.626+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57d1fbc0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..' 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] agent_stop 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 45 advance_pg new pool opts old pool opts 2019-12-07T19:34:22.626+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..) returns 0 SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239810 in raw 0x558c55239810 len 69 nref 1) ) 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] handle_activate_map: 45 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: op: Op(1:3c88320e:::smithi13140796-1:c v=45'28 tt=0'0 tid=31 reqid=osd.7.0:31 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] activate_map 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork react trimming 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written_set: {1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[]} 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] activate_map: Not dirtying info: last_persisted is 44 while current is 45 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: found obc in cache: 0x558c5524c480 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: 1:3c88320e:::smithi13140796-1:c,{} 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 NullEvt 2019-12-07T19:34:22.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:22.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: 1:3c88320e:::smithi13140796-1:head,{} 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: 0x558c5524c480 1:276d5164:::smithi13140796-22:e rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:e(42'5 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e,c]} 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: found obc in cache: 0x558c57945440 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: 0x558c57945440 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(42'6 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e,c]} 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:276d5164:::smithi13140796-22:e old_snaps c,e old snapset e=[]:{e=[e,c]} 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 send_incremental_map 44 -> 45 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] simple_opc_create 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c477972f:::inc_osdmap.45:0# 0x0~0 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:276d5164:::smithi13140796-22:e snaps c,e -> e 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55060850 blob([0x996000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91260 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c477972f:::inc_osdmap.45:0# 0x0~269 = 617 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 add_map_inc_bl 45 617 bytes 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(45..45 src has 1..45) v4 -- 0x558c575d4480 con 0x558c54c64000 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:276d5164:::smithi13140796-22:e new snapset e=[]:{e=[e]} on 1:276d5164:::smithi13140796-22:head(42'6 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 44 -> 45 (shared) 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:276d5164:::smithi13140796-22:e writing updated snapset on 1:276d5164:::smithi13140796-22:head, snapset is e=[]:{e=[e]} 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 45/28 ECSubWrite(tid=31, reqid=osd.7.0:31, at_version=45'28, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c54e41000 con 0x558c54c64000 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 send_incremental_map 44 -> 45 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(45..45 src has 1..45) v4 -- 0x558c54d2f680 con 0x558c54c19800 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] new_repop rep_tid 32 (no op) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 44 -> 45 (shared) 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] new_repop: repgather(0x558c576b6360 0'0 rep_tid=32 committed?=0 r=0) 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 45/28 ECSubWrite(tid=31, reqid=osd.7.0:31, at_version=45'28, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c574a2800 con 0x558c54c19800 2019-12-07T19:34:22.626+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] simple_opc_submit 0x558c576b6360 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 send_incremental_map 44 -> 45 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:22.626+0000 7f8f06986700 7 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] issue_repop rep_tid 32 o 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(45..45 src has 1..45) v4 -- 0x558c54d2efc0 con 0x558c54c5bc00 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 44 -> 45 (shared) 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 45/28 ECSubWrite(tid=31, reqid=osd.7.0:31, at_version=45'28, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c574a2000 con 0x558c54c5bc00 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] submit_transaction: op Op(1:276d5164:::smithi13140796-22:e v=45'8 tt=0'0 tid=32 reqid=osd.7.0:32 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 45/28 ECSubWrite(tid=31, reqid=osd.7.0:31, at_version=45'28, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c57e00000 con 0x558c54c5b400 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 send_incremental_map 44 -> 45 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: found on disk, size 90112 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(45..45 src has 1..45) v4 -- 0x558c57e03b00 con 0x558c54c19c00 2019-12-07T19:34:22.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 45 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 44 -> 45 (shared) 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key = 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 45/28 ECSubWrite(tid=31, reqid=osd.7.0:31, at_version=45'28, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c57e00800 con 0x558c54c19c00 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 snap_mapper.remove_oid 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:276d5164:::smithi13140796-22:e projected size 360448 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 snap_mapper._remove_oid 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:3c88320e:::smithi13140796-1:c c 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 snap_mapper.clear_snaps 1:3c88320e:::smithi13140796-1:c 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 snap_mapper.clear_snaps rm OBJ_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:22.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:34:22.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 snap_mapper._remove_oid rm SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:276d5164:::smithi13140796-22:head projected size 311296 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 trim 45'27 (35'19) delete 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 trim 45'28 (35'22) modify 1:3c88320e:::smithi13140796-1:head by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] start_rmw: Op(1:276d5164:::smithi13140796-22:e v=45'8 tt=0'0 tid=32 reqid=osd.7.0:32 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_state_to_reads: Op(1:276d5164:::smithi13140796-22:e v=45'8 tt=0'0 tid=32 reqid=osd.7.0:32 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 38'26 (0'0,38'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log log((0'0,38'26], crt=38'26) [45'27 (35'19) delete 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'28 (35'22) modify 1:3c88320e:::smithi13140796-1:head by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.626+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:e v=45'8 tt=0'0 tid=32 reqid=osd.7.0:32 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'27 (0'0,45'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] add_log_entry 45'27 (35'19) delete 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] generate_transactions: 1:276d5164:::smithi13140796-22:e, current size is 360448 buffers are {} 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] add_log_entry 45'28 (35'22) modify 1:3c88320e:::smithi13140796-1:head by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log approx pg log length = 28 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): new_size start 360448 2019-12-07T19:34:22.626+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'27, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_overwrite: {} 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_append: {} 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:276d5164:::smithi13140796-22:e resetting hinfo to logical size 360448 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] generate_transactions: 1:276d5164:::smithi13140796-22:head, current size is 311296 buffers are {} 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57ff0000 seq 37 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): new_size start 311296 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3c88320e:::smithi13140796-1:c#1b key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0x000000000000000c000000000000001b'o' 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_overwrite: {} 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_append: {} 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:276d5164:::smithi13140796-22:head resetting hinfo to logical size 311296 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# -> 0#1:3c88320e:::smithi13140796-1:c#1b 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c# -> 0#1:3c88320e:::smithi13140796-1:c#1b = 0 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written: {1:276d5164:::smithi13140796-22:e={},1:276d5164:::smithi13140796-22:head={}} 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:e v=45'8 tt=0'0 tid=32 reqid=osd.7.0:32 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys = 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written_set: {1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]} 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:e,{} 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C311C407.c.smithi13140796-1..' <- OBJ_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.626+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:head,{} 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1..' <- SNA_1_000000000000000C_.0_0000000000000001.C311C407.c.smithi13140796-1.. 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000045.00000000000000000027' <- 0000000045.00000000000000000027 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000045.00000000000000000028' <- 0000000045.00000000000000000028 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0000 cost 7540 (1 ios * 4000 + 3540 bytes) 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0000 onodes 0x558c5738d400,0x558c574a0f00 shared_blobs 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore.extentmap(0x558c5738d508) update 0#1:3c88320e:::smithi13140796-1:c#1b 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3c88320e:::smithi13140796-1:c#1b is 445 (409 bytes onode + 2 bytes spanning blobs + 34 bytes inline extents) 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1008) update 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3c88320e:::smithi13140796-1:head# is 1238 (483 bytes onode + 2 bytes spanning blobs + 753 bytes inline extents) 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 45/41 ECSubWrite(tid=32, reqid=osd.7.0:32, at_version=45'8, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c5748e800 con 0x558c54c64000 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 prepare 2019-12-07T19:34:22.626+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0000 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 io_done 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] op_applied version 45'28 2019-12-07T19:34:22.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 45/41 ECSubWrite(tid=32, reqid=osd.7.0:32, at_version=45'8, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c57e7b800 con 0x558c54c19800 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] eval_repop repgather(0x558c576b6c60 45'28 rep_tid=31 committed?=0 r=0) (no op) 2019-12-07T19:34:22.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.4s1 45/41 ECSubWrite(tid=32, reqid=osd.7.0:32, at_version=45'8, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c57e7c800 con 0x558c54c5bc00 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] calc_trim_to_aggressive limit = 38'26 2019-12-07T19:34:22.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 45/41 ECSubWrite(tid=32, reqid=osd.7.0:32, at_version=45'8, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c57e7a000 con 0x558c54c5b400 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork react trimming 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:22.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 45/41 ECSubWrite(tid=32, reqid=osd.7.0:32, at_version=45'8, trim_to=0'0, roll_forward_to=42'6)) v2 -- 0x558c57e7d000 con 0x558c54c19c00 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: found obc in cache: 0x558c576b2b40 2019-12-07T19:34:22.626+0000 7f8f06986700 20 snap_mapper.update_snaps 1:276d5164:::smithi13140796-22:e e was 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: 0x558c576b2b40 1:3a5a3f5b:::smithi13140796-23:e rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:e(38'25 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e,c]} 2019-12-07T19:34:22.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.626+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(38'26 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e,c]} 2019-12-07T19:34:22.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3a5a3f5b:::smithi13140796-23:e old_snaps c,e old snapset e=[]:{e=[e,c]} 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] simple_opc_create 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3a5a3f5b:::smithi13140796-23:e snaps c,e -> e 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3a5a3f5b:::smithi13140796-23:e new snapset e=[]:{e=[e]} on 1:3a5a3f5b:::smithi13140796-23:head(38'26 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 1:3a5a3f5b:::smithi13140796-23:e writing updated snapset on 1:3a5a3f5b:::smithi13140796-23:head, snapset is e=[]:{e=[e]} 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] new_repop rep_tid 33 (no op) 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] new_repop: repgather(0x558c576b6ea0 0'0 rep_tid=33 committed?=0 r=0) 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] simple_opc_submit 0x558c576b6ea0 2019-12-07T19:34:22.627+0000 7f8f0297e700 7 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] issue_repop rep_tid 33 o 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:e v=45'30 tt=0'0 tid=33 reqid=osd.7.0:33 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3a5a3f5b:::smithi13140796-23:e projected size 1474560 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=45'30 tt=0'0 tid=33 reqid=osd.7.0:33 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4144 < min_flush_size 524288 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:e v=45'30 tt=0'0 tid=33 reqid=osd.7.0:33 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=45'30 tt=0'0 tid=33 reqid=osd.7.0:33 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738d400 had 1 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0f00 had 1 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2c80 had 1 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4167 < min_flush_size 524288 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4167 < min_flush_size 524288 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f5ac9~1047 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f5ac9 0x1f5ac9~1047 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:e, current size is 1474560 buffers are {} 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c5ac9 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xac9 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb10 and padding block with 0x4f0 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): new_size start 1474560 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_overwrite: {} 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_append: {} 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3a5a3f5b:::smithi13140796-23:e resetting hinfo to logical size 1474560 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): new_size start 507904 2019-12-07T19:34:22.627+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 131 ==== MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=58, reqid=osd.2.0:58, at_version=45'12, trim_to=0'0, roll_forward_to=43'10)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c57e7b800 con 0x558c54c19800 2019-12-07T19:34:22.627+0000 7f8f24c58700 15 osd.7 45 enqueue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000025 epoch 45 MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=58, reqid=osd.2.0:58, at_version=45'12, trim_to=0'0, roll_forward_to=43'10)) v2 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_overwrite: {} 2019-12-07T19:34:22.627+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=58, reqid=osd.2.0:58, at_version=45'12, trim_to=0'0, roll_forward_to=43'10)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): to_append: {} 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.627+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.627+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=58, reqid=osd.2.0:58, at_version=45'12, trim_to=0'0, roll_forward_to=43'10)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:e={},1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:22.627+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 130 ==== osd_map(45..45 src has 1..45) v4 ==== 661+0+0 (crc 0 0 0) 0x558c5781c6c0 con 0x558c54c19800 2019-12-07T19:34:22.627+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.627+0000 7f8f1199c700 20 osd.7 45 OSD::ms_dispatch: osd_map(45..45 src has 1..45) v4 2019-12-07T19:34:22.627+0000 7f8f1199c700 10 osd.7 45 do_waiters -- start 2019-12-07T19:34:22.627+0000 7f8f1199c700 10 osd.7 45 do_waiters -- finish 2019-12-07T19:34:22.627+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=58, reqid=osd.2.0:58, at_version=45'12, trim_to=0'0, roll_forward_to=43'10)) v2) prio 127 cost 0 e45) pg 0x558c55141000 2019-12-07T19:34:22.627+0000 7f8f1199c700 20 osd.7 45 _dispatch 0x558c5781c6c0 osd_map(45..45 src has 1..45) v4 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:e v=45'30 tt=0'0 tid=33 reqid=osd.7.0:33 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:22.627+0000 7f8f1199c700 3 osd.7 45 handle_osd_map epochs [45,45], i have 45, src has [1,45] 2019-12-07T19:34:22.627+0000 7f8f1199c700 10 osd.7 45 no new maps here, dropping 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:22.627+0000 7f8f06185700 10 osd.7 45 dequeue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000067 MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=58, reqid=osd.2.0:58, at_version=45'12, trim_to=0'0, roll_forward_to=43'10)) v2 pg pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:e,{} 2019-12-07T19:34:22.627+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=58, reqid=osd.2.0:58, at_version=45'12, trim_to=0'0, roll_forward_to=43'10)) v2 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:34:22.627+0000 7f8f06185700 20 snap_mapper.update_snaps 1:bf81b261:::smithi13140796-37:13 e,13 was 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f06185700 20 snap_mapper.get_snaps 1:bf81b261:::smithi13140796-37:13 c,e,13 2019-12-07T19:34:22.627+0000 7f8f06185700 20 snap_mapper.set_snaps 1:bf81b261:::smithi13140796-37:13 e,13 2019-12-07T19:34:22.627+0000 7f8f06185700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:22.627+0000 7f8f06185700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:22.627+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 45/28 ECSubWrite(tid=33, reqid=osd.7.0:33, at_version=45'30, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c58154000 con 0x558c54c64000 2019-12-07T19:34:22.627+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 45/28 ECSubWrite(tid=33, reqid=osd.7.0:33, at_version=45'30, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c58154800 con 0x558c54c19800 2019-12-07T19:34:22.627+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 43'10 (0'0,43'10] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,43'10], crt=43'10) [45'11 (43'9) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:22.627548+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'12 (43'10) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:22.627548+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.627+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'11 (0'0,45'11] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=43'10 crt=43'10 lcod 43'10 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'11 (43'9) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:22.627548+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 45/28 ECSubWrite(tid=33, reqid=osd.7.0:33, at_version=45'30, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c58155000 con 0x558c54c5bc00 2019-12-07T19:34:22.627+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=43'10 crt=43'10 lcod 43'10 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'12 (43'10) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:22.627548+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 45/28 ECSubWrite(tid=33, reqid=osd.7.0:33, at_version=45'30, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c58155800 con 0x558c54c5b400 2019-12-07T19:34:22.627+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=43'10 crt=43'10 lcod 43'10 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 12 2019-12-07T19:34:22.627+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=43'10 crt=43'10 lcod 43'10 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.627+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.627+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 45/28 ECSubWrite(tid=33, reqid=osd.7.0:33, at_version=45'30, trim_to=0'0, roll_forward_to=38'26)) v2 -- 0x558c58156000 con 0x558c54c19c00 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 snap_mapper.update_snaps 1:3a5a3f5b:::smithi13140796-23:e e was 2019-12-07T19:34:22.627+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:e c,e 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 snap_mapper.set_snaps 1:3a5a3f5b:::smithi13140796-23:e e 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 trim 45'29 (38'25) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5815e000 seq 39 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 trim 45'30 (38'26) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'28 (0'0,45'28] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log log((0'0,45'28], crt=38'26) [45'29 (38'25) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'30 (38'26) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'29 (0'0,45'29] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=45'28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] add_log_entry 45'29 (38'25) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys = 0 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=45'28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] add_log_entry 45'30 (38'26) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=45'28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log approx pg log length = 30 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=45'28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.627+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'29, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 20 snap_mapper.get_snaps 1:276d5164:::smithi13140796-22:e c,e 2019-12-07T19:34:22.627+0000 7f8f06986700 20 snap_mapper.set_snaps 1:276d5164:::smithi13140796-22:e e 2019-12-07T19:34:22.627+0000 7f8f06986700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57ff0300 seq 38 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 1 keys 2019-12-07T19:34:22.627+0000 7f8f06986700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 1 keys = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key (42 bytes) 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 20 trim 45'7 (42'5) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 20 trim 45'8 (42'6) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- SNA_1_000000000000000C_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000045.00000000000000000029' <- 0000000045.00000000000000000029 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000045.00000000000000000030' <- 0000000045.00000000000000000030 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 42'6 (0'0,42'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log log((0'0,42'6], crt=42'6) [45'7 (42'5) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'8 (42'6) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0300 cost 7525 (1 ios * 4000 + 3525 bytes) 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0300 onodes 0x558c57f35b80,0x558c58092000 shared_blobs 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) update 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:e# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58092108) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 851 (541 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0300 allocated 0x[] released 0x[] 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'7 (0'0,45'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 lua=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] add_log_entry 45'7 (42'5) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 prepare 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0300 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 io_done 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 lua=45'28 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] op_applied version 45'30 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 lua=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] add_log_entry 45'8 (42'6) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 lua=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log approx pg log length = 8 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] eval_repop repgather(0x558c576b6ea0 45'30 rep_tid=33 committed?=0 r=0) (no op) 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 lua=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] calc_trim_to_aggressive limit = 38'26 2019-12-07T19:34:22.627+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/WaitRepops 2019-12-07T19:34:22.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] snap_trimmer complete 2019-12-07T19:34:22.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.627+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54ea0f00 seq 34 2019-12-07T19:34:22.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 1 keys 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 1 keys = 0 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f6b10 2019-12-07T19:34:22.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000623261s (0.000000053s flush + 0.000623208s kv commit) 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3281 < min_flush_size 524288 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000C_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000045.00000000000000000007' <- 0000000045.00000000000000000007 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000045.00000000000000000008' <- 0000000045.00000000000000000008 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35b80 had 1 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092000 had 1 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3304 < min_flush_size 524288 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3304 < min_flush_size 524288 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0f00 cost 8333 (1 ios * 4000 + 4333 bytes) 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0f00 onodes 0x558c57f35900,0x558c58093400 shared_blobs 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57f35a08) update 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f6b10~ce8 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:head# is 737 (541 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:34:22.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f6b10 0x1f6b10~ce8 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58093508) update 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c6b10 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb10 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:e# is 718 (412 bytes onode + 2 bytes spanning blobs + 304 bytes inline extents) 2019-12-07T19:34:22.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7f8 and padding block with 0x808 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 prepare 2019-12-07T19:34:22.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0f00 2019-12-07T19:34:22.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 io_done 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 lua=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] op_applied version 45'8 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0000> 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 kv_submitted 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0000 2019-12-07T19:34:22.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0000 finishing 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0000 onodes 0x558c5738d400,0x558c574a0f00 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] eval_repop repgather(0x558c576b6360 45'8 rep_tid=32 committed?=0 r=0) (no op) 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0000 done 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0300 kv_submitted 2019-12-07T19:34:22.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0000 [] 2019-12-07T19:34:22.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] calc_trim_to_aggressive limit = 42'6 2019-12-07T19:34:22.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.627+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:22.627+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/WaitRepops 2019-12-07T19:34:22.627+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] snap_trimmer complete 2019-12-07T19:34:22.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 61 2019-12-07T19:34:22.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.627+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=READY pgs=11 cs=1 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_000000000000000C_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:22.627+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.627+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000045.00000000000000000011' <- 0000000045.00000000000000000011 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000045.00000000000000000012' <- 0000000045.00000000000000000012 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:22.627+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815e000 cost 8375 (1 ios * 4000 + 4375 bytes) 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815e000 onodes 0x558c54d96c80,0x558c57976000 shared_blobs 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore.extentmap(0x558c54d96d88) update 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:34:22.628+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=READY pgs=11 cs=1 l=0 rx=0 tx=0)._fault initiating reconnect 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head# is 483 (475 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore.extentmap(0x558c57976108) update 2#1:bf81b261:::smithi13140796-37:13# 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:13# is 931 (412 bytes onode + 2 bytes spanning blobs + 517 bytes inline extents) 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815e000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e000 prepare 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815e000 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e000 io_done 2019-12-07T19:34:22.628+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=43'10 crt=43'10 lcod 43'10 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'12 2019-12-07T19:34:22.628+0000 7f8f06185700 10 osd.7 45 dequeue_op 0x558c578d2dc0 finish 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f77f8 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000300739s (0.000000081s flush + 0.000300658s kv commit) 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2056 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4283 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35900 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093400 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8548 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8571 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8571 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f77f8~217b to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 132 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a53c00 con 0x558c54c19800 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f77f8 0x1f77f8~217b to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c77f8 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7f8 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x973 and padding block with 0x68d 2019-12-07T19:34:22.628+0000 7f8f24c58700 15 osd.7 45 enqueue_peering_evt 1.5s4 epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:22.628+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e45) pg 0x558c57522000 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ff0300> 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 kv_submitted 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0300 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0300 finishing 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0300 onodes 0x558c57f35b80,0x558c58092000 2019-12-07T19:34:22.628+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0300 done 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0300 [] 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.628+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.628+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:22.628+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=BANNER_CONNECTING pgs=11 cs=2 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5815e300 seq 14 2019-12-07T19:34:22.628+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815e300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815e300 onodes shared_blobs 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815e300 allocated 0x[] released 0x[] 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e300 prepare 2019-12-07T19:34:22.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815e300 2019-12-07T19:34:22.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e300 io_done 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f9973 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000204647s (0.000000060s flush + 0.000204587s kv commit) 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0f00,0x558c5815e000> 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 kv_submitted 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0f00 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0f00 finishing 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0f00 onodes 0x558c57f35900,0x558c58093400 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0f00 done 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:22.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0f00 [] 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e000 kv_submitted 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815e000 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e000 finishing 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815e000 onodes 0x558c54d96c80,0x558c57976000 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815e000 done 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815e000 [] 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f9973~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f9973 0x1f9973~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c9973 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x973 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe08 and padding block with 0x1f8 2019-12-07T19:34:22.628+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 45/41 ECSubWriteReply(tid=58, last_complete=45'12, committed=1, applied=1)) v2 -- 0x558c5841ea00 con 0x558c54c19800 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1f9e08 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000102839s (0.000000040s flush + 0.000102799s kv commit) 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5815e300> 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e300 kv_submitted 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815e300 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e300 finishing 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815e300 onodes 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815e300 done 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815e300 [] 2019-12-07T19:34:22.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f1199c700 10 osd.7 45 ms_handle_connect con 0x558c54c19c00 2019-12-07T19:34:22.628+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 crc :-1 s=READY pgs=11 cs=2 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=e020c6588efc571e server_cookie=4bffdf0ee71c7be4 in_seq=86 out_seq=114 2019-12-07T19:34:22.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 188 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52000 con 0x558c54c5bc00 2019-12-07T19:34:22.628+0000 7f8f23c56700 15 osd.7 45 enqueue_peering_evt 1.fs5 epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:22.628+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e45) pg 0x558c57383000 2019-12-07T19:34:22.628+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:22.628+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.628+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 189 ==== osd_map(45..45 src has 1..45) v4 ==== 661+0+0 (crc 0 0 0) 0x558c54d2efc0 con 0x558c54c5bc00 2019-12-07T19:34:22.628+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:22.628+0000 7f8f1199c700 20 osd.7 45 OSD::ms_dispatch: osd_map(45..45 src has 1..45) v4 2019-12-07T19:34:22.628+0000 7f8f1199c700 10 osd.7 45 do_waiters -- start 2019-12-07T19:34:22.628+0000 7f8f1199c700 10 osd.7 45 do_waiters -- finish 2019-12-07T19:34:22.628+0000 7f8f1199c700 20 osd.7 45 _dispatch 0x558c54d2efc0 osd_map(45..45 src has 1..45) v4 2019-12-07T19:34:22.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:22.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c574fe000 seq 17 2019-12-07T19:34:22.628+0000 7f8f1199c700 3 osd.7 45 handle_osd_map epochs [45,45], i have 45, src has [1,45] 2019-12-07T19:34:22.628+0000 7f8f1199c700 10 osd.7 45 no new maps here, dropping 2019-12-07T19:34:22.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 190 ==== MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=124, reqid=osd.3.0:124, at_version=45'46, trim_to=0'0, roll_forward_to=43'44)) v2 ==== 3385+0+0 (crc 0 0 0) 0x558c58155000 con 0x558c54c5bc00 2019-12-07T19:34:22.628+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:22.628+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000026 epoch 45 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=124, reqid=osd.3.0:124, at_version=45'46, trim_to=0'0, roll_forward_to=43'44)) v2 2019-12-07T19:34:22.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:22.628+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=124, reqid=osd.3.0:124, at_version=45'46, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:22.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:22.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:22.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe000 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:22.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe000 onodes shared_blobs 2019-12-07T19:34:22.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 prepare 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.628+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe000 2019-12-07T19:34:22.628+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 io_done 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=124, reqid=osd.3.0:124, at_version=45'46, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 191 ==== MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=125, reqid=osd.3.0:125, at_version=45'48, trim_to=0'0, roll_forward_to=43'44)) v2 ==== 3426+0+0 (crc 0 0 0) 0x558c57e7c800 con 0x558c54c5bc00 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=124, reqid=osd.3.0:124, at_version=45'46, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.628+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c578d3760 prio 127 cost 0 latency 0.000015 epoch 45 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=125, reqid=osd.3.0:125, at_version=45'48, trim_to=0'0, roll_forward_to=43'44)) v2 2019-12-07T19:34:22.628+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=125, reqid=osd.3.0:125, at_version=45'48, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=125, reqid=osd.3.0:125, at_version=45'48, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1f9e08~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1f9e08 0x1f9e08~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 osd.7 45 dequeue_op 0x558c578d34a0 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=124, reqid=osd.3.0:124, at_version=45'46, trim_to=0'0, roll_forward_to=43'44)) v2 pg pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1c9e08 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe08 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x29d and padding block with 0xd63 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=124, reqid=osd.3.0:124, at_version=45'46, trim_to=0'0, roll_forward_to=43'44)) v2 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6c205177:::smithi13140796-8:10 e was 2019-12-07T19:34:22.628+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:10 c,e 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6c205177:::smithi13140796-8:10 e 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 43'44 (0'0,43'44] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,43'44], crt=43'44) [45'45 (41'36) modify 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:22.626290+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'46 (41'37) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:22.626290+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1fa29d 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'45 (0'0,45'45] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=43'44 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'45 (41'36) modify 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:22.626290+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129557s (0.000000085s flush + 0.000129472s kv commit) 2019-12-07T19:34:22.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'46 (0'0,45'46] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=43'44 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'46 (41'37) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:22.626290+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'46 (0'0,45'46] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=43'44 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 46 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'46 (0'0,45'46] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=43'44 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.628+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'45, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:22.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57b95500 seq 77 2019-12-07T19:34:22.628+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# 1 keys 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# 1 keys = 0 2019-12-07T19:34:22.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# hinfo_key (42 bytes) 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:22.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- SNA_1_000000000000000C_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.629+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000045.00000000000000000045' <- 0000000045.00000000000000000045 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000045.00000000000000000046' <- 0000000045.00000000000000000046 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe000> 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 kv_submitted 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe000 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe000 finishing 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57b95500 cost 8356 (1 ios * 4000 + 4356 bytes) 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe000 onodes 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57b95500 onodes 0x558c574a1b80,0x558c575ba280 shared_blobs 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe000 done 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore.extentmap(0x558c574a1c88) update 2#1:6c205177:::smithi13140796-8:10# 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe000 [] 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.629+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:10# is 687 (411 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:22.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 1584 (555 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57b95500 allocated 0x[] released 0x[] 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 prepare 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57b95500 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 io_done 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'46 (0'0,45'46] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=43'44 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'46 2019-12-07T19:34:22.629+0000 7f8f0197c700 10 osd.7 45 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5097 < min_flush_size 524288 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1b80 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5120 < min_flush_size 524288 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5120 < min_flush_size 524288 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1fa29d~1400 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1fa29d 0x1fa29d~1400 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1ca29d 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x29d 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x69d and padding block with 0x963 2019-12-07T19:34:22.629+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.629+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=125, reqid=osd.3.0:125, at_version=45'48, trim_to=0'0, roll_forward_to=43'44)) v2) prio 127 cost 0 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 45 dequeue_op 0x558c578d3760 prio 127 cost 0 latency 0.000623 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=125, reqid=osd.3.0:125, at_version=45'48, trim_to=0'0, roll_forward_to=43'44)) v2 pg pg[1.6s2( v 45'46 (0'0,45'46] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'46 (0'0,45'46] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=125, reqid=osd.3.0:125, at_version=45'48, trim_to=0'0, roll_forward_to=43'44)) v2 2019-12-07T19:34:22.629+0000 7f8f05984700 20 snap_mapper.update_snaps 1:6ad15c4c:::smithi13140796-31:13 e,13 was 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 20 snap_mapper.get_snaps 1:6ad15c4c:::smithi13140796-31:13 c,e,13 2019-12-07T19:34:22.629+0000 7f8f05984700 20 snap_mapper.set_snaps 1:6ad15c4c:::smithi13140796-31:13 e,13 2019-12-07T19:34:22.629+0000 7f8f05984700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:22.629+0000 7f8f05984700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'46 (0'0,45'46] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'46], crt=43'44) [45'47 (43'43) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:22.627413+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'48 (43'44) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:22.627413+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'47 (0'0,45'47] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'46 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'47 (43'43) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:22.627413+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'46 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'48 (43'44) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:22.627413+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'46 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 48 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'46 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.629+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.629+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'47, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55114000 seq 78 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:22.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 332 ==== MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=222, reqid=osd.1.0:222, at_version=45'30, trim_to=0'0, roll_forward_to=43'28)) v2 ==== 3417+0+0 (crc 0 0 0) 0x558c58156000 con 0x558c54c64000 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.000028 epoch 45 MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=222, reqid=osd.1.0:222, at_version=45'30, trim_to=0'0, roll_forward_to=43'28)) v2 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_000000000000000C_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:22.629+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=222, reqid=osd.1.0:222, at_version=45'30, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000045.00000000000000000047' <- 0000000045.00000000000000000047 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000045.00000000000000000048' <- 0000000045.00000000000000000048 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55114000 cost 7599 (1 ios * 4000 + 3599 bytes) 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55114000 onodes 0x558c54ce2280,0x558c57e8f900 shared_blobs 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:22.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 222 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c54c5b400 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 581 (579 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore.extentmap(0x558c57e8fa08) update 2#1:6ad15c4c:::smithi13140796-31:13# 2019-12-07T19:34:22.629+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5740c000 prio 196 cost 0 latency 0.000011 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:13# is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:22.629+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55114000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114000 prepare 2019-12-07T19:34:22.629+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55114000 2019-12-07T19:34:22.629+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114000 io_done 2019-12-07T19:34:22.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 223 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795da00 con 0x558c54c5b400 2019-12-07T19:34:22.629+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5740de40 prio 196 cost 0 latency 0.000007 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.629+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'46 crt=43'44 lcod 43'44 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'48 2019-12-07T19:34:22.629+0000 7f8f05984700 10 osd.7 45 dequeue_op 0x558c578d3760 finish 2019-12-07T19:34:22.629+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=222, reqid=osd.1.0:222, at_version=45'30, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1fb69d 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=222, reqid=osd.1.0:222, at_version=45'30, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e45) pg 0x558c5500a000 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000522152s (0.000000122s flush + 0.000522030s kv commit) 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3484 < min_flush_size 524288 2019-12-07T19:34:22.629+0000 7f8f04181700 10 osd.7 45 dequeue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.000133 MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=222, reqid=osd.1.0:222, at_version=45'30, trim_to=0'0, roll_forward_to=43'28)) v2 pg pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.629+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=222, reqid=osd.1.0:222, at_version=45'30, trim_to=0'0, roll_forward_to=43'28)) v2 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f900 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3507 < min_flush_size 524288 2019-12-07T19:34:22.629+0000 7f8f04181700 20 snap_mapper.update_snaps 1:9694a0e5:::smithi13140796-3:13 e,13 was 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3507 < min_flush_size 524288 2019-12-07T19:34:22.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1fb69d~db3 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1fb69d 0x1fb69d~db3 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1cb69d 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x69d 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x450 and padding block with 0xbb0 2019-12-07T19:34:22.629+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:13 c,e,13 2019-12-07T19:34:22.629+0000 7f8f04181700 20 snap_mapper.set_snaps 1:9694a0e5:::smithi13140796-3:13 e,13 2019-12-07T19:34:22.629+0000 7f8f04181700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:22.629+0000 7f8f04181700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57b95500> 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 kv_submitted 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57b95500 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57b95500 finishing 2019-12-07T19:34:22.629+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 43'28 (0'0,43'28] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,43'28], crt=43'28) [45'29 (43'25) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:22.629711+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'30 (43'26) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:22.629711+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57b95500 onodes 0x558c574a1b80,0x558c575ba280 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57b95500 done 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114000 kv_submitted 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57b95500 [] 2019-12-07T19:34:22.629+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5740c000 prio 196 cost 0 latency 0.000192 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.629+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'29 (0'0,45'29] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'28 crt=43'28 lcod 43'28 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'29 (43'25) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:22.629711+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.629+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5740c000 finish 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.629+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'30 (0'0,45'30] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'28 crt=43'28 lcod 43'28 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'30 (43'26) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:22.629711+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.629+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'30 (0'0,45'30] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'28 crt=43'28 lcod 43'28 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 30 2019-12-07T19:34:22.629+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'30 (0'0,45'30] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'28 crt=43'28 lcod 43'28 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.629+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.629+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5740de40 prio 196 cost 0 latency 0.000207 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.629+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5740de40 finish 2019-12-07T19:34:22.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.629+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'29, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1fc450 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000187049s (0.000000100s flush + 0.000186949s kv commit) 2019-12-07T19:34:22.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:22.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c54ddf800 seq 62 2019-12-07T19:34:22.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys = 0 2019-12-07T19:34:22.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:22.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_000000000000000C_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:22.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.630+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000045.00000000000000000029' <- 0000000045.00000000000000000029 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000045.00000000000000000030' <- 0000000045.00000000000000000030 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 87 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f200 con 0x558c54c19c00 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57f3f340 prio 196 cost 0 latency 0.000017 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:22.630+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddf800 cost 8396 (1 ios * 4000 + 4396 bytes) 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddf800 onodes 0x558c54ce2a00,0x558c54e45680 shared_blobs 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 579 (577 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore.extentmap(0x558c54e45788) update 4#1:9694a0e5:::smithi13140796-3:13# 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:13# is 906 (411 bytes onode + 2 bytes spanning blobs + 493 bytes inline extents) 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddf800 allocated 0x[] released 0x[] 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 prepare 2019-12-07T19:34:22.630+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddf800 2019-12-07T19:34:22.630+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 io_done 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.630+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'30 (0'0,45'30] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=43'28 crt=43'28 lcod 43'28 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'30 2019-12-07T19:34:22.630+0000 7f8f04181700 10 osd.7 45 dequeue_op 0x558c581ccdc0 finish 2019-12-07T19:34:22.630+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c57f3f340 prio 196 cost 0 latency 0.000053 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 333 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:22.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.es4 epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c57f3f340 finish 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 334 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55045200 con 0x558c54c64000 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57f3f4a0 prio 196 cost 0 latency 0.000008 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 335 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d000 con 0x558c54c64000 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000009 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 336 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795dc00 con 0x558c54c64000 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57f3fe40 prio 196 cost 0 latency 0.000007 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55114000> 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114000 kv_submitted 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55114000 2019-12-07T19:34:22.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114000 finishing 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55114000 onodes 0x558c54ce2280,0x558c57e8f900 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114000 done 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:22.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55114000 [] 2019-12-07T19:34:22.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e45) pg 0x558c54d98000 2019-12-07T19:34:22.630+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:22.630+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.630+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:22.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:22.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c55114300 seq 13 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.630+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:22.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:22.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4343 < min_flush_size 524288 2019-12-07T19:34:22.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:22.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:22.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55114300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:22.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55114300 onodes shared_blobs 2019-12-07T19:34:22.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55114300 allocated 0x[] released 0x[] 2019-12-07T19:34:22.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114300 prepare 2019-12-07T19:34:22.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55114300 2019-12-07T19:34:22.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114300 io_done 2019-12-07T19:34:22.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45680 had 1 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4366 < min_flush_size 524288 2019-12-07T19:34:22.630+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 45/28 ECSubWriteReply(tid=124, last_complete=45'46, committed=1, applied=1)) v2 -- 0x558c54dfae00 con 0x558c54c5bc00 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4366 < min_flush_size 524288 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1fc450~110e to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1fc450 0x1fc450~110e to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1cc450 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x450 2019-12-07T19:34:22.630+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 45/28 ECSubWriteReply(tid=125, last_complete=45'48, committed=1, applied=1)) v2 -- 0x558c54dfa400 con 0x558c54c5bc00 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x55e and padding block with 0xaa2 2019-12-07T19:34:22.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57f3f4a0 prio 196 cost 0 latency 0.000244 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57f3f4a0 finish 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000262 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57f3fe40 prio 196 cost 0 latency 0.000272 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57f3fe40 finish 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1fd55e 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000202696s (0.000000115s flush + 0.000202581s kv commit) 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1fd55e~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1fd55e 0x1fd55e~495 to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~1d0000 x_off 0x1cd55e 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x55e 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9f3 and padding block with 0x60d 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 88 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5501ce00 con 0x558c54c19c00 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57334000 prio 196 cost 0 latency 0.000011 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x1fd9f3 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124197s (0.000000085s flush + 0.000124112s kv commit) 2019-12-07T19:34:22.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c57334000 prio 196 cost 0 latency 0.000109 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c57334000 finish 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 192 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841fe00 con 0x558c54c5bc00 2019-12-07T19:34:22.630+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c578d38c0 prio 196 cost 0 latency 0.000020 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c578d38c0 prio 196 cost 0 latency 0.000044 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c578d38c0 finish 2019-12-07T19:34:22.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 89 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57612200 con 0x558c54c19c00 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000012 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 193 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5763de00 con 0x558c54c5bc00 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000008 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 224 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332c00 con 0x558c54c5b400 2019-12-07T19:34:22.630+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000008 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000068 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000100 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 194 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54dfa400 con 0x558c54c5bc00 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c578d31e0 prio 196 cost 0 latency 0.000006 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d3a20 finish 2019-12-07T19:34:22.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000095 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.630+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d31e0 prio 196 cost 0 latency 0.000060 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.631+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.631+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c578d31e0 finish 2019-12-07T19:34:22.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddf800,0x558c55114300> 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 kv_submitted 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddf800 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddf800 finishing 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddf800 onodes 0x558c54ce2a00,0x558c54e45680 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddf800 done 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddf800 [] 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114300 kv_submitted 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55114300 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114300 finishing 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55114300 onodes 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114300 done 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55114300 [] 2019-12-07T19:34:22.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.631+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.631+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 45/41 ECSubWriteReply(tid=222, last_complete=45'30, committed=1, applied=1)) v2 -- 0x558c55270600 con 0x558c54c64000 2019-12-07T19:34:22.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.631+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 225 ==== MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=86, reqid=osd.4.0:84, at_version=0'0, trim_to=0'0, roll_forward_to=45'17)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c58154000 con 0x558c54c5b400 2019-12-07T19:34:22.631+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5740d080 prio 127 cost 0 latency 0.000018 epoch 45 MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=86, reqid=osd.4.0:84, at_version=0'0, trim_to=0'0, roll_forward_to=45'17)) v2 2019-12-07T19:34:22.631+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=86, reqid=osd.4.0:84, at_version=0'0, trim_to=0'0, roll_forward_to=45'17)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.631+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 337 ==== MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=223, reqid=osd.1.0:223, at_version=45'32, trim_to=0'0, roll_forward_to=43'28)) v2 ==== 3385+0+0 (crc 0 0 0) 0x558c58155800 con 0x558c54c64000 2019-12-07T19:34:22.631+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5740c000 prio 127 cost 0 latency 0.000024 epoch 45 MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=223, reqid=osd.1.0:223, at_version=45'32, trim_to=0'0, roll_forward_to=43'28)) v2 2019-12-07T19:34:22.631+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=223, reqid=osd.1.0:223, at_version=45'32, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=223, reqid=osd.1.0:223, at_version=45'32, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=223, reqid=osd.1.0:223, at_version=45'32, trim_to=0'0, roll_forward_to=43'28)) v2) prio 127 cost 0 e45) pg 0x558c5500a000 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 45 dequeue_op 0x558c5740c000 prio 127 cost 0 latency 0.000082 MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=223, reqid=osd.1.0:223, at_version=45'32, trim_to=0'0, roll_forward_to=43'28)) v2 pg pg[1.9s4( v 45'30 (0'0,45'30] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'30 (0'0,45'30] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=223, reqid=osd.1.0:223, at_version=45'32, trim_to=0'0, roll_forward_to=43'28)) v2 2019-12-07T19:34:22.631+0000 7f8f04181700 20 snap_mapper.update_snaps 1:9b8adabe:::smithi13140796-6:11 e was 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.631+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 226 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=87, reqid=osd.4.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=45'25)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748e800 con 0x558c54c5b400 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.631+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5740de40 prio 127 cost 0 latency 0.000026 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=87, reqid=osd.4.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.631+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:11 c,e 2019-12-07T19:34:22.631+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=87, reqid=osd.4.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.631+0000 7f8f04181700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:11 e 2019-12-07T19:34:22.631+0000 7f8f04181700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:22.631+0000 7f8f04181700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'30 (0'0,45'30] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'30], crt=43'28) [45'31 (43'27) modify 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:22.631848+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'32 (43'28) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:22.631848+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'31 (0'0,45'31] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'30 crt=43'28 lcod 45'30 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'31 (43'27) modify 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:22.631848+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'30 crt=43'28 lcod 45'30 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'32 (43'28) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:22.631848+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'30 crt=43'28 lcod 45'30 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 32 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'30 crt=43'28 lcod 45'30 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.631+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.631+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'31, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c55115b00 seq 63 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# 1 keys 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# 1 keys = 0 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# hinfo_key (42 bytes) 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- SNA_1_000000000000000C_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000045.00000000000000000031' <- 0000000045.00000000000000000031 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000045.00000000000000000032' <- 0000000045.00000000000000000032 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55115b00 cost 7548 (1 ios * 4000 + 3548 bytes) 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55115b00 onodes 0x558c57a35400,0x558c57e8f180 shared_blobs 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 124501 (123442 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:22.631+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 133 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54dfbc00 con 0x558c54c19800 2019-12-07T19:34:22.631+0000 7f8f24c58700 15 osd.7 45 enqueue_op 0x558c5511e000 prio 196 cost 0 latency 0.000018 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.631+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore.extentmap(0x558c57e8f288) update 4#1:9b8adabe:::smithi13140796-6:11# 2019-12-07T19:34:22.631+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 134 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841ea00 con 0x558c54c19800 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:11# is 55375 (54316 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:22.631+0000 7f8f24c58700 15 osd.7 45 enqueue_op 0x558c5511e160 prio 196 cost 0 latency 0.000007 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.631+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.631+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 135 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841e400 con 0x558c54c19800 2019-12-07T19:34:22.631+0000 7f8f24c58700 15 osd.7 45 enqueue_op 0x558c5511e2c0 prio 196 cost 0 latency 0.000007 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.631+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55115b00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115b00 prepare 2019-12-07T19:34:22.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55115b00 2019-12-07T19:34:22.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115b00 io_done 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'30 crt=43'28 lcod 45'30 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'32 2019-12-07T19:34:22.631+0000 7f8f04181700 10 osd.7 45 dequeue_op 0x558c5740c000 finish 2019-12-07T19:34:22.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=86, reqid=osd.4.0:84, at_version=0'0, trim_to=0'0, roll_forward_to=45'17)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=86, reqid=osd.4.0:84, at_version=0'0, trim_to=0'0, roll_forward_to=45'17)) v2) prio 127 cost 0 e45) pg 0x558c5513e000 2019-12-07T19:34:22.631+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5740d080 prio 127 cost 0 latency 0.000690 MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=86, reqid=osd.4.0:84, at_version=0'0, trim_to=0'0, roll_forward_to=45'17)) v2 pg pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.3s4 45/41 ECSubWrite(tid=86, reqid=osd.4.0:84, at_version=0'0, trim_to=0'0, roll_forward_to=45'17)) v2 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=42'15 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'17], crt=42'15) [] 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'16 (42'14) modify 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:22.623849+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'17 (42'15) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:22.623849+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 17 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.632+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.632+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c583c8600 seq 54 2019-12-07T19:34:22.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:22.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c8600 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c8600 onodes shared_blobs 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c8600 allocated 0x[] released 0x[] 2019-12-07T19:34:22.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8600 prepare 2019-12-07T19:34:22.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c8600 2019-12-07T19:34:22.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8600 io_done 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=87, reqid=osd.4.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=87, reqid=osd.4.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5740de40 prio 127 cost 0 latency 0.000765 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=87, reqid=osd.4.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=45'25)) v2 pg pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=87, reqid=osd.4.0:85, at_version=0'0, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=44'21 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'25], crt=44'21) [] 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'22 (44'13) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:22.623726+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'23 (44'16) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:22.623726+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'24 (44'17) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:22.624660+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9741 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'25 (44'21) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:22.624660+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 25 2019-12-07T19:34:22.632+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.632+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 227 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c5b400 2019-12-07T19:34:22.632+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.bs1 epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.bs0( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:22.632+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.bs0( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 42509 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 228 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c5b400 2019-12-07T19:34:22.632+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.3s4 epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:22.632+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 75277 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 108045 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.bs0( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 140813 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 173581 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 181956 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f180 had 1 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 183094 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 183117 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 183117 < min_flush_size 524288 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x1fd9f3~2cb4d to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x1fd9f3 0x1fd9f3~2cb4d to file(ino 16 size 0x200000 mtime 2019-12-07T19:34:22.625099+0000 allocated 200000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~1d0000]) 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _allocate len 0x2a540 from 0 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _allocate len 0x2a540 from 1 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x30000/10000,30000,acd330000 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd330000~30000/10000,30000,acd330000 2019-12-07T19:34:22.632+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x230000 to include allocated 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 51 (was clean) 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~200000 x_off 0x1cd9f3 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9f3 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x540 and padding block with 0xac0 2019-12-07T19:34:22.632+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x230000 2019-12-07T19:34:22.632+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 195 ==== MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=126, reqid=osd.3.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=45'48)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c574a2000 con 0x558c54c5bc00 2019-12-07T19:34:22.632+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c57335e40 prio 127 cost 0 latency 0.000022 epoch 45 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=126, reqid=osd.3.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=45'48)) v2 2019-12-07T19:34:22.632+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=126, reqid=osd.3.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=45'48)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=126, reqid=osd.3.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=45'48)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=126, reqid=osd.3.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=45'48)) v2) prio 127 cost 0 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.632+0000 7f8f0197c700 10 osd.7 45 dequeue_op 0x558c57335e40 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=126, reqid=osd.3.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=45'48)) v2 pg pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=126, reqid=osd.3.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=45'48)) v2 2019-12-07T19:34:22.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=43'44 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'48], crt=43'44) [] 2019-12-07T19:34:22.632+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511e000 prio 196 cost 0 latency 0.001116 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'45 (41'36) modify 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:22.626290+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=31, last_complete=45'28, committed=1, applied=1)) v2 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'46 (41'37) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:22.626290+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] handle_sub_write_reply Calling on_all_commit on Op(1:3c88320e:::smithi13140796-1:c v=45'28 tt=0'0 tid=31 reqid=osd.7.0:31 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:22.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=38'26 crt=38'26 lcod 37'24 mlcod 37'24 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] repop_all_committed: repop tid 31 all committed 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'47 (43'43) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:22.627413+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] eval_repop repgather(0x558c576b6c60 45'28 rep_tid=31 committed?=1 r=0) (no op) 2019-12-07T19:34:22.632+0000 7f8f0197c700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'48 (43'44) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:22.627413+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] commit: repgather(0x558c576b6c60 45'28 rep_tid=31 committed?=1 r=0) 2019-12-07T19:34:22.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 48 2019-12-07T19:34:22.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.632+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:138 2019-12-07T19:34:22.632+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] removing repgather(0x558c576b6c60 45'28 rep_tid=31 committed?=1 r=0) 2019-12-07T19:34:22.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] q front is repgather(0x558c576b6c60 45'28 rep_tid=31 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] remove_repop repgather(0x558c576b6c60 45'28 rep_tid=31 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_finish_rmw: Op(1:3c88320e:::smithi13140796-1:c v=45'28 tt=0'0 tid=31 reqid=osd.7.0:31 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:c=[],1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5815fb00 seq 79 2019-12-07T19:34:22.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 136 ==== MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=59, reqid=osd.2.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=45'12)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c58154800 con 0x558c54c19800 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.633+0000 7f8f24c58700 15 osd.7 45 enqueue_op 0x558c5511e420 prio 127 cost 0 latency 0.000020 epoch 45 MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=59, reqid=osd.2.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=45'12)) v2 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=59, reqid=osd.2.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=45'12)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815fb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815fb00 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815fb00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511e000 finish 2019-12-07T19:34:22.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815fb00 prepare 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815fb00 2019-12-07T19:34:22.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815fb00 io_done 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.633+0000 7f8f0197c700 10 osd.7 45 dequeue_op 0x558c57335e40 finish 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=59, reqid=osd.2.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=45'12)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=59, reqid=osd.2.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=45'12)) v2) prio 127 cost 0 e45) pg 0x558c55141000 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511e160 prio 196 cost 0 latency 0.001271 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=32, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 osd.7 45 dequeue_op 0x558c5511e420 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=59, reqid=osd.2.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=45'12)) v2 pg pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.ds2 45/41 ECSubWrite(tid=59, reqid=osd.2.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=45'12)) v2 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] handle_sub_write_reply Calling on_all_commit on Op(1:276d5164:::smithi13140796-22:e v=45'8 tt=0'0 tid=32 reqid=osd.7.0:32 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=42'6 crt=42'6 lcod 37'4 mlcod 37'4 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] repop_all_committed: repop tid 32 all committed 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=43'10 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'12], crt=43'10) [] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] eval_repop repgather(0x558c576b6360 45'8 rep_tid=32 committed?=1 r=0) (no op) 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] commit: repgather(0x558c576b6360 45'8 rep_tid=32 committed?=1 r=0) 2019-12-07T19:34:22.632+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'11 (43'9) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:22.627548+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:110 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'12 (43'10) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:22.627548+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] removing repgather(0x558c576b6360 45'8 rep_tid=32 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 12 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] q front is repgather(0x558c576b6360 45'8 rep_tid=32 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c583c8900 seq 27 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/WaitRepops 2019-12-07T19:34:22.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.633+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c8900 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c8900 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c8900 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57a56900 seq 40 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8900 prepare 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c8900 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8900 io_done 2019-12-07T19:34:22.633+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:22.633+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5740de40 finish 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 45 queueing pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] for snaptrim 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=45) prio 5 cost 1048576 e45) 2019-12-07T19:34:22.633+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.633+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.633+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e45) pg 0x558c5513e000 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a56900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a56900 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a56900 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:111 2019-12-07T19:34:22.633+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56900 prepare 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a56900 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56900 io_done 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] remove_repop repgather(0x558c576b6360 45'8 rep_tid=32 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f0217d700 10 osd.7 45 dequeue_op 0x558c5511e420 finish 2019-12-07T19:34:22.633+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.633+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.633+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:e v=45'8 tt=0'0 tid=32 reqid=osd.7.0:32 roll_forward_to=42'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c583c8c00 seq 55 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:22.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c8c00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c8c00 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c8c00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=34 reqid=osd.7.0:32 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8c00 prepare 2019-12-07T19:34:22.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c8c00 2019-12-07T19:34:22.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8c00 io_done 2019-12-07T19:34:22.633+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.bs0( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e45) pg 0x558c575ef000 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 4(0) info: 1.bs0( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written: {} 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=34 reqid=osd.7.0:32 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57d0a900 seq 28 2019-12-07T19:34:22.633+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0a900 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0a900 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 338 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0a900 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 45/41 ECSubWrite(tid=34, reqid=osd.7.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c58156800 con 0x558c54c64000 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a900 prepare 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0a900 2019-12-07T19:34:22.633+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a900 io_done 2019-12-07T19:34:22.633+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.633+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.as5 epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:22.633+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 45/41 ECSubWrite(tid=34, reqid=osd.7.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c58157000 con 0x558c54c19800 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.4s1 45/41 ECSubWrite(tid=34, reqid=osd.7.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c58157800 con 0x558c54c5bc00 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 45/41 ECSubWrite(tid=34, reqid=osd.7.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c58155000 con 0x558c54c5b400 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 45/41 ECSubWrite(tid=34, reqid=osd.7.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c57a43000 con 0x558c54c19c00 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=42'6 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log log((0'0,45'8], crt=42'6) [] 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] rollforward: entry=45'7 (42'5) modify 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] rollforward: entry=45'8 (42'6) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:22.627658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log approx pg log length = 8 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.633+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.633+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 137 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a53c00 con 0x558c54c19800 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c57ff0600 seq 35 2019-12-07T19:34:22.633+0000 7f8f24c58700 15 osd.7 45 enqueue_peering_evt 1.ds2 epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:22.633+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff0600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff0600 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff0600 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 prepare 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff0600 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 io_done 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511e160 finish 2019-12-07T19:34:22.633+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.633+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e45) pg 0x558c55141000 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.633+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511e2c0 prio 196 cost 0 latency 0.001710 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.633+0000 7f8f06185700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=33, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.633+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:22.633+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:22.633+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5815e600 seq 41 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=45'30 tt=0'0 tid=33 reqid=osd.7.0:33 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:22.633+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:22.633+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'28 crt=38'26 lcod 38'26 mlcod 38'26 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] repop_all_committed: repop tid 33 all committed 2019-12-07T19:34:22.633+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] eval_repop repgather(0x558c576b6ea0 45'30 rep_tid=33 committed?=1 r=0) (no op) 2019-12-07T19:34:22.633+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815e600 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.633+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815e600 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815e600 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] commit: repgather(0x558c576b6ea0 45'30 rep_tid=33 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e600 prepare 2019-12-07T19:34:22.633+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815e600 2019-12-07T19:34:22.633+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e600 io_done 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.633+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:139 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] removing repgather(0x558c576b6ea0 45'30 rep_tid=33 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] q front is repgather(0x558c576b6ea0 45'30 rep_tid=33 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/WaitRepops 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 45 queueing pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] for snaptrim 2019-12-07T19:34:22.633+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=45) prio 5 cost 1048576 e45) 2019-12-07T19:34:22.633+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.633+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e45) pg 0x558c57e8b000 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,d~1,10~2] 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] publish_stats_to_osd 45:140 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] remove_repop repgather(0x558c576b6ea0 45'30 rep_tid=33 committed?=1 r=0) 2019-12-07T19:34:22.633+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:22.633+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=45'30 tt=0'0 tid=33 reqid=osd.7.0:33 roll_forward_to=38'26 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:22.633+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:22.633+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:22.633+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c548f6300 seq 12 2019-12-07T19:34:22.633+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=0'0 tt=0'0 tid=35 reqid=osd.7.0:33 roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:22.633+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:22.633+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:22.633+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.633+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c548f6300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:22.633+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c548f6300 onodes shared_blobs 2019-12-07T19:34:22.633+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c548f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:22.633+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 prepare 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written: {} 2019-12-07T19:34:22.633+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c548f6300 2019-12-07T19:34:22.633+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 io_done 2019-12-07T19:34:22.633+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:e v=0'0 tt=0'0 tid=35 reqid=osd.7.0:33 roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 45/28 ECSubWrite(tid=35, reqid=osd.7.0:33, at_version=0'0, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c57a43800 con 0x558c54c64000 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 45/28 ECSubWrite(tid=35, reqid=osd.7.0:33, at_version=0'0, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c57dfe800 con 0x558c54c19800 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 45/28 ECSubWrite(tid=35, reqid=osd.7.0:33, at_version=0'0, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c579e3000 con 0x558c54c5bc00 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 45/28 ECSubWrite(tid=35, reqid=osd.7.0:33, at_version=0'0, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c574a5800 con 0x558c54c5b400 2019-12-07T19:34:22.633+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 45/28 ECSubWrite(tid=35, reqid=osd.7.0:33, at_version=0'0, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c551dd000 con 0x558c54c19c00 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=38'26 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log log((0'0,45'30], crt=38'26) [] 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] rollforward: entry=45'27 (35'19) delete 1:3c88320e:::smithi13140796-1:c by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] rollforward: entry=45'28 (35'22) modify 1:3c88320e:::smithi13140796-1:head by unknown.0.0:0 2019-12-07T19:34:22.627234+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] rollforward: entry=45'29 (38'25) modify 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] rollforward: entry=45'30 (38'26) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:22.628213+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log approx pg log length = 30 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.633+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57ff1500 seq 39 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c#1b onode 0x558c5738d400 txc 0x558c57ff1500 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c#1b 0x0 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore.blob(0x558c54d753b0) put_ref 0x0~2000 Blob(0x558c54d753b0 blob([0x337000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore.extentmap(0x558c5738d508) dirty_range mark inline shard dirty 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d753b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=2)))) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x337000~2000] 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=1))) 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf8a80 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3c88320e:::smithi13140796-1:c#1b = 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ff1500 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ff1500 onodes shared_blobs 0x558c54cf8a80 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2859 is 11 SharedBlob(0x558c54cf8a80 loaded (sbid 0x2859 ref_map(0x337000~2000=1))) 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ff1500 allocated 0x[] released 0x[] 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 prepare 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ff1500 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 io_done 2019-12-07T19:34:22.633+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511e2c0 finish 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=45) prio 5 cost 1048576 e45) queued 2019-12-07T19:34:22.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=45) prio 5 cost 1048576 e45) pg 0x558c548ab000 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] snap_trimmer posting 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap c 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1eec0) upper_bound after SNA_1_000000000000000C_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.4' 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1eec0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000C_.0_0000000000000001.4) returns 0 SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[a~1,d~1,10~2]] SnapTrimmer state: adding snap c to purged_snaps 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[a~1,c~2,10~2]] SnapTrimmer state: purged_snaps now [a~1,c~2,10~2], snap_trimq now [] 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c585f6000 seq 36 2019-12-07T19:34:22.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f6000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f6000 onodes shared_blobs 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f6000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6000 prepare 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f6000 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6000 io_done 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[a~1,c~2,10~2]] share_pg_info 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c55053500 con 0x558c54c64000 2019-12-07T19:34:22.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 229 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55270600 con 0x558c54c5b400 2019-12-07T19:34:22.634+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57efe2c0 prio 196 cost 0 latency 0.000014 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.634+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c55055800 con 0x558c54c19800 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c55052000 con 0x558c54c5bc00 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c55052700 con 0x558c54c5b400 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 -- 0x558c55052e00 con 0x558c54c19c00 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[a~1,c~2,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[a~1,c~2,10~2]] exit Trimming 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,c~2,10~2] 2019-12-07T19:34:22.634+0000 7f8f0297e700 15 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] publish_stats_to_osd 45:112 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] enter NotTrimming 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] snap_trimmer complete 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57efe2c0 prio 196 cost 0 latency 0.000171 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x22a540 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57efe2c0 finish 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=45) prio 5 cost 1048576 e45) queued 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=45) prio 5 cost 1048576 e45) pg 0x558c57676000 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] snap_trimmer posting 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap c 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:22.634+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 138 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332a00 con 0x558c54c19800 2019-12-07T19:34:22.634+0000 7f8f24c58700 15 osd.7 45 enqueue_op 0x558c5511e580 prio 196 cost 0 latency 0.000010 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.634+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c575c8b40) upper_bound after SNA_1_000000000000000C_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.0_0000000000000001.C' 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (51) on file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]), flushing log 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c575c8b40) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 51 len 0x4a crc 0x3e6af6cd) 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x32000~1000 to file(ino 1 size 0x32000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x32000 0x32000~1000 to file(ino 1 size 0x32000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x33000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x32000 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000C_.0_0000000000000001.C) returns 0 SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[c~1] ps=[a~1,d~1,10~2]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[a~1,d~1,10~2]] SnapTrimmer state: adding snap c to purged_snaps 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[a~1,c~2,10~2]] SnapTrimmer state: purged_snaps now [a~1,c~2,10~2], snap_trimq now [] 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f6300 seq 40 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x33000 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:22.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f6300 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f6300 onodes shared_blobs 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f6300 allocated 0x[] released 0x[] 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6300 prepare 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f6300 2019-12-07T19:34:22.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6300 io_done 2019-12-07T19:34:22.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[a~1,c~2,10~2]] share_pg_info 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e45/45) v1 -- 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:22.634+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511e580 prio 196 cost 0 latency 0.000124 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.634+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e45/45) v1 -- 0x558c55054300 con 0x558c54c19800 2019-12-07T19:34:22.634+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511e580 finish 2019-12-07T19:34:22.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e45/45) v1 -- 0x558c574e5c00 con 0x558c54c5bc00 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e45/45) v1 -- 0x558c574e4000 con 0x558c54c5b400 2019-12-07T19:34:22.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e45/45) v1 -- 0x558c574e5500 con 0x558c54c19c00 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[a~1,c~2,10~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[a~1,c~2,10~2]] exit Trimming 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] prepare_stats_for_publish reporting purged_snaps [a~1,c~2,10~2] 2019-12-07T19:34:22.634+0000 7f8f0297e700 15 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] publish_stats_to_osd 45:141 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] enter NotTrimming 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:22.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] snap_trimmer complete 2019-12-07T19:34:22.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 339 ==== MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=224, reqid=osd.1.0:224, at_version=45'21, trim_to=0'0, roll_forward_to=44'19)) v2 ==== 3458+0+0 (crc 0 0 0) 0x558c574a5800 con 0x558c54c64000 2019-12-07T19:34:22.634+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57eff8c0 prio 127 cost 0 latency 0.000027 epoch 45 MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=224, reqid=osd.1.0:224, at_version=45'21, trim_to=0'0, roll_forward_to=44'19)) v2 2019-12-07T19:34:22.634+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=224, reqid=osd.1.0:224, at_version=45'21, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000499 2019-12-07T19:34:22.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=224, reqid=osd.1.0:224, at_version=45'21, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=224, reqid=osd.1.0:224, at_version=45'21, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e45) pg 0x558c54d5f000 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 45 dequeue_op 0x558c57eff8c0 prio 127 cost 0 latency 0.000353 MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=224, reqid=osd.1.0:224, at_version=45'21, trim_to=0'0, roll_forward_to=44'19)) v2 pg pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=224, reqid=osd.1.0:224, at_version=45'21, trim_to=0'0, roll_forward_to=44'19)) v2 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 snap_mapper.update_snaps 1:ecc72bdb:::smithi13140796-10:15 e,13,15 was 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 340 ==== MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=225, reqid=osd.1.0:225, at_version=45'23, trim_to=0'0, roll_forward_to=44'19)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c57a43800 con 0x558c54c64000 2019-12-07T19:34:22.635+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c581ccf20 prio 127 cost 0 latency 0.000027 epoch 45 MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=225, reqid=osd.1.0:225, at_version=45'23, trim_to=0'0, roll_forward_to=44'19)) v2 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.635+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=225, reqid=osd.1.0:225, at_version=45'23, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:15 c,e,13,15 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:15 e,13,15 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 51 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.002834206s (0.000000120s flush + 0.002834086s kv commit) 2019-12-07T19:34:22.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=225, reqid=osd.1.0:225, at_version=45'23, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 11 submitting 11 deferred done 0 stable 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 44'19 (0'0,44'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'19], crt=44'19) [45'20 (44'18) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:22.634777+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'21 (44'19) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:22.634777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'20 (0'0,45'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=44'19 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'20 (44'18) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:22.634777+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'21 (0'0,45'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=44'19 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'21 (44'19) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:22.634777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2252 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'21 (0'0,45'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=44'19 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 21 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'21 (0'0,45'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=44'19 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 2 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3366 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'20, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 2 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4492 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5618 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6732 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 2 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7858 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5740b500 seq 60 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9008 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys = 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10252 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_000000000000000C_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738d400 had 1 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 2 2019-12-07T19:34:22.635+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 11378 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000045.00000000000000000020' <- 0000000045.00000000000000000020 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12504 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000045.00000000000000000021' <- 0000000045.00000000000000000021 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12527 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12527 < min_flush_size 524288 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x22a540~30ef to file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740b500 cost 8447 (1 ios * 4000 + 4447 bytes) 2019-12-07T19:34:22.635+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x22a540 0x22a540~30ef to file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740b500 onodes 0x558c54d97180,0x558c575a2280 shared_blobs 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~200000 x_off 0x1fa540 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x540 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54d97288) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.635+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x62f and padding block with 0x9d1 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55115b00,0x558c583c8600> 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115b00 kv_submitted 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55115b00 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115b00 finishing 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55115b00 onodes 0x558c57a35400,0x558c57e8f180 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55115b00 done 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55115b00 [] 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 58064 (57920 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8600 kv_submitted 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c8600 2019-12-07T19:34:22.635+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8600 finishing 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c8600 onodes 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8600 done 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8c00 kv_submitted 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c8600 [] 2019-12-07T19:34:22.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.635+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 45/41 ECSubWriteReply(tid=223, last_complete=45'32, committed=1, applied=1)) v2 -- 0x558c55271200 con 0x558c54c64000 2019-12-07T19:34:22.635+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 45/41 ECSubWriteReply(tid=86, last_complete=45'17, committed=1, applied=1)) v2 -- 0x558c5795d800 con 0x558c54c5b400 2019-12-07T19:34:22.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2388) update 4#1:ecc72bdb:::smithi13140796-10:15# 2019-12-07T19:34:22.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:15# is 556 (412 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:22.635+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 196 ==== MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=127, reqid=osd.3.0:127, at_version=45'50, trim_to=0'0, roll_forward_to=45'48)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c57dff800 con 0x558c54c5bc00 2019-12-07T19:34:22.635+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c573354a0 prio 127 cost 0 latency 0.000044 epoch 45 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=127, reqid=osd.3.0:127, at_version=45'50, trim_to=0'0, roll_forward_to=45'48)) v2 2019-12-07T19:34:22.635+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=127, reqid=osd.3.0:127, at_version=45'50, trim_to=0'0, roll_forward_to=45'48)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.635+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=127, reqid=osd.3.0:127, at_version=45'50, trim_to=0'0, roll_forward_to=45'48)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=127, reqid=osd.3.0:127, at_version=45'50, trim_to=0'0, roll_forward_to=45'48)) v2) prio 127 cost 0 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.635+0000 7f8f05984700 10 osd.7 45 dequeue_op 0x558c573354a0 prio 127 cost 0 latency 0.000092 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=127, reqid=osd.3.0:127, at_version=45'50, trim_to=0'0, roll_forward_to=45'48)) v2 pg pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.635+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 139 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332400 con 0x558c54c19800 2019-12-07T19:34:22.635+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=127, reqid=osd.3.0:127, at_version=45'50, trim_to=0'0, roll_forward_to=45'48)) v2 2019-12-07T19:34:22.635+0000 7f8f24c58700 15 osd.7 45 enqueue_op 0x558c5511e6e0 prio 196 cost 0 latency 0.000015 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f05984700 20 snap_mapper.update_snaps 1:6ee4dfe7:::smithi13140796-47:e e was 2019-12-07T19:34:22.635+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.635+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.635+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511e6e0 prio 196 cost 0 latency 0.000044 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 90 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518c00 con 0x558c54c19c00 2019-12-07T19:34:22.635+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511f080 prio 196 cost 0 latency 0.000012 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511e6e0 finish 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 91 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795c800 con 0x558c54c19c00 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.635+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511f1e0 prio 196 cost 0 latency 0.000009 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511f080 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511f080 finish 2019-12-07T19:34:22.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 230 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518800 con 0x558c54c5b400 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511f340 prio 196 cost 0 latency 0.000007 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511f1e0 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511f1e0 finish 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511f340 prio 196 cost 0 latency 0.000047 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.635+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c5511f340 finish 2019-12-07T19:34:22.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 231 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=88, reqid=client.4540.0:367, at_version=45'26, trim_to=0'0, roll_forward_to=45'25)) v2 ==== 145332+0+0 (crc 0 0 0) 0x558c551dd000 con 0x558c54c5b400 2019-12-07T19:34:22.635+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511f4a0 prio 127 cost 0 latency 0.000034 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=88, reqid=client.4540.0:367, at_version=45'26, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.635+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=88, reqid=client.4540.0:367, at_version=45'26, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=88, reqid=client.4540.0:367, at_version=45'26, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=88, reqid=client.4540.0:367, at_version=45'26, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.635+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5511f4a0 prio 127 cost 0 latency 0.000084 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=88, reqid=client.4540.0:367, at_version=45'26, trim_to=0'0, roll_forward_to=45'25)) v2 pg pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=88, reqid=client.4540.0:367, at_version=45'26, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.635+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'25 (0'0,45'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log log((0'0,45'25], crt=45'25) [45'26 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:367 2019-12-07T19:34:22.630130+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.635+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'26 (0'0,45'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'25 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] add_log_entry 45'26 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:367 2019-12-07T19:34:22.630130+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.635+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'26 (0'0,45'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'25 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log approx pg log length = 26 2019-12-07T19:34:22.635+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'26 (0'0,45'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'25 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.635+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.635+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'26, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740b500 allocated 0x[] released 0x[] 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 prepare 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740b500 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 io_done 2019-12-07T19:34:22.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c583c8f00 seq 29 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:d07269f7:::smithi13140796-28:head# key 0x818000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'21 (0'0,45'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=44'19 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'21 2019-12-07T19:34:22.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:22.635+0000 7f8f0117b700 10 osd.7 45 dequeue_op 0x558c57eff8c0 finish 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1227 2019-12-07T19:34:22.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# = 0 2019-12-07T19:34:22.635+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:22.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:22.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:22.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:22.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=225, reqid=osd.1.0:225, at_version=45'23, trim_to=0'0, roll_forward_to=44'19)) v2) prio 127 cost 0 e45) pg 0x558c54d5f000 2019-12-07T19:34:22.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~23000 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0x0~23000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:22.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~23000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:22.635+0000 7f8f05183700 10 osd.7 45 dequeue_op 0x558c581ccf20 prio 127 cost 0 latency 0.000846 MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=225, reqid=osd.1.0:225, at_version=45'23, trim_to=0'0, roll_forward_to=44'19)) v2 pg pg[1.7s4( v 45'21 (0'0,45'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.635+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'21 (0'0,45'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=225, reqid=osd.1.0:225, at_version=45'23, trim_to=0'0, roll_forward_to=44'19)) v2 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c583c8f00 18 blobs 2019-12-07T19:34:22.635+0000 7f8f05183700 20 snap_mapper.update_snaps 1:ec1285ab:::smithi13140796-19:10 e was 2019-12-07T19:34:22.635+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.635+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.636+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x23000/1000,23000,0 2019-12-07T19:34:22.636+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x999000~23000/1000,23000,0 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x999000~23000] 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840f2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d79d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d79d0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c5840f2d0) get_ref 0x0~2000 Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d79d0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c5840f2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c580d79e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eac40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c70 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c583eac40) get_ref 0x0~2000 Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c70 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c583eac40) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57897c88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840f3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b90 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c5840f3b0) get_ref 0x0~2000 Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b90 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c5840f3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971ba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ead20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead20 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c578f1570) get_ref 0x0~2000 Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead20 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c578f1570) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c583ead38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b2e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840f420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f420 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c550b2e00) get_ref 0x0~2000 Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f420 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c550b2e00) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5840f438 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eb960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad650 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c583eb960) get_ref 0x0~2000 Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad650 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c583eb960) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad668 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e70 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970fc0) get_ref 0x0~2000 Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e70 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970e88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b20 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970ee0) get_ref 0x0~2000 Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b20 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578199d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d90 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c578199d0) get_ref 0x0~2000 Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d90 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c578199d0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970da8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ab0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970e00) get_ref 0x0~2000 Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ab0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970e00) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971ac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970cb0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57818d20) get_ref 0x0~2000 Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970cb0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57818d20) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970cc8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971a40 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970d20) get_ref 0x0~2000 Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971a40 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970d20) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970bd0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57818cb0) get_ref 0x0~2000 Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970bd0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57818cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970be8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579719d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579719d0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970c40) get_ref 0x0~2000 Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579719d0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970c40) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579719e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f50ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970af0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57f50ee0) get_ref 0x0~2000 Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970af0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57f50ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970b08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971960 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970b60) get_ref 0x0~2000 Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971960 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970b60) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b22a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b22a0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57819a40) get_ref 0x0~2000 Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b22a0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57819a40) get_ref init 0x2000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b22b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579718f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579718f0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970a80) get_ref 0x0~1000 Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579718f0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.blob(0x558c57970a80) get_ref init 0x1000, 1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~1000: 0x0~1000 Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971908 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x23000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark inline shard dirty 2019-12-07T19:34:22.636+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~23000 = 0 2019-12-07T19:34:22.636+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.636+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.636+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000026' <- 0000000045.00000000000000000026 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.636+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.636+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c8f00 cost 221772 (19 ios * 4000 + 145772 bytes) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c8f00 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update inline shard 342 bytes from 18 extents 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 761 (413 bytes onode + 2 bytes spanning blobs + 346 bytes inline extents) 2019-12-07T19:34:22.636+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c8f00 allocated 0x[999000~23000] released 0x[] 2019-12-07T19:34:22.636+0000 7f8f07187700 10 freelist allocate 0x999000~23000 2019-12-07T19:34:22.636+0000 7f8f07187700 20 freelist _xor first_key 0x980000 last_key 0x980000 2019-12-07T19:34:22.636+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8f00 prepare 2019-12-07T19:34:22.636+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c583c8f00 2019-12-07T19:34:22.636+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'26 (0'0,45'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'25 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] op_applied version 45'26 2019-12-07T19:34:22.636+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5511f4a0 finish 2019-12-07T19:34:22.636+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x22d62f 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.636+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 341 ==== MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=226, reqid=osd.1.0:223, at_version=0'0, trim_to=0'0, roll_forward_to=45'32)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57a43000 con 0x558c54c64000 2019-12-07T19:34:22.636+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511f600 prio 127 cost 0 latency 0.000017 epoch 45 MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=226, reqid=osd.1.0:223, at_version=0'0, trim_to=0'0, roll_forward_to=45'32)) v2 2019-12-07T19:34:22.636+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=226, reqid=osd.1.0:223, at_version=0'0, trim_to=0'0, roll_forward_to=45'32)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 11 cleaned 0 in 0.001598449s (0.000000101s flush + 0.001598348s kv commit) 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10705 < min_flush_size 524288 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=226, reqid=osd.1.0:223, at_version=0'0, trim_to=0'0, roll_forward_to=45'32)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=226, reqid=osd.1.0:223, at_version=0'0, trim_to=0'0, roll_forward_to=45'32)) v2) prio 127 cost 0 e45) pg 0x558c5500a000 2019-12-07T19:34:22.636+0000 7f8f04181700 10 osd.7 45 dequeue_op 0x558c5511f600 prio 127 cost 0 latency 0.000069 MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=226, reqid=osd.1.0:223, at_version=0'0, trim_to=0'0, roll_forward_to=45'32)) v2 pg pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 43473 < min_flush_size 524288 2019-12-07T19:34:22.636+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.9s4 45/41 ECSubWrite(tid=226, reqid=osd.1.0:223, at_version=0'0, trim_to=0'0, roll_forward_to=45'32)) v2 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61542 < min_flush_size 524288 2019-12-07T19:34:22.636+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=43'28 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'32], crt=43'28) [] 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'29 (43'25) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:22.629711+0000 0 snaps [e,13] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'30 (43'26) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:22.629711+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'31 (43'27) modify 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:22.631848+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97180 had 1 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2280 had 1 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'32 (43'28) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:22.631848+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:22.636+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 32 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61565 < min_flush_size 524288 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61565 < min_flush_size 524288 2019-12-07T19:34:22.636+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.636+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x22d62f~f07d to file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.636+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x22d62f 0x22d62f~f07d to file(ino 16 size 0x230000 mtime 2019-12-07T19:34:22.633802+0000 allocated 230000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~200000]) 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _allocate len 0xc6ac from 0 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _allocate len 0xc6ac from 1 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd360000 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd360000~10000/10000,10000,acd360000 2019-12-07T19:34:22.636+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x240000 to include allocated 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 52 (was clean) 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~210000 x_off 0x1fd62f 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x62f 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6ac and padding block with 0x954 2019-12-07T19:34:22.636+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x240000 2019-12-07T19:34:22.636+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:22.636+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5511c000 seq 64 2019-12-07T19:34:22.636+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:22.636+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:22.636+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.636+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.636+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:22.636+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511c000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.636+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511c000 onodes shared_blobs 2019-12-07T19:34:22.636+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511c000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.636+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c000 prepare 2019-12-07T19:34:22.636+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511c000 2019-12-07T19:34:22.636+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c000 io_done 2019-12-07T19:34:22.636+0000 7f8f04181700 10 osd.7 45 dequeue_op 0x558c5511f600 finish 2019-12-07T19:34:22.636+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5815fb00,0x558c583c8900,0x558c57a56900,0x558c583c8c00,0x558c57d0a900,0x558c57ff0600,0x558c5815e600,0x558c548f6300,0x558c57ff1500,0x558c585f6000,0x558c585f6300> 2019-12-07T19:34:22.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8f00 aio_wait 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.637+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c8f00 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815fb00 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815fb00 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815fb00 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815fb00 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815fb00 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815fb00 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8f00 io_done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8900 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c8900 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8900 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c8900 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8900 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0a900 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c8900 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56900 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 342 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a56900 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56900 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a56900 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a56900 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815e600 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a56900 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8c00 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c8c00 2019-12-07T19:34:22.637+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.9s4 epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8c00 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c8c00 onodes 2019-12-07T19:34:22.637+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8c00 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c8c00 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a900 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0a900 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0a900 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0a900 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0a900 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8f00 kv_queued 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0a900 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff0600 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff0600 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff0600 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff0600 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6000 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff0600 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e600 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815e600 2019-12-07T19:34:22.637+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e600 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815e600 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815e600 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815e600 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c548f6300 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c548f6300 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c548f6300 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c548f6300 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c548f6300 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ff1500 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ff1500 finishing 2019-12-07T19:34:22.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ff1500 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ff1500 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6300 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ff1500 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6000 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e45) pg 0x558c5500a000 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f6000 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6000 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f6000 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6000 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f6000 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6300 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f6300 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6300 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f6300 onodes 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6300 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f6300 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.637+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f08189700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.637+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:22.637+0000 7f8f05984700 20 snap_mapper.get_snaps 1:6ee4dfe7:::smithi13140796-47:e c,e 2019-12-07T19:34:22.637+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:22.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57cdfb00 seq 65 2019-12-07T19:34:22.637+0000 7f8f05984700 20 snap_mapper.set_snaps 1:6ee4dfe7:::smithi13140796-47:e e 2019-12-07T19:34:22.637+0000 7f8f05984700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.637+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:22.637+0000 7f8f05984700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:22.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:22.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.637+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:22.637+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cdfb00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cdfb00 onodes shared_blobs 2019-12-07T19:34:22.637+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ec1285ab:::smithi13140796-19:10 c,e 2019-12-07T19:34:22.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cdfb00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.637+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ec1285ab:::smithi13140796-19:10 e 2019-12-07T19:34:22.637+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdfb00 prepare 2019-12-07T19:34:22.637+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:22.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cdfb00 2019-12-07T19:34:22.637+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdfb00 io_done 2019-12-07T19:34:22.637+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'48 (0'0,45'48] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'48], crt=45'48) [45'49 (38'29) modify 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:22.634330+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'50 (38'30) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:22.634330+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f05183700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:22.637+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'49 (0'0,45'49] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'48 crt=45'48 lcod 45'48 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'49 (38'29) modify 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:22.634330+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.637+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'21 (0'0,45'21] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'21], crt=44'19) [45'22 (42'16) modify 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:22.635679+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'23 (42'17) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:22.635679+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.637+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'48 crt=45'48 lcod 45'48 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'50 (38'30) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:22.634330+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.637+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'48 crt=45'48 lcod 45'48 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 50 2019-12-07T19:34:22.637+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'22 (0'0,45'22] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'21 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'22 (42'16) modify 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:22.635679+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.637+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'48 crt=45'48 lcod 45'48 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.637+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.637+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'49, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.637+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'21 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'23 (42'17) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:22.635679+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.637+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'21 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 23 2019-12-07T19:34:22.637+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'21 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.637+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.637+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'22, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.637+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 45/41 ECSubWriteReply(tid=59, last_complete=45'12, committed=1, applied=1)) v2 -- 0x558c5841ec00 con 0x558c54c19800 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55114600 seq 80 2019-12-07T19:34:22.637+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:22.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# 1 keys 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c574ff800 seq 61 2019-12-07T19:34:22.637+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# 1 keys = 0 2019-12-07T19:34:22.637+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# hinfo_key (42 bytes) 2019-12-07T19:34:22.637+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 1 keys 2019-12-07T19:34:22.637+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# 1 keys = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:22.637+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key (42 bytes) 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.637+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.637+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- SNA_1_000000000000000C_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000045.00000000000000000049' <- 0000000045.00000000000000000049 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000045.00000000000000000050' <- 0000000045.00000000000000000050 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.637+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55114600 cost 7557 (1 ios * 4000 + 3557 bytes) 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55114600 onodes 0x558c54e12280,0x558c54e12780 shared_blobs 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore.extentmap(0x558c54e12388) update 2#1:6ee4dfe7:::smithi13140796-47:e# 2019-12-07T19:34:22.637+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:e# is 1069 (412 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- SNA_1_000000000000000C_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore.extentmap(0x558c54e12888) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1214 (557 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55114600 allocated 0x[] released 0x[] 2019-12-07T19:34:22.637+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000045.00000000000000000022' <- 0000000045.00000000000000000022 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114600 prepare 2019-12-07T19:34:22.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55114600 2019-12-07T19:34:22.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114600 io_done 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000045.00000000000000000023' <- 0000000045.00000000000000000023 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:22.637+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'48 crt=45'48 lcod 45'48 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'50 2019-12-07T19:34:22.637+0000 7f8f05984700 10 osd.7 45 dequeue_op 0x558c573354a0 finish 2019-12-07T19:34:22.637+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 45/28 ECSubWriteReply(tid=126, last_complete=45'48, committed=1, applied=1)) v2 -- 0x558c57518a00 con 0x558c54c5bc00 2019-12-07T19:34:22.637+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 343 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55271200 con 0x558c54c64000 2019-12-07T19:34:22.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511f760 prio 196 cost 0 latency 0.000016 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.637+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 45/41 ECSubWriteReply(tid=87, last_complete=45'25, committed=1, applied=1)) v2 -- 0x558c5795d600 con 0x558c54c5b400 2019-12-07T19:34:22.637+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.637+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x23c6ac 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.637+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 344 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d800 con 0x558c54c64000 2019-12-07T19:34:22.637+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511f8c0 prio 196 cost 0 latency 0.000007 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.637+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574ff800 cost 7559 (1 ios * 4000 + 3559 bytes) 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574ff800 onodes 0x558c54e44000,0x558c574a0500 shared_blobs 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (52) on file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]), flushing log 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1570 (557 bytes onode + 2 bytes spanning blobs + 1011 bytes inline extents) 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 52 len 0x4a crc 0x213fbb9f) 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore.extentmap(0x558c574a0608) update 4#1:ec1285ab:::smithi13140796-19:10# 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x33000~1000 to file(ino 1 size 0x33000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:10# is 688 (412 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x33000 0x33000~1000 to file(ino 1 size 0x33000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x34000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574ff800 allocated 0x[] released 0x[] 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x33000 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 prepare 2019-12-07T19:34:22.637+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574ff800 2019-12-07T19:34:22.637+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 io_done 2019-12-07T19:34:22.637+0000 7f8f05183700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'21 crt=44'19 lcod 44'19 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'23 2019-12-07T19:34:22.637+0000 7f8f05183700 10 osd.7 45 dequeue_op 0x558c581ccf20 finish 2019-12-07T19:34:22.637+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x34000 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000026 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.637+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511f760 prio 196 cost 0 latency 0.000169 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.637+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511f760 finish 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 52 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.637+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511f8c0 prio 196 cost 0 latency 0.000177 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000992168s (0.000000111s flush + 0.000992057s kv commit) 2019-12-07T19:34:22.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.637+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c5511f8c0 finish 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:34:22.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3377 < min_flush_size 524288 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4503 < min_flush_size 524288 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8805 < min_flush_size 524288 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12280 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12780 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13084 < min_flush_size 524288 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0500 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13107 < min_flush_size 524288 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13107 < min_flush_size 524288 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x23c6ac~3333 to file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x23c6ac 0x23c6ac~3333 to file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~210000 x_off 0x20c6ac 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6ac 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.637+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9df and padding block with 0x621 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5740b500> 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740b500 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740b500 finishing 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740b500 onodes 0x558c54d97180,0x558c575a2280 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740b500 done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740b500 [] 2019-12-07T19:34:22.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.637+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 345 ==== MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=227, reqid=osd.1.0:227, at_version=45'39, trim_to=0'0, roll_forward_to=44'37)) v2 ==== 3449+0+0 (crc 0 0 0) 0x558c58156800 con 0x558c54c64000 2019-12-07T19:34:22.637+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511fa20 prio 127 cost 0 latency 0.000025 epoch 45 MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=227, reqid=osd.1.0:227, at_version=45'39, trim_to=0'0, roll_forward_to=44'37)) v2 2019-12-07T19:34:22.637+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=227, reqid=osd.1.0:227, at_version=45'39, trim_to=0'0, roll_forward_to=44'37)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.637+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=227, reqid=osd.1.0:227, at_version=45'39, trim_to=0'0, roll_forward_to=44'37)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.637+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.637+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=227, reqid=osd.1.0:227, at_version=45'39, trim_to=0'0, roll_forward_to=44'37)) v2) prio 127 cost 0 e45) pg 0x558c57380000 2019-12-07T19:34:22.637+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c5511fa20 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=227, reqid=osd.1.0:227, at_version=45'39, trim_to=0'0, roll_forward_to=44'37)) v2 pg pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.637+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=227, reqid=osd.1.0:227, at_version=45'39, trim_to=0'0, roll_forward_to=44'37)) v2 2019-12-07T19:34:22.637+0000 7f8f0898a700 20 snap_mapper.update_snaps 1:131421e8:::smithi13140796-7:15 e,13,15 was 2019-12-07T19:34:22.637+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.637+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.638+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:15 c,e,13,15 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 snap_mapper.set_snaps 1:131421e8:::smithi13140796-7:15 e,13,15 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 44'37 (0'0,44'37] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,44'37], crt=44'37) [45'38 (44'34) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:22.638555+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'39 (44'37) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:22.638555+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'38 (0'0,45'38] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'37 crt=44'37 lcod 44'37 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'38 (44'34) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:22.638555+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'37 crt=44'37 lcod 44'37 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'39 (44'37) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:22.638555+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'37 crt=44'37 lcod 44'37 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 39 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'37 crt=44'37 lcod 44'37 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.638+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'38, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57db2900 seq 29 2019-12-07T19:34:22.638+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_000000000000000C_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000045.00000000000000000038' <- 0000000045.00000000000000000038 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000045.00000000000000000039' <- 0000000045.00000000000000000039 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db2900 cost 8436 (1 ios * 4000 + 4436 bytes) 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db2900 onodes 0x558c57598000,0x558c575f9b80 shared_blobs 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57598108) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 1010 (572 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575f9c88) update 5#1:131421e8:::smithi13140796-7:15# 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:15# is 849 (411 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db2900 allocated 0x[] released 0x[] 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2900 prepare 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db2900 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2900 io_done 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=44'37 crt=44'37 lcod 44'37 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'39 2019-12-07T19:34:22.638+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c5511fa20 finish 2019-12-07T19:34:22.638+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.638+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.638+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 45/41 ECSubWriteReply(tid=224, last_complete=45'21, committed=1, applied=1)) v2 -- 0x558c55045e00 con 0x558c54c64000 2019-12-07T19:34:22.638+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.639+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 197 ==== MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518a00 con 0x558c54c5bc00 2019-12-07T19:34:22.639+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c57335760 prio 196 cost 0 latency 0.000013 epoch 45 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.639+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.639+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 198 ==== MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54dfae00 con 0x558c54c5bc00 2019-12-07T19:34:22.639+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c57335080 prio 196 cost 0 latency 0.000004 epoch 45 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.639+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.639+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) 2019-12-07T19:34:22.639+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.639+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.639+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c548ab000 2019-12-07T19:34:22.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.639+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c57335760 prio 196 cost 0 latency 0.000045 MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 pg pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) queued 2019-12-07T19:34:22.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2) prio 196 cost 0 e45) pg 0x558c57676000 2019-12-07T19:34:22.639+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.4s0 45/41 ECSubWriteReply(tid=34, last_complete=45'8, committed=1, applied=1)) v2 2019-12-07T19:34:22.639+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=34 reqid=osd.7.0:32 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:22.639+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57335080 prio 196 cost 0 latency 0.000044 MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.639+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:22.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWriteReply(1.cs0 45/28 ECSubWriteReply(tid=35, last_complete=45'30, committed=1, applied=1)) v2 2019-12-07T19:34:22.639+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:22.639+0000 7f8f06986700 10 osd.7 45 dequeue_op 0x558c57335760 finish 2019-12-07T19:34:22.639+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=0'0 tt=0'0 tid=35 reqid=osd.7.0:33 roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:22.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:22.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:22.639+0000 7f8f0297e700 10 osd.7 45 dequeue_op 0x558c57335080 finish 2019-12-07T19:34:22.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.639+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 232 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=89, reqid=client.4540.0:368, at_version=45'27, trim_to=0'0, roll_forward_to=45'25)) v2 ==== 108824+0+0 (crc 0 0 0) 0x558c58155000 con 0x558c54c5b400 2019-12-07T19:34:22.639+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511fb80 prio 127 cost 0 latency 0.000021 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=89, reqid=client.4540.0:368, at_version=45'27, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.639+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=89, reqid=client.4540.0:368, at_version=45'27, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=89, reqid=client.4540.0:368, at_version=45'27, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=89, reqid=client.4540.0:368, at_version=45'27, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c5511fb80 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=89, reqid=client.4540.0:368, at_version=45'27, trim_to=0'0, roll_forward_to=45'25)) v2 pg pg[1.bs1( v 45'26 (0'0,45'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x23f9df 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'26 (0'0,45'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=89, reqid=client.4540.0:368, at_version=45'27, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 5 cleaned 0 in 0.001955596s (0.000003804s flush + 0.001951792s kv commit) 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1569 < min_flush_size 524288 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4760 < min_flush_size 524288 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'26 (0'0,45'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log log((0'0,45'26], crt=45'25) [45'27 (45'26) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:368 2019-12-07T19:34:22.633053+0000 0 ObjectCleanRegions clean_offsets: [0~573440,999424~18446744073708552191], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57598000 had 1 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9b80 had 1 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'27 (0'0,45'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'26 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] add_log_entry 45'27 (45'26) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:368 2019-12-07T19:34:22.633053+0000 0 ObjectCleanRegions clean_offsets: [0~573440,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4783 < min_flush_size 524288 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4783 < min_flush_size 524288 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'27 (0'0,45'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'26 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log approx pg log length = 27 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x23f9df~12af to file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'27 (0'0,45'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'26 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x23f9df 0x23f9df~12af to file(ino 16 size 0x240000 mtime 2019-12-07T19:34:22.638059+0000 allocated 240000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~210000]) 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _allocate len 0xc8e from 0 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _allocate len 0xc8e from 1 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd370000 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd370000~10000/10000,10000,acd370000 2019-12-07T19:34:22.639+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x250000 to include allocated 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 53 (was clean) 2019-12-07T19:34:22.639+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'27, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x20f9df 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9df 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc8e and padding block with 0x372 2019-12-07T19:34:22.639+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x250000 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57d0ac00 seq 30 2019-12-07T19:34:22.639+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:22.639+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:22.639+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x23000~1a000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0x23000~1a000 - have 0x23000 (143360) bytes fadvise_flags 0x0 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x23000~1a000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57d0ac00 13 blobs 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5511c000,0x558c583c8f00,0x558c57cdfb00,0x558c55114600,0x558c574ff800> 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c000 kv_submitted 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511c000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c000 finishing 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511c000 onodes 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511c000 done 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cdfb00 kv_submitted 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511c000 [] 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8f00 kv_submitted 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c8f00 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c8f00 finishing 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c8f00 onodes 0x558c54e44780 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad668 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c455f0 space 0x558c576ad668 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897c88 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45830 space 0x558c57897c88 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970b08 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77c20 space 0x558c57970b08 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970be8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77d40 space 0x558c57970be8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970cc8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45320 space 0x558c57970cc8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970da8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45440 space 0x558c57970da8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970e88 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45560 space 0x558c57970e88 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971908 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759eab0 space 0x558c57971908 0x0~1000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971978 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77b90 space 0x558c57971978 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579719e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77cb0 space 0x558c579719e8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971a58 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45290 space 0x558c57971a58 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971ac8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c453b0 space 0x558c57971ac8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971b38 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c454d0 space 0x558c57971b38 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971ba8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c457a0 space 0x558c57971ba8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b22b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f77b00 space 0x558c579b22b8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d79e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759e7e0 space 0x558c580d79e8 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ead38 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45710 space 0x558c583ead38 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840f438 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45680 space 0x558c5840f438 0x0~2000 writing nocache) 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c8f00 done 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0ac00 prepare 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c8f00 [] 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdfb00 kv_submitted 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cdfb00 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdfb00 finishing 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cdfb00 onodes 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cdfb00 done 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cdfb00 [] 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114600 kv_submitted 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55114600 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114600 finishing 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55114600 onodes 0x558c54e12280,0x558c54e12780 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114600 done 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55114600 [] 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 kv_submitted 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574ff800 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574ff800 finishing 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574ff800 onodes 0x558c54e44000,0x558c574a0500 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574ff800 done 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574ff800 [] 2019-12-07T19:34:22.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1a000/1000,1a000,0 2019-12-07T19:34:22.639+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x9bc000~1a000/1000,1a000,0 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x9bc000~1a000] 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.639+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.639+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 45/41 ECSubWriteReply(tid=226, last_complete=45'32, committed=1, applied=1)) v2 -- 0x558c57569a00 con 0x558c54c64000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3650 sbid 0x0)) 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore.blob(0x558c550b36c0) get_ref 0x0~2000 Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3650 sbid 0x0)) 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore.blob(0x558c550b36c0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3668 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.639+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:22.639+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 45/28 ECSubWriteReply(tid=127, last_complete=45'50, committed=1, applied=1)) v2 -- 0x558c576b5400 con 0x558c54c5bc00 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579702a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore.blob(0x558c579702a0) get_ref 0x0~2000 Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore.blob(0x558c579702a0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:22.639+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57970230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c552790a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552790a0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x240c8e 2019-12-07T19:34:22.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57970230) get_ref 0x0~2000 Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552790a0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c57970230) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c552790b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677810 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c574d2fc0) get_ref 0x0~2000 Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677810 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c574d2fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54677828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f1810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fd960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd960 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c578f1810) get_ref 0x0~2000 Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd960 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c578f1810) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574fd978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fd570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b2ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2ee0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c574fd570) get_ref 0x0~2000 Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2ee0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c574fd570) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b2ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55106e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55106e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106e70 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55106e00) get_ref 0x0~2000 Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106e70 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55106e00) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55106e88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (53) on file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]), flushing log 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55106ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55106f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106f50 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 53 len 0x4a crc 0x5f024838) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55106ee0) get_ref 0x0~2000 Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106f50 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55106ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55106f68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x34000~1000 to file(ino 1 size 0x34000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55106fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107030 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55106fc0) get_ref 0x0~2000 Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107030 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55106fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x34000 0x34000~1000 to file(ino 1 size 0x34000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107048 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x35000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x34000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c551070a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107110 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c551070a0) get_ref 0x0~2000 Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107110 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c551070a0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107128 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551071f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551071f0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55107180) get_ref 0x0~2000 Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551071f0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55107180) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551072d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551072d0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55107260) get_ref 0x0~2000 Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551072d0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55107260) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x35000 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551072e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551073b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551073b0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55107340) get_ref 0x0~2000 Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551073b0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.blob(0x558c55107340) get_ref init 0x2000, 1000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551073c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x3d000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark inline shard dirty 2019-12-07T19:34:22.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x23000~1a000 = 0 2019-12-07T19:34:22.640+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.640+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000037 2019-12-07T19:34:22.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000027' <- 0000000045.00000000000000000027 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0ac00 cost 164474 (14 ios * 4000 + 108474 bytes) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0ac00 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update inline shard 590 bytes from 31 extents 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 1009 (413 bytes onode + 2 bytes spanning blobs + 594 bytes inline extents) 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0ac00 allocated 0x[9bc000~1a000] released 0x[] 2019-12-07T19:34:22.640+0000 7f8f0317f700 10 freelist allocate 0x9bc000~1a000 2019-12-07T19:34:22.640+0000 7f8f0317f700 20 freelist _xor first_key 0x980000 last_key 0x980000 2019-12-07T19:34:22.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0ac00 prepare 2019-12-07T19:34:22.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57d0ac00 2019-12-07T19:34:22.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.640+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 45/41 ECSubWriteReply(tid=225, last_complete=45'23, committed=1, applied=1)) v2 -- 0x558c55045c00 con 0x558c54c64000 2019-12-07T19:34:22.640+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.640+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 199 ==== MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=128, reqid=osd.3.0:127, at_version=0'0, trim_to=0'0, roll_forward_to=45'50)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e3000 con 0x558c54c5bc00 2019-12-07T19:34:22.640+0000 7f8f23c56700 15 osd.7 45 enqueue_op 0x558c552b7b80 prio 127 cost 0 latency 0.000018 epoch 45 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=128, reqid=osd.3.0:127, at_version=0'0, trim_to=0'0, roll_forward_to=45'50)) v2 2019-12-07T19:34:22.640+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=128, reqid=osd.3.0:127, at_version=0'0, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=128, reqid=osd.3.0:127, at_version=0'0, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=128, reqid=osd.3.0:127, at_version=0'0, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 osd.7 45 dequeue_op 0x558c552b7b80 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=128, reqid=osd.3.0:127, at_version=0'0, trim_to=0'0, roll_forward_to=45'50)) v2 pg pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.6s2 45/28 ECSubWrite(tid=128, reqid=osd.3.0:127, at_version=0'0, trim_to=0'0, roll_forward_to=45'50)) v2 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'48 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'50], crt=45'48) [] 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'49 (38'29) modify 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:22.634330+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'50 (38'30) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:22.634330+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 50 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.640+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54d58000 seq 81 2019-12-07T19:34:22.640+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58000 onodes shared_blobs 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 prepare 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58000 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 io_done 2019-12-07T19:34:22.640+0000 7f8f0197c700 10 osd.7 45 dequeue_op 0x558c552b7b80 finish 2019-12-07T19:34:22.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.640+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 200 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c574e5c00 con 0x558c54c5bc00 2019-12-07T19:34:22.640+0000 7f8f23c56700 15 osd.7 45 enqueue_peering_evt 1.6s2 epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:22.640+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.641+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.641+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.641+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s)) prio 255 cost 10 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.641+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 54.116153259s ub 54.116153259s int 16s) 2019-12-07T19:34:22.641+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.641+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:22.641+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:22.641+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55114900 seq 82 2019-12-07T19:34:22.641+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:22.641+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:22.641+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:22.641+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:22.641+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55114900 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.641+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55114900 onodes shared_blobs 2019-12-07T19:34:22.641+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55114900 allocated 0x[] released 0x[] 2019-12-07T19:34:22.641+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114900 prepare 2019-12-07T19:34:22.641+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55114900 2019-12-07T19:34:22.641+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114900 io_done 2019-12-07T19:34:22.641+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 346 ==== MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=228, reqid=osd.1.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=45'23)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57e7d000 con 0x558c54c64000 2019-12-07T19:34:22.641+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511fce0 prio 127 cost 0 latency 0.000020 epoch 45 MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=228, reqid=osd.1.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=45'23)) v2 2019-12-07T19:34:22.641+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=228, reqid=osd.1.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=45'23)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 347 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:22.641+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.7s4 epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:22.641+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 233 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=90, reqid=client.4540.0:369, at_version=45'28, trim_to=0'0, roll_forward_to=45'25)) v2 ==== 88344+0+0 (crc 0 0 0) 0x558c57e7a000 con 0x558c54c5b400 2019-12-07T19:34:22.641+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511fe40 prio 127 cost 0 latency 0.000028 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=90, reqid=client.4540.0:369, at_version=45'28, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.641+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=90, reqid=client.4540.0:369, at_version=45'28, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 348 ==== MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=229, reqid=osd.1.0:229, at_version=45'19, trim_to=0'0, roll_forward_to=38'17)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c54e41000 con 0x558c54c64000 2019-12-07T19:34:22.641+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c581cda20 prio 127 cost 0 latency 0.000024 epoch 45 MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=229, reqid=osd.1.0:229, at_version=45'19, trim_to=0'0, roll_forward_to=38'17)) v2 2019-12-07T19:34:22.641+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=229, reqid=osd.1.0:229, at_version=45'19, trim_to=0'0, roll_forward_to=38'17)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e45) pg 0x558c54d5f000 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=229, reqid=osd.1.0:229, at_version=45'19, trim_to=0'0, roll_forward_to=38'17)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=229, reqid=osd.1.0:229, at_version=45'19, trim_to=0'0, roll_forward_to=38'17)) v2) prio 127 cost 0 e45) pg 0x558c5500d000 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c581cda20 prio 127 cost 0 latency 0.000465 MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=229, reqid=osd.1.0:229, at_version=45'19, trim_to=0'0, roll_forward_to=38'17)) v2 pg pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=229, reqid=osd.1.0:229, at_version=45'19, trim_to=0'0, roll_forward_to=38'17)) v2 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 snap_mapper.update_snaps 1:0064fe1d:::smithi13140796-44:e e was 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54d59b00 seq 62 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.642+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d59b00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d59b00 onodes shared_blobs 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d59b00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:0064fe1d:::smithi13140796-44:e c,e 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 snap_mapper.set_snaps 1:0064fe1d:::smithi13140796-44:e e 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 snap_mapper.set_snaps set OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 prepare 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d59b00 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 io_done 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 snap_mapper.update_snaps rm SNA_1_000000000000000C_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=228, reqid=osd.1.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=45'23)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=228, reqid=osd.1.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=45'23)) v2) prio 127 cost 0 e45) pg 0x558c54d5f000 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 38'17 (0'0,38'17] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,38'17], crt=38'17) [45'18 (38'16) modify 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:22.642501+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,45'19 (38'17) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:22.642501+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'18 (0'0,45'18] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=38'17 crt=38'17 lcod 38'17 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'18 (38'16) modify 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:22.642501+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 45 dequeue_op 0x558c5511fce0 prio 127 cost 0 latency 0.001210 MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=228, reqid=osd.1.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=45'23)) v2 pg pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=38'17 crt=38'17 lcod 38'17 active mbc={} ps=[a~1,d~1,10~2]] add_log_entry 45'19 (38'17) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:22.642501+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=38'17 crt=38'17 lcod 38'17 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 19 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=38'17 crt=38'17 lcod 38'17 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWrite(1.7s4 45/41 ECSubWrite(tid=228, reqid=osd.1.0:225, at_version=0'0, trim_to=0'0, roll_forward_to=45'23)) v2 2019-12-07T19:34:22.642+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'18, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=44'19 active mbc={} ps=[a~1,c~2,10~2]] append_log log((0'0,45'23], crt=44'19) [] 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'20 (44'18) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:22.634777+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c57db2c00 seq 36 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# 1 keys 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# 1 keys = 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# hinfo_key (42 bytes) 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'21 (44'19) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:22.634777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'22 (42'16) modify 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:22.635679+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- SNA_1_000000000000000C_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'23 (42'17) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:22.635679+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000045.00000000000000000018' <- 0000000045.00000000000000000018 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000045.00000000000000000019' <- 0000000045.00000000000000000019 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] append_log approx pg log length = 23 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db2c00 cost 8365 (1 ios * 4000 + 4365 bytes) 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db2c00 onodes 0x558c575bb900,0x558c57f34000 shared_blobs 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 572 (570 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57f34108) update 3#1:0064fe1d:::smithi13140796-44:e# 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:e# is 1039 (412 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:22.642+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db2c00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 53 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2c00 prepare 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db2c00 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2c00 io_done 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002700528s (0.000000105s flush + 0.002700423s kv commit) 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c54ea1500 seq 63 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=38'17 crt=38'17 lcod 38'17 active mbc={} ps=[a~1,d~1,10~2]] op_applied version 45'19 2019-12-07T19:34:22.642+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:22.642+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c581cda20 finish 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.642+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 2 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2240 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1500 onodes shared_blobs 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1500 allocated 0x[] released 0x[] 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3366 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 2 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 prepare 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1500 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 io_done 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7810 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f0117b700 10 osd.7 45 dequeue_op 0x558c5511fce0 finish 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34000 had 1 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7833 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7833 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x240c8e~1e99 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x240c8e 0x240c8e~1e99 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x210c8e 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc8e 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb27 and padding block with 0x4d9 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57db2900> 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2900 kv_submitted 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db2900 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2900 finishing 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db2900 onodes 0x558c57598000,0x558c575f9b80 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db2900 done 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db2900 [] 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.642+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 45/41 ECSubWriteReply(tid=227, last_complete=45'39, committed=1, applied=1)) v2 -- 0x558c57518600 con 0x558c54c64000 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x242b27 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000304358s (0.000000100s flush + 0.000304258s kv commit) 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x242b27~471 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x242b27 0x242b27~471 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x212b27 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb27 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf98 and padding block with 0x68 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58000,0x558c55114900,0x558c54d59b00,0x558c57db2c00> 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 kv_submitted 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58000 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58000 finishing 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58000 onodes 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58000 done 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114900 kv_submitted 2019-12-07T19:34:22.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58000 [] 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114900 kv_submitted 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55114900 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114900 finishing 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55114900 onodes 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114900 done 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55114900 [] 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 kv_submitted 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d59b00 2019-12-07T19:34:22.642+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 45/28 ECSubWriteReply(tid=128, last_complete=45'50, committed=1, applied=1)) v2 -- 0x558c576b4e00 con 0x558c54c5bc00 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d59b00 finishing 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d59b00 onodes 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d59b00 done 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d59b00 [] 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2c00 kv_submitted 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db2c00 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2c00 finishing 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db2c00 onodes 0x558c575bb900,0x558c57f34000 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db2c00 done 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db2c00 [] 2019-12-07T19:34:22.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.642+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 45/18 ECSubWriteReply(tid=229, last_complete=45'19, committed=1, applied=1)) v2 -- 0x558c54ceac00 con 0x558c54c64000 2019-12-07T19:34:22.642+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.642+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 234 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=91, reqid=client.4540.0:370, at_version=45'29, trim_to=0'0, roll_forward_to=45'25)) v2 ==== 2235+0+0 (crc 0 0 0) 0x558c57e00800 con 0x558c54c5b400 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'27 (0'0,45'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'26 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] op_applied version 45'27 2019-12-07T19:34:22.642+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c57f3f8c0 prio 127 cost 0 latency 0.000024 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=91, reqid=client.4540.0:370, at_version=45'29, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.642+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=91, reqid=client.4540.0:370, at_version=45'29, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c5511fb80 finish 2019-12-07T19:34:22.642+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=90, reqid=client.4540.0:369, at_version=45'28, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.642+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.642+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=90, reqid=client.4540.0:369, at_version=45'28, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c5511fe40 prio 127 cost 0 latency 0.001298 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=90, reqid=client.4540.0:369, at_version=45'28, trim_to=0'0, roll_forward_to=45'25)) v2 pg pg[1.bs1( v 45'27 (0'0,45'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'27 (0'0,45'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=90, reqid=client.4540.0:369, at_version=45'28, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'27 (0'0,45'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log log((0'0,45'27], crt=45'25) [45'28 (45'27) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:369 2019-12-07T19:34:22.635357+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1343488~18446744073708208127], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'28 (0'0,45'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'27 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] add_log_entry 45'28 (45'27) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:369 2019-12-07T19:34:22.635357+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'28 (0'0,45'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'27 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log approx pg log length = 28 2019-12-07T19:34:22.642+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'28 (0'0,45'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'27 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.643+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'28, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57d0af00 seq 31 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x3d000~15000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0x3d000~15000 - have 0x3d000 (249856) bytes fadvise_flags 0x0 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x3d000~15000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57d0af00 11 blobs 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x15000/1000,15000,0 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x9d6000~15000/1000,15000,0 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x9d6000~15000] 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e633b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57567110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567110 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c57e633b0) get_ref 0x0~2000 Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567110 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c57e633b0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57567128 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c551079d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107a40 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c551079d0) get_ref 0x0~2000 Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107a40 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c551079d0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed1e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf96c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf96c0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c57ed1e30) get_ref 0x0~2000 Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf96c0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c57ed1e30) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 349 ==== MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=230, reqid=osd.1.0:227, at_version=0'0, trim_to=0'0, roll_forward_to=45'39)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57e00000 con 0x558c54c64000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c54cf96d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107b20 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511e000 prio 127 cost 0 latency 0.000023 epoch 45 MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=230, reqid=osd.1.0:227, at_version=0'0, trim_to=0'0, roll_forward_to=45'39)) v2 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107ab0) get_ref 0x0~2000 Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107b20 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=230, reqid=osd.1.0:227, at_version=0'0, trim_to=0'0, roll_forward_to=45'39)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107c00 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107b90) get_ref 0x0~2000 Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107c00 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107b90) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ce0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107c70) get_ref 0x0~2000 Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ce0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107c70) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107cf8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=230, reqid=osd.1.0:227, at_version=0'0, trim_to=0'0, roll_forward_to=45'39)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107dc0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=230, reqid=osd.1.0:227, at_version=0'0, trim_to=0'0, roll_forward_to=45'39)) v2) prio 127 cost 0 e45) pg 0x558c57380000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107d50) get_ref 0x0~2000 Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107dc0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107d50) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107dd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ea0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107e30) get_ref 0x0~2000 Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ea0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107e30) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c5511e000 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=230, reqid=osd.1.0:227, at_version=0'0, trim_to=0'0, roll_forward_to=45'39)) v2 pg pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107eb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55107f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55107f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107f80 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.8s5 45/41 ECSubWrite(tid=230, reqid=osd.1.0:227, at_version=0'0, trim_to=0'0, roll_forward_to=45'39)) v2 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107f10) get_ref 0x0~2000 Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107f80 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c55107f10) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55107f98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a2000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a2070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=44'37 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'39], crt=44'37) [] 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2070 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2000) get_ref 0x0~2000 Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2070 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2000) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580a2088 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a20e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'38 (44'34) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:22.638555+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2150 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c580a20e0) get_ref 0x1000~1000 Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2150 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.blob(0x558c580a20e0) get_ref init 0x2000, 1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x51000~1000: 0x1000~1000 Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580a2168 in 0x558c53a59c00) _discard 0x1000~1000 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x52000 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'39 (44'37) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:22.638555+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark inline shard dirty 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x3d000~15000 = 0 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 39 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.643+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000028' <- 0000000045.00000000000000000028 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0af00 cost 135994 (12 ios * 4000 + 87994 bytes) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0af00 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update inline shard 812 bytes from 42 extents 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57db2f00 seq 30 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 1231 (413 bytes onode + 2 bytes spanning blobs + 816 bytes inline extents) 2019-12-07T19:34:22.643+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0af00 allocated 0x[9d6000~15000] released 0x[] 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 freelist allocate 0x9d6000~15000 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 freelist _xor first_key 0x980000 last_key 0x980000 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db2f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db2f00 onodes shared_blobs 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0af00 prepare 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57d0af00 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db2f00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2f00 prepare 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db2f00 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2f00 io_done 2019-12-07T19:34:22.643+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c5511e000 finish 2019-12-07T19:34:22.643+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'28 (0'0,45'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'27 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] op_applied version 45'28 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c5511fe40 finish 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=91, reqid=client.4540.0:370, at_version=45'29, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=91, reqid=client.4540.0:370, at_version=45'29, trim_to=0'0, roll_forward_to=45'25)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c57f3f8c0 prio 127 cost 0 latency 0.000604 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=91, reqid=client.4540.0:370, at_version=45'29, trim_to=0'0, roll_forward_to=45'25)) v2 pg pg[1.bs1( v 45'28 (0'0,45'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'28 (0'0,45'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=91, reqid=client.4540.0:370, at_version=45'29, trim_to=0'0, roll_forward_to=45'25)) v2 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'28 (0'0,45'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log log((0'0,45'28], crt=45'25) [45'29 (45'28) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:370 2019-12-07T19:34:22.635389+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'28 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] add_log_entry 45'29 (45'28) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:370 2019-12-07T19:34:22.635389+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'28 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log approx pg log length = 29 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'28 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.643+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 45'29, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c57d0b200 seq 32 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys = 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000045.00000000000000000029' <- 0000000045.00000000000000000029 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0b200 cost 5885 (1 ios * 4000 + 1885 bytes) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0b200 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 1304 (486 bytes onode + 2 bytes spanning blobs + 816 bytes inline extents) 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0b200 allocated 0x[] released 0x[] 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b200 prepare 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0b200 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0b200 blocked by 0x558c57d0af00 aio_wait 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'28 crt=45'25 lcod 45'25 active mbc={} ps=[a~1,c~2,10~2]] op_applied version 45'29 2019-12-07T19:34:22.643+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c57f3f8c0 finish 2019-12-07T19:34:22.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.643+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0ac00 aio_wait 2019-12-07T19:34:22.643+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0ac00 2019-12-07T19:34:22.643+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0ac00 io_done 2019-12-07T19:34:22.643+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 45/41 ECSubWriteReply(tid=88, last_complete=45'26, committed=1, applied=1)) v2 -- 0x558c5841fe00 con 0x558c54c5b400 2019-12-07T19:34:22.643+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 13 ==== osd_ping(ping e45 up_from 13 ping_stamp 2019-12-07T19:34:22.644817+0000/46.116536446s send_stamp 46.116536446s delta_ub 0.036615155s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54ceac00 con 0x558c552d1c00 2019-12-07T19:34:22.643+0000 7f8f24457700 20 osd.7 45 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036146400s,0.036615155s]) 2019-12-07T19:34:22.643+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 13 ==== osd_ping(ping e45 up_from 13 ping_stamp 2019-12-07T19:34:22.644817+0000/46.116536446s send_stamp 46.116536446s delta_ub 0.036615155s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841ec00 con 0x558c552d1800 2019-12-07T19:34:22.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e45 up_from 13 ping_stamp 2019-12-07T19:34:22.644817+0000/46.116536446s send_stamp 46.080390046s delta_ub 18446744073.673405216s) v5 -- 0x558c58332600 con 0x558c552d1c00 2019-12-07T19:34:22.643+0000 7f8f24c58700 20 osd.7 45 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036121748s,0.036615155s]) 2019-12-07T19:34:22.643+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e45 up_from 13 ping_stamp 2019-12-07T19:34:22.644817+0000/46.116536446s send_stamp 46.080414698s delta_ub 18446744073.673429868s) v5 -- 0x558c58332000 con 0x558c552d1800 2019-12-07T19:34:22.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 350 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:22.644+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.8s5 epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:22.644+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.644+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.644+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.644+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.644+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e45) pg 0x558c57380000 2019-12-07T19:34:22.644+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:22.644+0000 7f8f04982700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.644+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:22.644+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:22.644+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c574fef00 seq 31 2019-12-07T19:34:22.644+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:22.644+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:22.644+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:22.644+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:22.644+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fef00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.644+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fef00 onodes shared_blobs 2019-12-07T19:34:22.644+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fef00 allocated 0x[] released 0x[] 2019-12-07T19:34:22.644+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 prepare 2019-12-07T19:34:22.644+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fef00 2019-12-07T19:34:22.644+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 io_done 2019-12-07T19:34:22.644+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x242f98 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002556099s (0.000000089s flush + 0.002556010s kv commit) 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1500> 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 kv_submitted 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1500 2019-12-07T19:34:22.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1500 finishing 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1500 onodes 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1500 done 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:22.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1500 [] 2019-12-07T19:34:22.645+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.645+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0af00 aio_wait 2019-12-07T19:34:22.645+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0af00 2019-12-07T19:34:22.645+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0af00 io_done 2019-12-07T19:34:22.645+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b200 io_done 2019-12-07T19:34:22.645+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:22.645+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 45/41 ECSubWriteReply(tid=228, last_complete=45'23, committed=1, applied=1)) v2 -- 0x558c55045a00 con 0x558c54c64000 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 2 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3263 < min_flush_size 524288 2019-12-07T19:34:22.645+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 3 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 3 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4389 < min_flush_size 524288 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4412 < min_flush_size 524288 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4412 < min_flush_size 524288 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x242f98~113c to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.645+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x242f98 0x242f98~113c to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x212f98 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf98 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.645+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd4 and padding block with 0xf2c 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2440d4 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.002345487s (0.000089337s flush + 0.002256150s kv commit) 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2371 < min_flush_size 524288 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 2 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 2 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4715 < min_flush_size 524288 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57db2f00,0x558c57d0ac00,0x558c574fef00> 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2f00 kv_submitted 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db2f00 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db2f00 finishing 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db2f00 onodes 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db2f00 done 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4738 < min_flush_size 524288 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fef00 kv_submitted 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4738 < min_flush_size 524288 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db2f00 [] 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0ac00 kv_submitted 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0ac00 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0ac00 finishing 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2440d4~1282 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0ac00 onodes 0x558c54e44780 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677828 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fb90 space 0x558c54677828 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2440d4 0x2440d4~1282 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b2ef8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fcb0 space 0x558c550b2ef8 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x2140d4 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3668 in 0x558c53a59c00) _finish_write discard buffer(0x558c581f4240 space 0x558c550b3668 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd4 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55106e88 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fd40 space 0x558c55106e88 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x356 and padding block with 0xcaa 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55106f68 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fdd0 space 0x558c55106f68 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107048 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fe60 space 0x558c55107048 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107128 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fef0 space 0x558c55107128 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107208 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e000 space 0x558c55107208 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c551072e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e090 space 0x558c551072e8 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c551073c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e120 space 0x558c551073c8 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c552790b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fb00 space 0x558c552790b8 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fd978 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fc20 space 0x558c574fd978 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759fa70 space 0x558c576adba8 0x0~2000 writing nocache) 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0ac00 done 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0af00 kv_submitted 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0ac00 [] 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 kv_submitted 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fef00 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fef00 finishing 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fef00 onodes 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fef00 done 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fef00 [] 2019-12-07T19:34:22.647+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.647+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.647+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.647+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 45/41 ECSubWriteReply(tid=89, last_complete=45'27, committed=1, applied=1)) v2 -- 0x558c5795c600 con 0x558c54c5b400 2019-12-07T19:34:22.647+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 45/41 ECSubWriteReply(tid=230, last_complete=45'39, committed=1, applied=1)) v2 -- 0x558c576b4c00 con 0x558c54c64000 2019-12-07T19:34:22.647+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.647+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.647+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.649+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x245356 2019-12-07T19:34:22.649+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.649+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.649+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.002322247s (0.000002720s flush + 0.002319527s kv commit) 2019-12-07T19:34:22.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57d0af00,0x558c57d0b200> 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0af00 kv_submitted 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0af00 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0af00 finishing 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0af00 onodes 0x558c54e44780 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf96d8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e2d0 space 0x558c54cf96d8 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107a58 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e240 space 0x558c55107a58 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107b38 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e360 space 0x558c55107b38 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107c18 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e3f0 space 0x558c55107c18 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107cf8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e480 space 0x558c55107cf8 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107dd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e510 space 0x558c55107dd8 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107eb8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e5a0 space 0x558c55107eb8 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55107f98 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e630 space 0x558c55107f98 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57567128 in 0x558c53a59c00) _finish_write discard buffer(0x558c5759ea20 space 0x558c57567128 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a2088 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e6c0 space 0x558c580a2088 0x0~2000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a2168 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e750 space 0x558c580a2168 0x1000~1000 writing nocache) 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0af00 done 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0b200 kv_submitted 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0af00 [] 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b200 kv_submitted 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0b200 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b200 finishing 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0b200 onodes 0x558c54e44780 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0b200 done 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0b200 [] 2019-12-07T19:34:22.649+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.649+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 45/41 ECSubWriteReply(tid=90, last_complete=45'28, committed=1, applied=1)) v2 -- 0x558c55045e00 con 0x558c54c5b400 2019-12-07T19:34:22.649+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.650+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 45/41 ECSubWriteReply(tid=91, last_complete=45'29, committed=1, applied=1)) v2 -- 0x558c57569a00 con 0x558c54c5b400 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.650+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.650+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 351 ==== MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=231, reqid=osd.1.0:229, at_version=0'0, trim_to=0'0, roll_forward_to=45'19)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e1000 con 0x558c54c64000 2019-12-07T19:34:22.650+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511e160 prio 127 cost 0 latency 0.000020 epoch 45 MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=231, reqid=osd.1.0:229, at_version=0'0, trim_to=0'0, roll_forward_to=45'19)) v2 2019-12-07T19:34:22.650+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=231, reqid=osd.1.0:229, at_version=0'0, trim_to=0'0, roll_forward_to=45'19)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=231, reqid=osd.1.0:229, at_version=0'0, trim_to=0'0, roll_forward_to=45'19)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=231, reqid=osd.1.0:229, at_version=0'0, trim_to=0'0, roll_forward_to=45'19)) v2) prio 127 cost 0 e45) pg 0x558c5500d000 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c5511e160 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=231, reqid=osd.1.0:229, at_version=0'0, trim_to=0'0, roll_forward_to=45'19)) v2 pg pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] _handle_message: MOSDECSubOpWrite(1.0s3 45/18 ECSubWrite(tid=231, reqid=osd.1.0:229, at_version=0'0, trim_to=0'0, roll_forward_to=45'19)) v2 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=38'17 active mbc={} ps=[a~1,d~1,10~2]] append_log log((0'0,45'19], crt=38'17) [] 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'18 (38'16) modify 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:22.642501+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,d~1,10~2]] rollforward: entry=45'19 (38'17) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:22.642501+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,d~1,10~2]] append_log approx pg log length = 19 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,d~1,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.650+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c57db3200 seq 37 2019-12-07T19:34:22.650+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57db3200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57db3200 onodes shared_blobs 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57db3200 allocated 0x[] released 0x[] 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3200 prepare 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57db3200 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3200 io_done 2019-12-07T19:34:22.650+0000 7f8f0898a700 10 osd.7 45 dequeue_op 0x558c5511e160 finish 2019-12-07T19:34:22.650+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:22.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:22.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:22.650+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 235 ==== MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=92, reqid=client.4540.0:370, at_version=0'0, trim_to=0'0, roll_forward_to=45'29)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e3800 con 0x558c54c5b400 2019-12-07T19:34:22.650+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.650+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c5511e2c0 prio 127 cost 0 latency 0.000013 epoch 45 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=92, reqid=client.4540.0:370, at_version=0'0, trim_to=0'0, roll_forward_to=45'29)) v2 2019-12-07T19:34:22.650+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=92, reqid=client.4540.0:370, at_version=0'0, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e45) 2019-12-07T19:34:22.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x245356~471 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.650+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x245356 0x245356~471 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x215356 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x356 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.650+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7c7 and padding block with 0x839 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=92, reqid=client.4540.0:370, at_version=0'0, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=92, reqid=client.4540.0:370, at_version=0'0, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.650+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5511e2c0 prio 127 cost 0 latency 0.000039 MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=92, reqid=client.4540.0:370, at_version=0'0, trim_to=0'0, roll_forward_to=45'29)) v2 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.650+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpWrite(1.bs1 45/41 ECSubWrite(tid=92, reqid=client.4540.0:370, at_version=0'0, trim_to=0'0, roll_forward_to=45'29)) v2 2019-12-07T19:34:22.650+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'25 active mbc={} ps=[a~1,c~2,10~2]] append_log log((0'0,45'29], crt=45'25) [] 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'26 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:367 2019-12-07T19:34:22.630130+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'27 (45'26) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:368 2019-12-07T19:34:22.633053+0000 0 ObjectCleanRegions clean_offsets: [0~573440,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'28 (45'27) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:369 2019-12-07T19:34:22.635357+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] rollforward: entry=45'29 (45'28) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:370 2019-12-07T19:34:22.635389+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:22.650+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] append_log approx pg log length = 29 2019-12-07T19:34:22.650+0000 7f8f07187700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] append_log transaction_applied = 1 2019-12-07T19:34:22.650+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:22.650+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:22.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:22.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c583c9200 seq 33 2019-12-07T19:34:22.650+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:22.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:22.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:22.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:22.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:22.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c9200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:22.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c9200 onodes shared_blobs 2019-12-07T19:34:22.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c9200 allocated 0x[] released 0x[] 2019-12-07T19:34:22.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9200 prepare 2019-12-07T19:34:22.650+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c9200 2019-12-07T19:34:22.650+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9200 io_done 2019-12-07T19:34:22.650+0000 7f8f07187700 10 osd.7 45 dequeue_op 0x558c5511e2c0 finish 2019-12-07T19:34:22.650+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 352 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e45/45) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55054a00 con 0x558c54c64000 2019-12-07T19:34:22.651+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.0s3 epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:22.651+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 236 ==== MOSDECSubOpRead(1.bs1 45/41 ECSubRead(tid=93, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c55113400 con 0x558c54c5b400 2019-12-07T19:34:22.651+0000 7f8f24457700 15 osd.7 45 enqueue_op 0x558c578d26e0 prio 127 cost 0 latency 0.000014 epoch 45 MOSDECSubOpRead(1.bs1 45/41 ECSubRead(tid=93, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:22.651+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 45/41 ECSubRead(tid=93, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e45) 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 45/41 ECSubRead(tid=93, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e45) queued 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 45/41 ECSubRead(tid=93, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e45) pg 0x558c575ef000 2019-12-07T19:34:22.651+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c578d26e0 prio 127 cost 0 latency 0.000066 MOSDECSubOpRead(1.bs1 45/41 ECSubRead(tid=93, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] 2019-12-07T19:34:22.651+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] _handle_message: MOSDECSubOpRead(1.bs1 45/41 ECSubRead(tid=93, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:22.651+0000 7f8f0317f700 25 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:22.651+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x52000 (335872) 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:22.651+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2457c7 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001880293s (0.000000096s flush + 0.001880197s kv commit) 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57db3200> 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3200 kv_submitted 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57db3200 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57db3200 finishing 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57db3200 onodes 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57db3200 done 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57db3200 [] 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e45) pg 0x558c5500d000 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,d~1,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2457c7~471 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,d~1,10~2]] proc_primary_info updating purged_snaps to [a~1,c~2,10~2] 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2457c7 0x2457c7~471 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x2157c7 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7c7 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc38 and padding block with 0x3c8 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c55122000 seq 38 2019-12-07T19:34:22.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:22.652+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:22.652+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d79e8 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:22.652+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57b8e7e0 space 0x558c580d79e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55122000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55122000 onodes shared_blobs 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55122000 allocated 0x[] released 0x[] 2019-12-07T19:34:22.652+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 = 4096 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122000 prepare 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55122000 2019-12-07T19:34:22.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122000 io_done 2019-12-07T19:34:22.652+0000 7f8f0317f700 20 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:22.652+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 45/18 ECSubWriteReply(tid=231, last_complete=45'19, committed=1, applied=1)) v2 -- 0x558c54dfbc00 con 0x558c54c64000 2019-12-07T19:34:22.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:22.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.652+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:22.652+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:22.652+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x245c38 2019-12-07T19:34:22.652+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.652+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 45/41 ECSubReadReply(tid=93, attrs_read=0)) v2 -- 0x558c57573900 con 0x558c54c5b400 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000139799s (0.000000102s flush + 0.000139697s kv commit) 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583c9200> 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9200 kv_submitted 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c9200 2019-12-07T19:34:22.652+0000 7f8f0317f700 10 osd.7 45 dequeue_op 0x558c578d26e0 finish 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9200 finishing 2019-12-07T19:34:22.652+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c9200 onodes 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c9200 done 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c9200 [] 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x245c38~47d to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x245c38 0x245c38~47d to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x215c38 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc38 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb5 and padding block with 0xf4b 2019-12-07T19:34:22.652+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 45/41 ECSubWriteReply(tid=92, last_complete=45'29, committed=1, applied=1)) v2 -- 0x558c57518600 con 0x558c54c5b400 2019-12-07T19:34:22.652+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2460b5 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:22.652+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000133119s (0.000000080s flush + 0.000133039s kv commit) 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:22.652+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55122000> 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122000 kv_submitted 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55122000 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122000 finishing 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55122000 onodes 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55122000 done 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55122000 [] 2019-12-07T19:34:22.652+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:22.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.652+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:22.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:22.656+0000 7f8f0998c700 15 osd.7 45 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:22.656+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) queued 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:22.656+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.5 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.4 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.1 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.2 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.3 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until readable_until[_ub] 54.092497800s (sent 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.cs3 pg_lease(ru 54.092497800s ub 62.092697193s int 16s) e45/45) v1 -- 0x558c57518a00 con 0x558c54c64000 2019-12-07T19:34:22.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.cs4 pg_lease(ru 54.092497800s ub 62.092697193s int 16s) e45/45) v1 -- 0x558c57518800 con 0x558c54c19800 2019-12-07T19:34:22.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.cs5 pg_lease(ru 54.092497800s ub 62.092697193s int 16s) e45/45) v1 -- 0x558c5795c800 con 0x558c54c5bc00 2019-12-07T19:34:22.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.cs2 pg_lease(ru 54.092497800s ub 62.092697193s int 16s) e45/45) v1 -- 0x558c57518c00 con 0x558c54c5b400 2019-12-07T19:34:22.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.cs1 pg_lease(ru 54.092497800s ub 62.092697193s int 16s) e45/45) v1 -- 0x558c58332400 con 0x558c54c19c00 2019-12-07T19:34:22.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 92 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 62.092697193s) e45/45) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5734b0e0 con 0x558c54c19c00 2019-12-07T19:34:22.656+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.cs0 epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.5 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.5 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.5 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.5 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) pg 0x558c57676000 2019-12-07T19:34:22.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.5 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 140 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 62.092697193s) e45/45) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54d9ef00 con 0x558c54c19800 2019-12-07T19:34:22.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 237 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 62.092697193s) e45/45) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5734b0e0 con 0x558c54c5b400 2019-12-07T19:34:22.656+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.cs0 epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.4 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f24c58700 15 osd.7 45 enqueue_peering_evt 1.cs0 epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.2 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.4 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.656+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.2 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.5 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.4 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.1 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.2 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.4 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.3 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until readable_until[_ub] 54.092497800s (sent 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 353 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 62.092697193s) e45/45) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5734b0e0 con 0x558c54c64000 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:22.656+0000 7f8f24457700 15 osd.7 45 enqueue_peering_evt 1.cs0 epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.1 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:22.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.1 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.2 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.4 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) pg 0x558c57676000 2019-12-07T19:34:22.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 201 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 62.092697193s) e45/45) v1 ==== 42+0+0 (crc 0 0 0) 0x558c57edef00 con 0x558c54c5bc00 2019-12-07T19:34:22.656+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.4 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f23c56700 15 osd.7 45 enqueue_peering_evt 1.cs0 epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.3 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.3 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.5 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.4 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.1 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.2 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.3 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until readable_until[_ub] 54.092497800s (sent 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:22.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.1 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.2 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) pg 0x558c57676000 2019-12-07T19:34:22.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.2 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.5 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.4 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.1 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.3 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until readable_until[_ub] 54.092497800s (sent 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:22.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.3 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.1 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) pg 0x558c57676000 2019-12-07T19:34:22.656+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.1 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.5 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.4 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.1 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.3 ruub 54.092497800s 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until readable_until[_ub] 54.092497800s (sent 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:22.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:22.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.3 pg_lease_ack(ruub 62.092697193s)) prio 255 cost 10 e45) pg 0x558c57676000 2019-12-07T19:34:22.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MLeaseAck epoch 45 from osd.3 pg_lease_ack(ruub 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.5 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.4 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.1 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until peer osd.3 ruub 62.092697193s 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recalc_readable_until readable_until[_ub] 62.092697193s (sent 62.092697193s) 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:22.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:22.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:22.657+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 202 ==== pg_lease(1.6s2 pg_lease(ru 54.116153259s ub 62.116337411s int 16s) e45/45) v1 ==== 58+0+0 (crc 0 0 0) 0x558c576b4e00 con 0x558c54c5bc00 2019-12-07T19:34:22.657+0000 7f8f23c56700 15 osd.7 45 enqueue_peering_evt 1.6s2 epoch_sent: 45 epoch_requested: 45 MLease epoch 45 from osd.3 pg_lease(ru 54.116153259s ub 62.116337411s int 16s) 2019-12-07T19:34:22.657+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLease epoch 45 from osd.3 pg_lease(ru 54.116153259s ub 62.116337411s int 16s)) prio 255 cost 10 e45) 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLease epoch 45 from osd.3 pg_lease(ru 54.116153259s ub 62.116337411s int 16s)) prio 255 cost 10 e45) queued 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 45 epoch_requested: 45 MLease epoch 45 from osd.3 pg_lease(ru 54.116153259s ub 62.116337411s int 16s)) prio 255 cost 10 e45) pg 0x558c54d5c000 2019-12-07T19:34:22.657+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 45 epoch_requested: 45 MLease epoch 45 from osd.3 pg_lease(ru 54.116153259s ub 62.116337411s int 16s) 2019-12-07T19:34:22.657+0000 7f8f05984700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] proc_lease pg_lease(ru 54.116153259s ub 62.116337411s int 16s) 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] peer_clock_delta_ub 0.022361165s -> ru 54.093792094s 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] proc_lease readable_until now 54.093792094s 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] peer_clock_delta_lb 0.022015059s -> ruub 62.094322352s 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] proc_lease readable_until_ub now 62.094322352s 2019-12-07T19:34:22.657+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.6s0 pg_lease_ack(ruub 62.116337411s) e45/45) v1 -- 0x558c57edf0e0 con 0x558c54c5bc00 2019-12-07T19:34:22.657+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:22.657+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:22.665+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.715+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 10 osd.7 45 tick_without_osd_lock 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f615000/0x40000000/0x1680000000, data 0xb21c99/0x9e9000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 25 osd.7 45 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075394+0000 last_rx_front 2019-12-07T19:34:22.075339+0000 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 25 osd.7 45 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075420+0000 last_rx_front 2019-12-07T19:34:22.075379+0000 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 25 osd.7 45 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075351+0000 last_rx_front 2019-12-07T19:34:22.075363+0000 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 25 osd.7 45 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075252+0000 last_rx_front 2019-12-07T19:34:22.075295+0000 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 25 osd.7 45 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075263+0000 last_rx_front 2019-12-07T19:34:22.075319+0000 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 20 osd.7 45 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 10 osd.7 45 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 20 osd.7 45 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:22.717+0000 7f8f1fbbe700 10 osd.7 45 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:22.726+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:22.754+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 12 ==== osd_ping(ping e45 up_from 13 ping_stamp 2019-12-07T19:34:22.755661+0000/46.212584505s send_stamp 46.212584505s delta_ub 0.021796272s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4e00 con 0x558c54c19000 2019-12-07T19:34:22.754+0000 7f8f23c56700 20 osd.7 45 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021341840s,0.021796272s]) 2019-12-07T19:34:22.754+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e45 up_from 13 ping_stamp 2019-12-07T19:34:22.755661+0000/46.212584505s send_stamp 46.191242665s delta_ub 18446744073.688209776s) v5 -- 0x558c576b5400 con 0x558c54c19000 2019-12-07T19:34:22.754+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 12 ==== osd_ping(ping e45 up_from 13 ping_stamp 2019-12-07T19:34:22.755661+0000/46.212584505s send_stamp 46.212584505s delta_ub 0.021796272s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795c800 con 0x558c54c18800 2019-12-07T19:34:22.754+0000 7f8f23c56700 20 osd.7 45 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021267595s,0.021796272s]) 2019-12-07T19:34:22.754+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e45 up_from 13 ping_stamp 2019-12-07T19:34:22.755661+0000/46.212584505s send_stamp 46.191316910s delta_ub 18446744073.688284021s) v5 -- 0x558c576b5400 con 0x558c54c18800 2019-12-07T19:34:22.765+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.815+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.866+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.915+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:22.915+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:22.965+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.015+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.065+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.115+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.166+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.208+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:23.216+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.266+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.316+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.367+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.395+0000 7f8f21451700 10 osd.7 45 tick 2019-12-07T19:34:23.395+0000 7f8f21451700 10 osd.7 45 do_waiters -- start 2019-12-07T19:34:23.395+0000 7f8f21451700 10 osd.7 45 do_waiters -- finish 2019-12-07T19:34:23.395+0000 7f8f21451700 20 osd.7 45 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:23.410+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:23.416+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.467+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.517+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.567+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1210936 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.602+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 28 ==== osd_map(45..46 src has 1..46) v4 ==== 1318+0+0 (crc 0 0 0) 0x558c57324480 con 0x558c5476d000 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 osd.7 45 OSD::ms_dispatch: osd_map(45..46 src has 1..46) v4 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 osd.7 45 do_waiters -- start 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 osd.7 45 do_waiters -- finish 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 osd.7 45 _dispatch 0x558c57324480 osd_map(45..46 src has 1..46) v4 2019-12-07T19:34:23.602+0000 7f8f164bb700 3 osd.7 45 handle_osd_map epochs [45,46], i have 45, src has [1,46] 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 osd.7 45 handle_osd_map got inc map for epoch 46 2019-12-07T19:34:23.602+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ce8e826:::osdmap.45:0# 0x0~0 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c39110 blob([0x997000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c39180 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c391f0 blob([!~1000,0x998000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57c39260 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ce8e826:::osdmap.45:0# 0x0~1464 = 5220 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 osd.7 45 add_map_bl 45 5220 bytes 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 osd.7 45 got_full_map 46, nothing requested 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 osd.7 45 handle_osd_map pg_num_history pg_num_history(e46 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {46={1=[a~1,d~1,10~1]}} 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) lower_bound to PSN__1_0000000000000009 key 0x00000000000000000000000000000403'.PSN__1_0000000000000009' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 9 key 'PSN__1_0000000000000009' lower_bound not found 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) lower_bound to PSN__1_000000000000000b key 0x00000000000000000000000000000403'.PSN__1_000000000000000b' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap b key 'PSN__1_000000000000000b' lower_bound not found 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) lower_bound to PSN__1_000000000000000c key 0x00000000000000000000000000000403'.PSN__1_000000000000000c' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap c key 'PSN__1_000000000000000c' lower_bound not found 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) lower_bound to PSN__1_000000000000000e key 0x00000000000000000000000000000403'.PSN__1_000000000000000e' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap e key 'PSN__1_000000000000000e' lower_bound not found 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) lower_bound to PSN__1_000000000000000f key 0x00000000000000000000000000000403'.PSN__1_000000000000000f' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap f key 'PSN__1_000000000000000f' lower_bound not found 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:23.602+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) lower_bound to PSN__1_0000000000000011 key 0x00000000000000000000000000000403'.PSN__1_0000000000000011' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c575c80c0) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:23.602+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 11 key 'PSN__1_0000000000000011' lower_bound not found 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 3 keys 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 osd.7 45 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e46 [1,46] lci=[13,46]) 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c54d58c00 seq 37 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cff7972f:::inc_osdmap.46:0# key 0x7f7fffffffffffffffcff797'/!inc_osdmap.46!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cff7972f:::inc_osdmap.46:0# 0x0~289 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1228 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cff7972f:::inc_osdmap.46:0# 0x0~289 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~289 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd77 on front/back, now 0x0~1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58c00 1 blobs 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x9eb000~1000/1000,1000,0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x9eb000~1000] 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff40e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060a10 blob([0x9eb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff40e0 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c55060a10) get_ref 0x0~289 Blob(0x558c55060a10 blob([0x9eb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff40e0 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c55060a10) get_ref init 0x1000, 1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~289: 0x0~289 Blob(0x558c55060a10 blob([0x9eb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x289) SharedBlob(0x558c57ff40e0 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57ff40f8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x289 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c108) dirty_range mark inline shard dirty 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cff7972f:::inc_osdmap.46:0# 0x0~289 = 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3328e826:::osdmap.46:0# key 0x7f7fffffffffffffff3328e8'&!osdmap.46!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3328e826:::osdmap.46:0# 0x0~1474 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1229 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3328e826:::osdmap.46:0# 0x0~1474 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~474 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb8c on front/back, now 0x0~1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58c00 2 blobs 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x9ec000~2000/1000,2000,0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x9ec000~2000] 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df8540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060540 blob([0x9ec000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8540 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c55060540) get_ref 0x0~1000 Blob(0x558c55060540 blob([0x9ec000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8540 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c55060540) get_ref init 0x1000, 1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55060540 blob([0x9ec000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df8540 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54df8558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df84d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54df9b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df84d0 blob([!~1000,0x9ed000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9b20 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54df84d0) get_ref 0x1000~474 Blob(0x558c54df84d0 blob([!~1000,0x9ed000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c54df9b20 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54df84d0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~474: 0x1000~474 Blob(0x558c54df84d0 blob([!~1000,0x9ed000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,474]) SharedBlob(0x558c54df9b20 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54df9b38 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1474 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c388) dirty_range mark inline shard dirty 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3328e826:::osdmap.46:0# 0x0~1474 = 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58c00 0 blobs 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_000000000000000a' <- PSN__1_000000000000000a 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_000000000000000d' <- PSN__1_000000000000000d 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000010' <- PSN__1_0000000000000010 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54d58c00 0 blobs 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54d58c00 cost 23531 (4 ios * 4000 + 7531 bytes) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54d58c00 onodes 0x558c53a91900,0x558c55276c80,0x558c5738c000,0x558c5738c280 shared_blobs 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c108) update #-1:cff7972f:::inc_osdmap.46:0# 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c108) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cff7972f:::inc_osdmap.46:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c388) update #-1:3328e826:::osdmap.46:0# 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738c388) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3328e826:::osdmap.46:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54d58c00 allocated 0x[9eb000~3000] released 0x[] 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 freelist allocate 0x9eb000~3000 2019-12-07T19:34:23.603+0000 7f8f164bb700 20 freelist _xor first_key 0x980000 last_key 0x980000 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 prepare 2019-12-07T19:34:23.603+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54d58c00 2019-12-07T19:34:23.603+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 aio_wait 2019-12-07T19:34:23.603+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54d58c00 2019-12-07T19:34:23.603+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 io_done 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8011 < min_flush_size 524288 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9007 < min_flush_size 524288 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738c000 had 1 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738c280 had 1 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9030 < min_flush_size 524288 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9030 < min_flush_size 524288 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2460b5~2346 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2460b5 0x2460b5~2346 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x2160b5 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb5 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3fb and padding block with 0xc05 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2483fb 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.603+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000155407s (0.000040407s flush + 0.000115000s kv commit) 2019-12-07T19:34:23.603+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54d58c00> 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.603+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 kv_submitted 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54d58c00 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c54d58c00 osr 0x558c54ea4b40 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 34 0x1b000~1000 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7 45 _committed_osd_maps 46..46 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7 45 advance to epoch 46 (<= last 46 <= newest_map 46) 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 35 0x1b000~1000 crc 5bf8beb 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 34 0x2000~1000 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 35 0x2000~1000 crc 71e189 2019-12-07T19:34:23.603+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.603+0000 7f8f141a1700 7 osd.7 46 consume_map version 46 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.8s5 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.0s3 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.1s1 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.9s4 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.as5 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.2s0 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.bs1 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.3s4 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.cs0 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.4s0 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.5s4 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.ds2 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.es4 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.6s2 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.fs5 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 identify_splits_and_merges 1.7s4 e45 to e46 pg_nums {18=16} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 46 prune_pg_created 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:0.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:1.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:2.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:3.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:4.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:5.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:6.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7:7.consume_map 46 (was 45) 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:23.603+0000 7f8f141a1700 10 osd.7 46 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.8s5 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.0s3 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.1s1 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.9s4 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.as5 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.2s0 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c5736a000 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c57380000 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c57e8b000 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.bs1 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.3s4 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.cs0 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.4s0 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.5s4 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.ds2 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 45 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.es4 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.6s2 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.fs5 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f141a1700 15 osd.7 46 enqueue_peering_evt 1.7s4 epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:23.603+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f141a1700 7 osd.7 46 activate_map version 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c54d98000 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c57383000 2019-12-07T19:34:23.603+0000 7f8f0918b700 20 osd.7 46 agent_entry empty queue 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c57e88000 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c5500a000 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c57522000 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 45 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:23.603+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 45 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:23.603+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c57676000 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[a~1,c~2,10~2]] state: Active advmap 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] on_active_advmap new purged_snaps [a~1,d~1,10~1], now [c~1,11~1] 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] share_pg_info 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[a~1,c~2,10~2]] state: Active advmap 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c57392a00 con 0x558c54c64000 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] on_active_advmap new purged_snaps [a~1,d~1,10~1], now [c~1,11~1] 2019-12-07T19:34:23.603+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] share_pg_info 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.603+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:23.603+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e46/46) v1 -- 0x558c5737b800 con 0x558c54c64000 2019-12-07T19:34:23.603+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c55055100 con 0x558c54c19800 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e46/46) v1 -- 0x558c58134000 con 0x558c54c19800 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c55055800 con 0x558c54c5bc00 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 45 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e46/46) v1 -- 0x558c58137100 con 0x558c54c5bc00 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c54d5f000 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c57ed2000 con 0x558c54c5b400 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c55141000 2019-12-07T19:34:23.603+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e46/46) v1 -- 0x558c58134e00 con 0x558c54c5b400 2019-12-07T19:34:23.603+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c54e06a00 con 0x558c54c19c00 2019-12-07T19:34:23.603+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e46/46) v1 -- 0x558c58137800 con 0x558c54c19c00 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 45 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 45 -> 46 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 45 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:23.603+0000 7f8f06986700 15 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] publish_stats_to_osd 45:142 2019-12-07T19:34:23.603+0000 7f8f03980700 15 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] publish_stats_to_osd 45:127 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.603+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:23.603+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.603+0000 7f8f03980700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] activate_map 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] state: Active: handling ActMap 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] Active: kicking snap trim 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.603+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:23.603+0000 7f8f06986700 7 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] state: handle ActMap primary 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.603+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:23.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.603+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c575ef000 2019-12-07T19:34:23.603+0000 7f8f06986700 15 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] publish_stats_to_osd 45:143 2019-12-07T19:34:23.603+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c5500d000 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.604+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 45 -> 46 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.604+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 45 -> 46 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:23.604+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 45 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c54ea1b00 seq 41 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.603+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] handle_activate_map: 46 2019-12-07T19:34:23.604+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.604+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:23.604+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] activate_map 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.604+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] state: Active: handling ActMap 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.604+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] Active: kicking snap trim 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea1b00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.604+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea1b00 onodes shared_blobs 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:23.604+0000 7f8f03980700 7 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] state: handle ActMap primary 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 prepare 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea1b00 2019-12-07T19:34:23.604+0000 7f8f03980700 20 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 io_done 2019-12-07T19:34:23.604+0000 7f8f0898a700 20 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.604+0000 7f8f03980700 15 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] publish_stats_to_osd 45:128 2019-12-07T19:34:23.604+0000 7f8f0898a700 20 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.604+0000 7f8f0898a700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.604+0000 7f8f03980700 20 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c548ab000 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.604+0000 7f8f0898a700 20 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.604+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 NullEvt) prio 255 cost 10 e46) pg 0x558c5513e000 2019-12-07T19:34:23.604+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.604+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.604+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c54ddfb00 seq 22 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 45 -> 46 2019-12-07T19:34:23.604+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 45 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:23.604+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:23.604+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map: 46 2019-12-07T19:34:23.604+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 45 -> 46 2019-12-07T19:34:23.604+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[a~1,c~2,10~2]] state: Active advmap 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 45 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] on_active_advmap new purged_snaps [a~1,d~1,10~1], now [c~1,11~1] 2019-12-07T19:34:23.604+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddfb00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.604+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddfb00 onodes shared_blobs 2019-12-07T19:34:23.604+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddfb00 allocated 0x[] released 0x[] 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] share_pg_info 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] state: Started advmap 2019-12-07T19:34:23.604+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 prepare 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.604+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddfb00 2019-12-07T19:34:23.604+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 io_done 2019-12-07T19:34:23.604+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:23.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] plpg_on_pool_change 2019-12-07T19:34:23.604+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c57ed5800 con 0x558c54c64000 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] hit_set_clear 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] agent_stop 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] handle_activate_map: 46 2019-12-07T19:34:23.604+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c57ed5100 con 0x558c54c19800 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] activate_map 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c58124300 con 0x558c54c5bc00 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.604+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:23.604+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c57a53c00 con 0x558c54c5b400 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.604+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 -- 0x558c57a55100 con 0x558c54c19c00 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.604+0000 7f8f06986700 15 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] publish_stats_to_osd 45:113 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2483fb~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2483fb 0x2483fb~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x2183fb 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3fb 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcee and padding block with 0x312 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 46 advance_pg new pool opts old pool opts 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] handle_activate_map: 46 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] activate_map 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] state: Active: handling ActMap 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] Active: kicking snap trim 2019-12-07T19:34:23.604+0000 7f8f06986700 7 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] state: handle ActMap primary 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:23.604+0000 7f8f06986700 15 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] publish_stats_to_osd 45:114 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 45 while current is 46 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x248cee 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.604+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000149233s (0.000000090s flush + 0.000149143s kv commit) 2019-12-07T19:34:23.604+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] do_peering_event: epoch_sent: 46 epoch_requested: 46 NullEvt 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.604+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea1b00,0x558c54ddfb00> 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 kv_submitted 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea1b00 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c54ddef00 seq 37 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea1b00 finishing 2019-12-07T19:34:23.604+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea1b00 onodes 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea1b00 done 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea1b00 [] 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddef00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddef00 onodes shared_blobs 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 kv_submitted 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddef00 allocated 0x[] released 0x[] 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddfb00 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddfb00 finishing 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddfb00 onodes 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddfb00 done 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 prepare 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:23.604+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddef00 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddfb00 [] 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.604+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 io_done 2019-12-07T19:34:23.604+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x248cee~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x248cee 0x248cee~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x218cee 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcee 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x173 and padding block with 0xe8d 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x249173 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.604+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119348s (0.000000084s flush + 0.000119264s kv commit) 2019-12-07T19:34:23.604+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddef00> 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 kv_submitted 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddef00 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddef00 finishing 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddef00 onodes 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddef00 done 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddef00 [] 2019-12-07T19:34:23.604+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.604+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.604+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:23.604+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:23.606+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 141 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57ed5100 con 0x558c54c19800 2019-12-07T19:34:23.607+0000 7f8f24c58700 15 osd.7 46 enqueue_peering_evt 1.ds2 epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:23.607+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e46) pg 0x558c55141000 2019-12-07T19:34:23.607+0000 7f8f06185700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:23.607+0000 7f8f06185700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.607+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:23.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:23.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5815e900 seq 42 2019-12-07T19:34:23.607+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:23.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:23.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:23.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:23.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:23.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815e900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815e900 onodes shared_blobs 2019-12-07T19:34:23.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815e900 allocated 0x[] released 0x[] 2019-12-07T19:34:23.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e900 prepare 2019-12-07T19:34:23.607+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815e900 2019-12-07T19:34:23.607+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e900 io_done 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x249173~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x249173 0x249173~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x219173 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x173 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5f8 and padding block with 0xa08 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2495f8 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122313s (0.000000074s flush + 0.000122239s kv commit) 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5815e900> 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e900 kv_submitted 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815e900 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815e900 finishing 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815e900 onodes 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815e900 done 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815e900 [] 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:23.607+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 142 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57ed5100 con 0x558c54c19800 2019-12-07T19:34:23.607+0000 7f8f24c58700 15 osd.7 46 enqueue_peering_evt 1.5s4 epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:23.607+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e46) pg 0x558c57522000 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c57a56c00 seq 15 2019-12-07T19:34:23.607+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a56c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a56c00 onodes shared_blobs 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a56c00 allocated 0x[] released 0x[] 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56c00 prepare 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a56c00 2019-12-07T19:34:23.607+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56c00 io_done 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2495f8~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2495f8 0x2495f8~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x2195f8 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5f8 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa7d and padding block with 0x583 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x249a7d 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.607+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119270s (0.000000085s flush + 0.000119185s kv commit) 2019-12-07T19:34:23.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a56c00> 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56c00 kv_submitted 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a56c00 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56c00 finishing 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a56c00 onodes 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a56c00 done 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a56c00 [] 2019-12-07T19:34:23.607+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.607+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:23.607+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:23.607+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.608+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 203 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c58124300 con 0x558c54c5bc00 2019-12-07T19:34:23.608+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.1s1 epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:23.608+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.608+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 204 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c58124300 con 0x558c54c5bc00 2019-12-07T19:34:23.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e46) pg 0x558c5736a000 2019-12-07T19:34:23.608+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.fs5 epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:23.608+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.608+0000 7f8f08189700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:23.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.608+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 205 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c58124300 con 0x558c54c5bc00 2019-12-07T19:34:23.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.608+0000 7f8f08189700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.608+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.6s2 epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:23.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e46) pg 0x558c57383000 2019-12-07T19:34:23.608+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.608+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:23.608+0000 7f8f05183700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:23.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.608+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:23.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.608+0000 7f8f05183700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.608+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c55123800 seq 16 2019-12-07T19:34:23.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.608+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:23.608+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:23.608+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:23.608+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 3(0) info: 1.6s0( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:23.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:23.608+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:23.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c574fe600 seq 18 2019-12-07T19:34:23.608+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:23.608+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:23.608+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:23.608+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.608+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55123800 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.608+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55123800 onodes shared_blobs 2019-12-07T19:34:23.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:23.608+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55123800 allocated 0x[] released 0x[] 2019-12-07T19:34:23.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:23.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:23.608+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:23.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:23.608+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123800 prepare 2019-12-07T19:34:23.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c574fe600 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.608+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55123800 2019-12-07T19:34:23.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c574fe600 onodes shared_blobs 2019-12-07T19:34:23.608+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123800 io_done 2019-12-07T19:34:23.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c574fe600 allocated 0x[] released 0x[] 2019-12-07T19:34:23.608+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:23.608+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55114c00 seq 83 2019-12-07T19:34:23.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 prepare 2019-12-07T19:34:23.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.608+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c574fe600 2019-12-07T19:34:23.608+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:23.608+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 io_done 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.608+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:23.608+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:23.608+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:23.608+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.608+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:23.608+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55114c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.608+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55114c00 onodes shared_blobs 2019-12-07T19:34:23.608+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55114c00 allocated 0x[] released 0x[] 2019-12-07T19:34:23.608+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114c00 prepare 2019-12-07T19:34:23.608+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55114c00 2019-12-07T19:34:23.608+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114c00 io_done 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:23.608+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x249a7d~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x249a7d 0x249a7d~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x219a7d 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa7d 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf02 and padding block with 0xfe 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x249f02 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000120391s (0.000000109s flush + 0.000120282s kv commit) 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55123800> 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123800 kv_submitted 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55123800 2019-12-07T19:34:23.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123800 finishing 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55123800 onodes 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55123800 done 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:23.608+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55123800 [] 2019-12-07T19:34:23.608+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.608+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:23.608+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.608+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x249f02~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x249f02 0x249f02~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x219f02 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf02 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.608+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7f5 and padding block with 0x80b 2019-12-07T19:34:23.609+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24a7f5 2019-12-07T19:34:23.609+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.609+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.609+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000118599s (0.000000070s flush + 0.000118529s kv commit) 2019-12-07T19:34:23.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c574fe600,0x558c55114c00> 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 kv_submitted 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c574fe600 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c574fe600 finishing 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c574fe600 onodes 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c574fe600 done 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:23.609+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c574fe600 [] 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114c00 kv_submitted 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55114c00 2019-12-07T19:34:23.609+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114c00 finishing 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55114c00 onodes 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114c00 done 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55114c00 [] 2019-12-07T19:34:23.609+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.609+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.609+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.616+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 206 ==== osd_map(46..46 src has 1..46) v4 ==== 693+0+0 (crc 0 0 0) 0x558c573246c0 con 0x558c54c5bc00 2019-12-07T19:34:23.616+0000 7f8f1199c700 20 osd.7 46 OSD::ms_dispatch: osd_map(46..46 src has 1..46) v4 2019-12-07T19:34:23.616+0000 7f8f1199c700 10 osd.7 46 do_waiters -- start 2019-12-07T19:34:23.616+0000 7f8f1199c700 10 osd.7 46 do_waiters -- finish 2019-12-07T19:34:23.616+0000 7f8f1199c700 20 osd.7 46 _dispatch 0x558c573246c0 osd_map(46..46 src has 1..46) v4 2019-12-07T19:34:23.616+0000 7f8f1199c700 3 osd.7 46 handle_osd_map epochs [46,46], i have 46, src has [1,46] 2019-12-07T19:34:23.616+0000 7f8f1199c700 10 osd.7 46 no new maps here, dropping 2019-12-07T19:34:23.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 207 ==== MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=129, reqid=client.4540.0:373, at_version=46'52, trim_to=0'0, roll_forward_to=45'50)) v2 ==== 150802+0+0 (crc 0 0 0) 0x558c58157800 con 0x558c54c5bc00 2019-12-07T19:34:23.616+0000 7f8f23c56700 15 osd.7 46 enqueue_op 0x558c57f3e2c0 prio 127 cost 0 latency 0.000032 epoch 46 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=129, reqid=client.4540.0:373, at_version=46'52, trim_to=0'0, roll_forward_to=45'50)) v2 2019-12-07T19:34:23.616+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=129, reqid=client.4540.0:373, at_version=46'52, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e46) 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=129, reqid=client.4540.0:373, at_version=46'52, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e46) queued 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=129, reqid=client.4540.0:373, at_version=46'52, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e2c0 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=129, reqid=client.4540.0:373, at_version=46'52, trim_to=0'0, roll_forward_to=45'50)) v2 pg pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 46 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 45 -> 46 (as per caller) 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] _handle_message: MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=129, reqid=client.4540.0:373, at_version=46'52, trim_to=0'0, roll_forward_to=45'50)) v2 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.add_oid 1:6336c6f2:::smithi13140796-16:17 e,13,15,17 2019-12-07T19:34:23.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6336c6f2:::smithi13140796-16:17 got.empty() 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6336c6f2:::smithi13140796-16:17 e,13,15,17 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.616+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000017_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 45'50 (0'0,45'50] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] append_log log((0'0,45'50], crt=45'50) [46'51 (21'6) clone 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:33:58.643819+0000 0 snaps [17,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,46'52 (21'6) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:373 2019-12-07T19:34:23.614921+0000 0 ObjectCleanRegions clean_offsets: [0~262144,851968~18446744073708699647], clean_omap: 1, new_object: 0] 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'51 (0'0,46'51] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'50 crt=45'50 lcod 45'50 active mbc={} ps=[c~1,11~1]] add_log_entry 46'51 (21'6) clone 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:33:58.643819+0000 0 snaps [17,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'52 (0'0,46'52] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'50 crt=45'50 lcod 45'50 active mbc={} ps=[c~1,11~1]] add_log_entry 46'52 (21'6) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:373 2019-12-07T19:34:23.614921+0000 0 ObjectCleanRegions clean_offsets: [0~262144,851968~18446744073708699647], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'52 (0'0,46'52] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'50 crt=45'50 lcod 45'50 active mbc={} ps=[c~1,11~1]] append_log approx pg log length = 52 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'52 (0'0,46'52] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'50 crt=45'50 lcod 45'50 active mbc={} ps=[c~1,11~1]] append_log transaction_applied = 1 2019-12-07T19:34:23.616+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:23.616+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 46'51, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:23.617+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1214172 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c54ea0600 seq 84 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6336c6f2:::smithi13140796-16:17# key 0x8280000000000000016336c6f221736d'ithi13140796-16!='0x0000000000000017ffffffffffffffff'o' 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# -> 2#1:6336c6f2:::smithi13140796-16:17# 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1230 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.onode(0x558c54ce3900).flush flush done 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 0x0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# -> 2#1:6336c6f2:::smithi13140796-16:17# 0x0~10000 -> 0x0~10000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10558 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3c8c0 blob([0x63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3c8c0 blob([0x63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c57654f50 blob([0x63000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c57654f50) get_ref 0x0~2000 Blob(0x558c57654f50 blob([0x63000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c57654f50) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57654f50 blob([0x63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10559 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3cee0 blob([0x65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3cee0 blob([0x65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c55060690 blob([0x65000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c55060690) get_ref 0x0~2000 Blob(0x558c55060690 blob([0x65000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c55060690) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c55060690 blob([0x65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10560 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54d3cb60 blob([0x67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54d3cb60 blob([0x67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c550b2700 blob([0x67000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c550b2700) get_ref 0x0~2000 Blob(0x558c550b2700 blob([0x67000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c550b2700) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c550b2700 blob([0x67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10561 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54cf8e00 blob([0x69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54cf8e00 blob([0x69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c55060770 blob([0x69000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c55060770) get_ref 0x0~2000 Blob(0x558c55060770 blob([0x69000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c55060770) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c55060770 blob([0x69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10562 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54cf8e70 blob([0x6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54cf8e70 blob([0x6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c550607e0 blob([0x6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c550607e0) get_ref 0x0~2000 Blob(0x558c550607e0 blob([0x6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c550607e0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c550607e0 blob([0x6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10563 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54cf8f50 blob([0x6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54cf8f50 blob([0x6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c55278a10 blob([0x6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c55278a10) get_ref 0x0~2000 Blob(0x558c55278a10 blob([0x6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c55278a10) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c55278a10 blob([0x6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10564 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54cf9030 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54cf9030 blob([0x6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c550605b0 blob([0x6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c550605b0) get_ref 0x0~2000 Blob(0x558c550605b0 blob([0x6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c550605b0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c550605b0 blob([0x6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10565 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54cf9110 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54cf9110 blob([0x71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=1)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup new Blob(0x558c580a2e00 blob([0x71000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a2e00) get_ref 0x0~2000 Blob(0x558c580a2e00 blob([0x71000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a2e00) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c580a2e00 blob([0x71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=2)))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dirty_range mark inline shard dirty 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e608) dirty_range mark inline shard dirty 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# -> 2#1:6336c6f2:::smithi13140796-16:17# = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# snapset 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# snapset = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x10000~24000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6336c6f2:::smithi13140796-16:head# 0x10000~24000 - have 0x10000 (65536) bytes fadvise_flags 0x0 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x10000~24000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c54ea0600 18 blobs 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x24000/1000,24000,0 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x9ee000~24000/1000,24000,0 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x9ee000~24000] 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a2e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a2ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a2e70 blob([0x9ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2ee0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a2e70) get_ref 0x0~2000 Blob(0x558c580a2e70 blob([0x9ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2ee0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a2e70) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c580a2e70 blob([0x9ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2ee0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a2ef8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a2f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a2fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a2f50 blob([0x9f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2fc0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a2f50) get_ref 0x0~2000 Blob(0x558c580a2f50 blob([0x9f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2fc0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a2f50) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c580a2f50 blob([0x9f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2fc0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a2fd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a30a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3030 blob([0x9f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a30a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3030) get_ref 0x0~2000 Blob(0x558c580a3030 blob([0x9f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a30a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3030) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c580a3030 blob([0x9f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a30a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a30b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3110 blob([0x9f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3180 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3110) get_ref 0x0~2000 Blob(0x558c580a3110 blob([0x9f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3180 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3110) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c580a3110 blob([0x9f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3180 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a31f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a31f0 blob([0x9f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3260 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a31f0) get_ref 0x0~2000 Blob(0x558c580a31f0 blob([0x9f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3260 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a31f0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c580a31f0 blob([0x9f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3260 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3278 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a32d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a32d0 blob([0x9f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3340 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a32d0) get_ref 0x0~2000 Blob(0x558c580a32d0 blob([0x9f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3340 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a32d0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c580a32d0 blob([0x9f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3340 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3358 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a33b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a33b0 blob([0x9fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3420 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a33b0) get_ref 0x0~2000 Blob(0x558c580a33b0 blob([0x9fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3420 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a33b0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c580a33b0 blob([0x9fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3420 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3438 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3490 blob([0x9fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3500 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3490) get_ref 0x0~2000 Blob(0x558c580a3490 blob([0x9fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3500 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3490) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c580a3490 blob([0x9fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3500 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3518 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a35e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3570 blob([0x9fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a35e0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3570) get_ref 0x0~2000 Blob(0x558c580a3570 blob([0x9fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a35e0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3570) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c580a3570 blob([0x9fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a35e0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a35f8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a36c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3650 blob([0xa00000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a36c0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3650) get_ref 0x0~2000 Blob(0x558c580a3650 blob([0xa00000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a36c0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3650) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c580a3650 blob([0xa00000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a36c0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a36d8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a37a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3730 blob([0xa02000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a37a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3730) get_ref 0x0~2000 Blob(0x558c580a3730 blob([0xa02000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a37a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3730) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c580a3730 blob([0xa02000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a37a0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a37b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3810 blob([0xa04000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3880 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3810) get_ref 0x0~2000 Blob(0x558c580a3810 blob([0xa04000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3880 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3810) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c580a3810 blob([0xa04000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3880 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3898 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a38f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a38f0 blob([0xa06000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3960 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a38f0) get_ref 0x0~2000 Blob(0x558c580a38f0 blob([0xa06000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3960 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a38f0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c580a38f0 blob([0xa06000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3960 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3978 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a39d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a39d0 blob([0xa08000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3a40 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a39d0) get_ref 0x0~2000 Blob(0x558c580a39d0 blob([0xa08000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3a40 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c580a39d0 blob([0xa08000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3a40 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3a58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a48c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bead20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a48c0 blob([0xa0a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bead20 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a48c0) get_ref 0x0~2000 Blob(0x558c580a48c0 blob([0xa0a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bead20 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a48c0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c580a48c0 blob([0xa0a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bead20 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c57bead38 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c573969a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c573969a0 blob([0xa0c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3ab0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c573969a0) get_ref 0x0~2000 Blob(0x558c573969a0 blob([0xa0c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3ab0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c573969a0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c573969a0 blob([0xa0c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3ab0 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3ac8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3b20 blob([0xa0e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3b90 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3b20) get_ref 0x0~2000 Blob(0x558c580a3b20 blob([0xa0e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3b90 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3b20) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c580a3b20 blob([0xa0e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3b90 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3ba8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a3c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a3c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a3c00 blob([0xa10000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3c70 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3c00) get_ref 0x0~2000 Blob(0x558c580a3c00 blob([0xa10000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a3c70 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.blob(0x558c580a3c00) get_ref init 0x2000, 1000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c580a3c00 blob([0xa10000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a3c70 sbid 0x0)) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c580a3c88 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x34000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dirty_range mark inline shard dirty 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x10000~24000 = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_0000000000000013_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_0000000000000015_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000017_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_0000000000000017_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000046.00000000000000000051' <- 0000000046.00000000000000000051 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000046.00000000000000000052' <- 0000000046.00000000000000000052 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0600 cost 228311 (19 ios * 4000 + 152311 bytes) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0600 onodes 0x558c54ce3900,0x558c57f8e500 shared_blobs 0x558c54cf8af0,0x558c54cf8d90,0x558c54cf8ee0,0x558c54cf8fc0,0x558c54cf90a0,0x558c54cf9180,0x558c54d3cc40,0x558c54d75ea0 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) update inline shard 561 bytes from 26 extents 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1148 (581 bytes onode + 2 bytes spanning blobs + 565 bytes inline extents) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e608) update 2#1:6336c6f2:::smithi13140796-16:17# 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e608) update inline shard 219 bytes from 8 extents 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:17# is 637 (412 bytes onode + 2 bytes spanning blobs + 223 bytes inline extents) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2941 is 11 SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2940 is 11 SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2942 is 11 SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2943 is 11 SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2944 is 11 SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2945 is 11 SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293f is 11 SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293e is 11 SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=2))) 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0600 allocated 0x[9ee000~24000] released 0x[] 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 freelist allocate 0x9ee000~24000 2019-12-07T19:34:23.617+0000 7f8f0197c700 20 freelist _xor first_key 0x980000 last_key 0xa00000 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 prepare 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c54ea0600 2019-12-07T19:34:23.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'52 (0'0,46'52] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=45'50 crt=45'50 lcod 45'50 active mbc={} ps=[c~1,11~1]] op_applied version 46'52 2019-12-07T19:34:23.618+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e2c0 finish 2019-12-07T19:34:23.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 aio_wait 2019-12-07T19:34:23.618+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0600 2019-12-07T19:34:23.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 io_done 2019-12-07T19:34:23.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 238 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c5b400 2019-12-07T19:34:23.619+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.bs1 epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:23.619+0000 7f8f24457700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.619+0000 7f8f24457700 0 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 msgr2=0x558c54c6fb80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send injecting socket failure 2019-12-07T19:34:23.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 msgr2=0x558c54c6fb80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send send error: (32) Broken pipe 2019-12-07T19:34:23.619+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 crc :-1 s=READY pgs=9 cs=1 l=0 rx=0 tx=0).write_event send msg failed 2019-12-07T19:34:23.619+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 crc :-1 s=READY pgs=9 cs=1 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:23.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e46) pg 0x558c575ef000 2019-12-07T19:34:23.619+0000 7f8f07187700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:23.619+0000 7f8f07187700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:23.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:23.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c583c9500 seq 34 2019-12-07T19:34:23.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:23.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:23.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:23.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:23.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:23.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c9500 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c9500 onodes shared_blobs 2019-12-07T19:34:23.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c9500 allocated 0x[] released 0x[] 2019-12-07T19:34:23.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9500 prepare 2019-12-07T19:34:23.619+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c9500 2019-12-07T19:34:23.619+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9500 io_done 2019-12-07T19:34:23.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5117 < min_flush_size 524288 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e500 had 1 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6251 < min_flush_size 524288 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6274 < min_flush_size 524288 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6274 < min_flush_size 524288 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24a7f5~1882 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24a7f5 0x24a7f5~1882 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x21a7f5 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7f5 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x77 and padding block with 0xf89 2019-12-07T19:34:23.619+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c551b6400 0x558c53aa5b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24c077 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.619+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c551b6400 0x558c53aa5b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000179314s (0.000044615s flush + 0.000134699s kv commit) 2019-12-07T19:34:23.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ea0600,0x558c583c9500> 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 kv_submitted 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0600 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0600 finishing 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0600 onodes 0x558c54ce3900,0x558c57f8e500 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bead38 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f0e0 space 0x558c57bead38 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a2ef8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9b290 space 0x558c580a2ef8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a2fd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fbdd0 space 0x558c580a2fd8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a30b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fbe60 space 0x558c580a30b8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3198 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f76990 space 0x558c580a3198 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3278 in 0x558c53a58fc0) _finish_write discard buffer(0x558c551a99e0 space 0x558c580a3278 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 46/28 ECSubWriteReply(tid=129, last_complete=46'52, committed=1, applied=1)) v2 -- 0x558c576b5000 con 0x558c54c5bc00 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3358 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4e2d0 space 0x558c580a3358 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3438 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4e240 space 0x558c580a3438 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3518 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4e1b0 space 0x558c580a3518 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a35f8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4e120 space 0x558c580a35f8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a36d8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4e090 space 0x558c580a36d8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a37b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4e000 space 0x558c580a37b8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3898 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8ef30 space 0x558c580a3898 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3978 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8efc0 space 0x558c580a3978 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3a58 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f050 space 0x558c580a3a58 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3ac8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f170 space 0x558c580a3ac8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3ba8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f200 space 0x558c580a3ba8 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a3c88 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f290 space 0x558c580a3c88 0x0~2000 writing nocache) 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0600 done 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0600 [] 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9500 kv_submitted 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c9500 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9500 finishing 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c9500 onodes 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c9500 done 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c9500 [] 2019-12-07T19:34:23.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:23.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:23.619+0000 7f8f23c56700 10 osd.7 46 ms_handle_authentication new session 0x558c57405ba0 con 0x558c551b6400 entity osd.4 addr 2019-12-07T19:34:23.619+0000 7f8f23c56700 10 osd.7 46 ms_handle_authentication session 0x558c57405ba0 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:23.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 354 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:23.619+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.7s4 epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:23.619+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 355 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:23.619+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.9s4 epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:23.619+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 356 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:23.619+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.as5 epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:23.619+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 357 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:23.619+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.es4 epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:23.619+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e46) pg 0x558c54d98000 2019-12-07T19:34:23.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 358 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:23.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e46) pg 0x558c57e8b000 2019-12-07T19:34:23.620+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.8s5 epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:23.620+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.620+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:23.620+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:23.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c57a56300 seq 14 2019-12-07T19:34:23.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e46) pg 0x558c5500a000 2019-12-07T19:34:23.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:23.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:23.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:23.620+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:23.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54ea0000 seq 13 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a56300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a56300 onodes shared_blobs 2019-12-07T19:34:23.620+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:23.620+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a56300 allocated 0x[] released 0x[] 2019-12-07T19:34:23.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56300 prepare 2019-12-07T19:34:23.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a56300 2019-12-07T19:34:23.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:23.620+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56300 io_done 2019-12-07T19:34:23.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:23.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:23.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ea0000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ea0000 onodes shared_blobs 2019-12-07T19:34:23.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:23.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ea0000 allocated 0x[] released 0x[] 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5511c300 seq 66 2019-12-07T19:34:23.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 prepare 2019-12-07T19:34:23.620+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:23.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ea0000 2019-12-07T19:34:23.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 io_done 2019-12-07T19:34:23.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:23.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:23.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:23.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511c300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511c300 onodes shared_blobs 2019-12-07T19:34:23.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511c300 allocated 0x[] released 0x[] 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c300 prepare 2019-12-07T19:34:23.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511c300 2019-12-07T19:34:23.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c300 io_done 2019-12-07T19:34:23.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24c077~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24c077 0x24c077~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x21c077 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x77 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x96a and padding block with 0x696 2019-12-07T19:34:23.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 359 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e46/46) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:23.620+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.0s3 epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:23.620+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.620+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c551b6400 0x558c53aa5b80 crc :-1 s=SESSION_ACCEPTING pgs=26 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558c54c5b400 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e46) pg 0x558c57380000 2019-12-07T19:34:23.620+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c551b6400 0x558c53aa5b80 crc :-1 s=SESSION_ACCEPTING pgs=26 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:34:23.620+0000 7f8f04982700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:23.620+0000 7f8f04982700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c57ef8900 seq 32 2019-12-07T19:34:23.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:23.620+0000 7f8f1199c700 2 osd.7 46 ms_handle_reset con 0x558c551b6400 session 0x558c57405ba0 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24c96a 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8900 onodes shared_blobs 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8900 allocated 0x[] released 0x[] 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 prepare 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8900 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 io_done 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000172986s (0.000000119s flush + 0.000172867s kv commit) 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e46) pg 0x558c5500d000 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:23.620+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=READY pgs=26 cs=2 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=f8982639b78e8be6 server_cookie=84aece1fce04073f in_seq=238 out_seq=242 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f04982700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:23.620+0000 7f8f04982700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:23.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c5730a000 seq 39 2019-12-07T19:34:23.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5730a000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5730a000 onodes shared_blobs 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5730a000 allocated 0x[] released 0x[] 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24c96a~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 prepare 2019-12-07T19:34:23.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5730a000 2019-12-07T19:34:23.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 io_done 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24c96a 0x24c96a~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x21c96a 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x96a 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x25d and padding block with 0xda3 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a56300,0x558c54ea0000> 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56300 kv_submitted 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a56300 2019-12-07T19:34:23.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 239 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e46/46) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c58124300 con 0x558c54c5b400 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56300 finishing 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a56300 onodes 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a56300 done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a56300 [] 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 kv_submitted 2019-12-07T19:34:23.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ea0000 2019-12-07T19:34:23.620+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.3s4 epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ea0000 finishing 2019-12-07T19:34:23.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ea0000 onodes 2019-12-07T19:34:23.620+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e46) 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ea0000 done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ea0000 [] 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e46) pg 0x558c5513e000 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c57d0b500 seq 56 2019-12-07T19:34:23.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0b500 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0b500 onodes shared_blobs 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0b500 allocated 0x[] released 0x[] 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b500 prepare 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0b500 2019-12-07T19:34:23.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b500 io_done 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24d25d 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000238427s (0.000000093s flush + 0.000238334s kv commit) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5511c300,0x558c57ef8900> 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c300 kv_submitted 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511c300 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c300 finishing 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511c300 onodes 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511c300 done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511c300 [] 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.620+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 kv_submitted 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8900 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8900 finishing 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8900 onodes 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8900 done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8900 [] 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24d25d~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24d25d 0x24d25d~8f3 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x21d25d 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x25d 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb50 and padding block with 0x4b0 2019-12-07T19:34:23.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e46) queued 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e46) pg 0x558c54d5f000 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] do_peering_event: epoch_sent: 46 epoch_requested: 46 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24db50 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[a~1,c~2,10~2]] proc_primary_info updating purged_snaps to [c~1,11~1] 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000148932s (0.000000079s flush + 0.000148853s kv commit) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5730a000,0x558c57d0b500> 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 kv_submitted 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5730a000 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5730a000 finishing 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5730a000 onodes 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5730a000 done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5730a000 [] 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b500 kv_submitted 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0b500 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b500 finishing 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0b500 onodes 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0b500 done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0b500 [] 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b4000 seq 64 2019-12-07T19:34:23.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:23.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b4000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b4000 onodes shared_blobs 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b4000 allocated 0x[] released 0x[] 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4000 prepare 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b4000 2019-12-07T19:34:23.620+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4000 io_done 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24db50~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24db50 0x24db50~485 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x21db50 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb50 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfd5 and padding block with 0x2b 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24dfd5 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000109543s (0.000000070s flush + 0.000109473s kv commit) 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586b4000> 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4000 kv_submitted 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b4000 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4000 finishing 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b4000 onodes 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b4000 done 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b4000 [] 2019-12-07T19:34:23.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:23.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:23.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 208 ==== MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=130, reqid=client.4540.0:374, at_version=46'53, trim_to=0'0, roll_forward_to=45'50)) v2 ==== 145880+0+0 (crc 0 0 0) 0x558c57dff000 con 0x558c54c5bc00 2019-12-07T19:34:23.621+0000 7f8f23c56700 15 osd.7 46 enqueue_op 0x558c57f3f760 prio 127 cost 0 latency 0.000022 epoch 46 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=130, reqid=client.4540.0:374, at_version=46'53, trim_to=0'0, roll_forward_to=45'50)) v2 2019-12-07T19:34:23.621+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=130, reqid=client.4540.0:374, at_version=46'53, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e46) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=130, reqid=client.4540.0:374, at_version=46'53, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e46) queued 2019-12-07T19:34:23.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=130, reqid=client.4540.0:374, at_version=46'53, trim_to=0'0, roll_forward_to=45'50)) v2) prio 127 cost 0 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.621+0000 7f8f05984700 10 osd.7 46 dequeue_op 0x558c57f3f760 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=130, reqid=client.4540.0:374, at_version=46'53, trim_to=0'0, roll_forward_to=45'50)) v2 pg pg[1.6s2( v 46'52 (0'0,46'52] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] 2019-12-07T19:34:23.621+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'52 (0'0,46'52] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] _handle_message: MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=130, reqid=client.4540.0:374, at_version=46'53, trim_to=0'0, roll_forward_to=45'50)) v2 2019-12-07T19:34:23.621+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'52 (0'0,46'52] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] append_log log((0'0,46'52], crt=45'50) [46'53 (46'52) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:374 2019-12-07T19:34:23.618359+0000 0 ObjectCleanRegions clean_offsets: [0~851968,1425408~18446744073708126207], clean_omap: 1, new_object: 0] 2019-12-07T19:34:23.621+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'53 (0'0,46'53] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'52 crt=45'50 lcod 46'52 active mbc={} ps=[c~1,11~1]] add_log_entry 46'53 (46'52) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:374 2019-12-07T19:34:23.618359+0000 0 ObjectCleanRegions clean_offsets: [0~851968,1425408~18446744073708126207], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.621+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'53 (0'0,46'53] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'52 crt=45'50 lcod 46'52 active mbc={} ps=[c~1,11~1]] append_log approx pg log length = 53 2019-12-07T19:34:23.621+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'53 (0'0,46'53] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'52 crt=45'50 lcod 46'52 active mbc={} ps=[c~1,11~1]] append_log transaction_applied = 1 2019-12-07T19:34:23.621+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:23.621+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 46'53, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:23.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55114f00 seq 85 2019-12-07T19:34:23.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:34:23.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:34:23.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:23.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:23.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x34000~23000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6336c6f2:::smithi13140796-16:head# 0x34000~23000 - have 0x34000 (212992) bytes fadvise_flags 0x0 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:23.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x34000~23000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c55114f00 18 blobs 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.621+0000 7f8f05984700 10 fbmap_alloc 0x558c53a72e00 allocate 0x23000/1000,23000,0 2019-12-07T19:34:23.621+0000 7f8f05984700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa12000~23000/1000,23000,0 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa12000~23000] 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b2000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b2000 blob([0xa12000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c850 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c550b2000) get_ref 0x0~2000 Blob(0x558c550b2000 blob([0xa12000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3c850 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c550b2000) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c550b2000 blob([0xa12000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c850 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c54d3c868 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a07880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff43f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a07880 blob([0xa14000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff43f0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57a07880) get_ref 0x0~2000 Blob(0x558c57a07880 blob([0xa14000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff43f0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57a07880) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c57a07880 blob([0xa14000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff43f0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57ff4408 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57654a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654cb0 blob([0xa16000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654a80 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654cb0) get_ref 0x0~2000 Blob(0x558c57654cb0 blob([0xa16000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654a80 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c57654cb0 blob([0xa16000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57654a80 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57654a98 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060700 blob([0xa18000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2150 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c55060700) get_ref 0x0~2000 Blob(0x558c55060700 blob([0xa18000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2150 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c55060700) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c55060700 blob([0xa18000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2150 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c550b2168 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809b880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55060930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809b880 blob([0xa1a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060930 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c5809b880) get_ref 0x0~2000 Blob(0x558c5809b880 blob([0xa1a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55060930 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c5809b880) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c5809b880 blob([0xa1a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55060930 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c55060948 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654af0 blob([0xa1c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b8f0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654af0) get_ref 0x0~2000 Blob(0x558c57654af0 blob([0xa1c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b8f0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654af0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c57654af0 blob([0xa1c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b8f0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5809b908 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654b60 blob([0xa1e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d490 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654b60) get_ref 0x0~2000 Blob(0x558c57654b60 blob([0xa1e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d490 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654b60) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c57654b60 blob([0xa1e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d490 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c5794d4a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57654c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654bd0 blob([0xa20000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654c40 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654bd0) get_ref 0x0~2000 Blob(0x558c57654bd0 blob([0xa20000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57654c40 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c57654bd0 blob([0xa20000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57654c40 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c57654c58 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57654d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d6e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57654d20 blob([0xa22000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6e00 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654d20) get_ref 0x0~2000 Blob(0x558c57654d20 blob([0xa22000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6e00 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c57654d20) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c57654d20 blob([0xa22000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d6e00 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d6e18 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d6e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d6ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d6e70 blob([0xa24000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6ee0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c581d6e70) get_ref 0x0~2000 Blob(0x558c581d6e70 blob([0xa24000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6ee0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c581d6e70) get_ref init 0x2000, 1000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c581d6e70 blob([0xa24000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d6ee0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d6ef8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d6f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d6fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d6f50 blob([0xa26000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6fc0 sbid 0x0)) 2019-12-07T19:34:23.621+0000 7f8f05984700 20 bluestore.blob(0x558c581d6f50) get_ref 0x0~2000 Blob(0x558c581d6f50 blob([0xa26000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6fc0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d6f50) get_ref init 0x2000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c581d6f50 blob([0xa26000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d6fc0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d6fd8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d7030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d70a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d7030 blob([0xa28000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d70a0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7030) get_ref 0x0~2000 Blob(0x558c581d7030 blob([0xa28000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d70a0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7030) get_ref init 0x2000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c581d7030 blob([0xa28000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d70a0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d70b8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d7110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d7180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d7110 blob([0xa2a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7180 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7110) get_ref 0x0~2000 Blob(0x558c581d7110 blob([0xa2a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7180 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7110) get_ref init 0x2000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c581d7110 blob([0xa2a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d7180 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d7198 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d71f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d7260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d71f0 blob([0xa2c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7260 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d71f0) get_ref 0x0~2000 Blob(0x558c581d71f0 blob([0xa2c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7260 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d71f0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c581d71f0 blob([0xa2c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d7260 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d7278 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d72d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d7340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d72d0 blob([0xa2e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7340 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d72d0) get_ref 0x0~2000 Blob(0x558c581d72d0 blob([0xa2e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7340 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d72d0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c581d72d0 blob([0xa2e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d7340 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d7358 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d73b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d7420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d73b0 blob([0xa30000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7420 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d73b0) get_ref 0x0~2000 Blob(0x558c581d73b0 blob([0xa30000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7420 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d73b0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c581d73b0 blob([0xa30000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d7420 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d7438 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d7490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d7500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d7490 blob([0xa32000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7500 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7490) get_ref 0x0~2000 Blob(0x558c581d7490 blob([0xa32000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d7500 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7490) get_ref init 0x2000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c581d7490 blob([0xa32000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d7500 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d7518 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d7570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d75e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d7570 blob([0xa34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d75e0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7570) get_ref 0x0~1000 Blob(0x558c581d7570 blob([0xa34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d75e0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.blob(0x558c581d7570) get_ref init 0x1000, 1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~1000: 0x0~1000 Blob(0x558c581d7570 blob([0xa34000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581d75e0 sbid 0x0)) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.BufferSpace(0x558c581d75f8 in 0x558c53a58fc0) _discard 0x0~1000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x57000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) dirty_range mark inline shard dirty 2019-12-07T19:34:23.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x34000~23000 = 0 2019-12-07T19:34:23.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:23.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:23.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000046.00000000000000000053' <- 0000000046.00000000000000000053 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:23.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:23.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55114f00 cost 221530 (19 ios * 4000 + 145530 bytes) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55114f00 onodes 0x558c54ce3900 shared_blobs 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) update inline shard 900 bytes from 44 extents 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1487 (581 bytes onode + 2 bytes spanning blobs + 904 bytes inline extents) 2019-12-07T19:34:23.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55114f00 allocated 0x[a12000~23000] released 0x[] 2019-12-07T19:34:23.622+0000 7f8f05984700 10 freelist allocate 0xa12000~23000 2019-12-07T19:34:23.622+0000 7f8f05984700 20 freelist _xor first_key 0xa00000 last_key 0xa00000 2019-12-07T19:34:23.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114f00 prepare 2019-12-07T19:34:23.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c55114f00 2019-12-07T19:34:23.622+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'53 (0'0,46'53] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'52 crt=45'50 lcod 46'52 active mbc={} ps=[c~1,11~1]] op_applied version 46'53 2019-12-07T19:34:23.622+0000 7f8f05984700 10 osd.7 46 dequeue_op 0x558c57f3f760 finish 2019-12-07T19:34:23.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.622+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114f00 aio_wait 2019-12-07T19:34:23.622+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55114f00 2019-12-07T19:34:23.622+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114f00 io_done 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2723 < min_flush_size 524288 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2746 < min_flush_size 524288 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2746 < min_flush_size 524288 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24dfd5~aba to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24dfd5 0x24dfd5~aba to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x21dfd5 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfd5 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa8f and padding block with 0x571 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24ea8f 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000169080s (0.000038550s flush + 0.000130530s kv commit) 2019-12-07T19:34:23.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55114f00> 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114f00 kv_submitted 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55114f00 2019-12-07T19:34:23.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55114f00 finishing 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55114f00 onodes 0x558c54ce3900 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c868 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f76240 space 0x558c54d3c868 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55060948 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54cdf9e0 space 0x558c55060948 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b2168 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4f440 space 0x558c550b2168 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57654a98 in 0x558c53a58fc0) _finish_write discard buffer(0x558c573d8750 space 0x558c57654a98 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57654c58 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57ecfe60 space 0x558c57654c58 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d4a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57ece630 space 0x558c5794d4a8 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff4408 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57f4e360 space 0x558c57ff4408 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b908 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54cdf830 space 0x558c5809b908 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d6e18 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57ecfef0 space 0x558c581d6e18 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d6ef8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8fc20 space 0x558c581d6ef8 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d6fd8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8fcb0 space 0x558c581d6fd8 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 46/28 ECSubWriteReply(tid=130, last_complete=46'53, committed=1, applied=1)) v2 -- 0x558c576b4a00 con 0x558c54c5bc00 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d70b8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8fd40 space 0x558c581d70b8 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d7198 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8fdd0 space 0x558c581d7198 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d7278 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8fe60 space 0x558c581d7278 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d7358 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8fef0 space 0x558c581d7358 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d7438 in 0x558c53a58fc0) _finish_write discard buffer(0x558c586c2000 space 0x558c581d7438 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d7518 in 0x558c53a58fc0) _finish_write discard buffer(0x558c586c2090 space 0x558c581d7518 0x0~2000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d75f8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c586c2120 space 0x558c581d75f8 0x0~1000 writing nocache) 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55114f00 done 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:23.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55114f00 [] 2019-12-07T19:34:23.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 209 ==== MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=131, reqid=client.4540.0:375, at_version=46'54, trim_to=0'0, roll_forward_to=46'52)) v2 ==== 51672+0+0 (crc 0 0 0) 0x558c551dd800 con 0x558c54c5bc00 2019-12-07T19:34:23.623+0000 7f8f23c56700 15 osd.7 46 enqueue_op 0x558c57f3e6e0 prio 127 cost 0 latency 0.000023 epoch 46 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=131, reqid=client.4540.0:375, at_version=46'54, trim_to=0'0, roll_forward_to=46'52)) v2 2019-12-07T19:34:23.623+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=131, reqid=client.4540.0:375, at_version=46'54, trim_to=0'0, roll_forward_to=46'52)) v2) prio 127 cost 0 e46) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=131, reqid=client.4540.0:375, at_version=46'54, trim_to=0'0, roll_forward_to=46'52)) v2) prio 127 cost 0 e46) queued 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=131, reqid=client.4540.0:375, at_version=46'54, trim_to=0'0, roll_forward_to=46'52)) v2) prio 127 cost 0 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e6e0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=131, reqid=client.4540.0:375, at_version=46'54, trim_to=0'0, roll_forward_to=46'52)) v2 pg pg[1.6s2( v 46'53 (0'0,46'53] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'53 (0'0,46'53] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] _handle_message: MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=131, reqid=client.4540.0:375, at_version=46'54, trim_to=0'0, roll_forward_to=46'52)) v2 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'53 (0'0,46'53] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=45'50 active mbc={} ps=[c~1,11~1]] append_log log((0'0,46'53], crt=45'50) [46'54 (46'53) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:375 2019-12-07T19:34:23.619502+0000 0 ObjectCleanRegions clean_offsets: [0~1425408,1622016~18446744073707929599], clean_omap: 1, new_object: 0] 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'53 crt=45'50 lcod 46'53 active mbc={} ps=[c~1,11~1]] add_log_entry 46'54 (46'53) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:375 2019-12-07T19:34:23.619502+0000 0 ObjectCleanRegions clean_offsets: [0~1425408,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'53 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] rollforward: entry=46'51 (21'6) clone 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:33:58.643819+0000 0 snaps [17,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'53 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] rollforward: entry=46'52 (21'6) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:373 2019-12-07T19:34:23.614921+0000 0 ObjectCleanRegions clean_offsets: [0~262144,851968~18446744073708699647], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'53 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] append_log approx pg log length = 54 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'53 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] append_log transaction_applied = 1 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:23.623+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 46'54, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57cdef00 seq 86 2019-12-07T19:34:23.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x57000~c000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:6336c6f2:::smithi13140796-16:head# 0x57000~c000 - have 0x57000 (356352) bytes fadvise_flags 0x0 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x57000~c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57cdef00 6 blobs 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate 0xc000/1000,c000,0 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa35000~c000/1000,c000,0 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa35000~c000] 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b2620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b2690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b2620 blob([0xa35000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2690 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c550b2620) get_ref 0x0~2000 Blob(0x558c550b2620 blob([0xa35000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2690 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c550b2620) get_ref init 0x2000, 1000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x57000~2000: 0x0~2000 Blob(0x558c550b2620 blob([0xa35000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2690 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c550b26a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf8150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf8150 blob([0xa37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf10 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf8150) get_ref 0x0~2000 Blob(0x558c54cf8150 blob([0xa37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf10 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf8150) get_ref init 0x2000, 1000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x59000~2000: 0x0~2000 Blob(0x558c54cf8150 blob([0xa37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf10 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c5809bf28 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d6540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d65b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d6540 blob([0xa39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d65b0 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6540) get_ref 0x0~2000 Blob(0x558c581d6540 blob([0xa39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d65b0 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6540) get_ref init 0x2000, 1000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5b000~2000: 0x0~2000 Blob(0x558c581d6540 blob([0xa39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d65b0 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c581d65c8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d6620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d6690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d6620 blob([0xa3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6690 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6620) get_ref 0x0~2000 Blob(0x558c581d6620 blob([0xa3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6690 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6620) get_ref init 0x2000, 1000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5d000~2000: 0x0~2000 Blob(0x558c581d6620 blob([0xa3b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d6690 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c581d66a8 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d6700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d6770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d6700 blob([0xa3d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6770 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6700) get_ref 0x0~2000 Blob(0x558c581d6700 blob([0xa3d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6770 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6700) get_ref init 0x2000, 1000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5f000~2000: 0x0~2000 Blob(0x558c581d6700 blob([0xa3d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d6770 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c581d6788 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c581d67e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c581d6850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c581d67e0 blob([0xa3f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6850 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d67e0) get_ref 0x0~2000 Blob(0x558c581d67e0 blob([0xa3f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c581d6850 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.blob(0x558c581d67e0) get_ref init 0x2000, 1000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x61000~2000: 0x0~2000 Blob(0x558c581d67e0 blob([0xa3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581d6850 sbid 0x0)) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.BufferSpace(0x558c581d6868 in 0x558c53a58fc0) _discard 0x0~2000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x63000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) dirty_range mark inline shard dirty 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x57000~c000 = 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000046.00000000000000000054' <- 0000000046.00000000000000000054 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cdef00 cost 79322 (7 ios * 4000 + 51322 bytes) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cdef00 onodes 0x558c54ce3900 shared_blobs 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) update inline shard 1015 bytes from 50 extents 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1602 (581 bytes onode + 2 bytes spanning blobs + 1019 bytes inline extents) 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cdef00 allocated 0x[a35000~c000] released 0x[] 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 freelist allocate 0xa35000~c000 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 freelist _xor first_key 0xa00000 last_key 0xa00000 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdef00 prepare 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57cdef00 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'53 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] op_applied version 46'54 2019-12-07T19:34:23.623+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e6e0 finish 2019-12-07T19:34:23.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 210 ==== MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=132, reqid=client.4540.0:376, at_version=46'55, trim_to=0'0, roll_forward_to=46'52)) v2 ==== 2487+0+0 (crc 0 0 0) 0x558c57e7b000 con 0x558c54c5bc00 2019-12-07T19:34:23.623+0000 7f8f23c56700 15 osd.7 46 enqueue_op 0x558c57f3f1e0 prio 127 cost 0 latency 0.000029 epoch 46 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=132, reqid=client.4540.0:376, at_version=46'55, trim_to=0'0, roll_forward_to=46'52)) v2 2019-12-07T19:34:23.623+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=132, reqid=client.4540.0:376, at_version=46'55, trim_to=0'0, roll_forward_to=46'52)) v2) prio 127 cost 0 e46) 2019-12-07T19:34:23.623+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdef00 aio_wait 2019-12-07T19:34:23.623+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cdef00 2019-12-07T19:34:23.623+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdef00 io_done 2019-12-07T19:34:23.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2838 < min_flush_size 524288 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3858432000 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998169) bluestore 86 GiB free (0.959888), bluefs_ratio 0.0399345 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2861 < min_flush_size 524288 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2861 < min_flush_size 524288 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24ea8f~b2d to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.624+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24ea8f 0x24ea8f~b2d to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.624+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=132, reqid=client.4540.0:376, at_version=46'55, trim_to=0'0, roll_forward_to=46'52)) v2) prio 127 cost 0 e46) queued 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~220000 x_off 0x21ea8f 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa8f 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.624+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5bc and padding block with 0xa44 2019-12-07T19:34:23.624+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=132, reqid=client.4540.0:376, at_version=46'55, trim_to=0'0, roll_forward_to=46'52)) v2) prio 127 cost 0 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 46 dequeue_op 0x558c57f3f1e0 prio 127 cost 0 latency 0.000193 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=132, reqid=client.4540.0:376, at_version=46'55, trim_to=0'0, roll_forward_to=46'52)) v2 pg pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] _handle_message: MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=132, reqid=client.4540.0:376, at_version=46'55, trim_to=0'0, roll_forward_to=46'52)) v2 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x24f5bc 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'54 (0'0,46'54] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] append_log log((0'0,46'54], crt=46'52) [46'55 (46'54) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:376 2019-12-07T19:34:23.619518+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000169133s (0.000041482s flush + 0.000127651s kv commit) 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57cdef00> 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdef00 kv_submitted 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'54 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] add_log_entry 46'55 (46'54) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:376 2019-12-07T19:34:23.619518+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cdef00 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdef00 finishing 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cdef00 onodes 0x558c54ce3900 2019-12-07T19:34:23.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'54 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] append_log approx pg log length = 55 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b26a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c579fbcb0 space 0x558c550b26a8 0x0~2000 writing nocache) 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809bf28 in 0x558c53a58fc0) _finish_write discard buffer(0x558c54c9a240 space 0x558c5809bf28 0x0~2000 writing nocache) 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d65c8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f3b0 space 0x558c581d65c8 0x0~2000 writing nocache) 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d66a8 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f440 space 0x558c581d66a8 0x0~2000 writing nocache) 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d6788 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f4d0 space 0x558c581d6788 0x0~2000 writing nocache) 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c581d6868 in 0x558c53a58fc0) _finish_write discard buffer(0x558c57b8f560 space 0x558c581d6868 0x0~2000 writing nocache) 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'54 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] append_log transaction_applied = 1 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cdef00 done 2019-12-07T19:34:23.624+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cdef00 [] 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.624+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 46'55, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:23.624+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55115200 seq 87 2019-12-07T19:34:23.624+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 3 keys 2019-12-07T19:34:23.624+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 3 keys = 0 2019-12-07T19:34:23.624+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:23.624+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:23.624+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000046.00000000000000000055' <- 0000000046.00000000000000000055 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:23.624+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:23.624+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55115200 cost 6137 (1 ios * 4000 + 2137 bytes) 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55115200 onodes 0x558c54ce3900 shared_blobs 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1603 (582 bytes onode + 2 bytes spanning blobs + 1019 bytes inline extents) 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55115200 allocated 0x[] released 0x[] 2019-12-07T19:34:23.624+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115200 prepare 2019-12-07T19:34:23.624+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55115200 2019-12-07T19:34:23.624+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115200 io_done 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'54 crt=46'52 lcod 46'53 active mbc={} ps=[c~1,11~1]] op_applied version 46'55 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.624+0000 7f8f05984700 10 osd.7 46 dequeue_op 0x558c57f3f1e0 finish 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2628 < min_flush_size 524288 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2810 < min_flush_size 524288 2019-12-07T19:34:23.624+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 46/28 ECSubWriteReply(tid=131, last_complete=46'54, committed=1, applied=1)) v2 -- 0x558c576b5400 con 0x558c54c5bc00 2019-12-07T19:34:23.624+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2833 < min_flush_size 524288 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2833 < min_flush_size 524288 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x24f5bc~b11 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x24f5bc 0x24f5bc~b11 to file(ino 16 size 0x250000 mtime 2019-12-07T19:34:22.640938+0000 allocated 250000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~220000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _allocate len 0xcd from 0 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _allocate len 0xcd from 1 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd380000 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd380000~10000/10000,10000,acd380000 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x260000 to include allocated 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 54 (was clean) 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x21f5bc 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5bc 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcd and padding block with 0xf33 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x260000 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2500cd 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (54) on file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]), flushing log 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 54 len 0x4a crc 0x4cee057) 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x35000~1000 to file(ino 1 size 0x35000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x35000 0x35000~1000 to file(ino 1 size 0x35000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x36000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x35000 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x36000 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:23.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000016 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 54 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000545810s (0.000000092s flush + 0.000545718s kv commit) 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55115200> 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115200 kv_submitted 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55115200 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115200 finishing 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55115200 onodes 0x558c54ce3900 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55115200 done 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:23.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55115200 [] 2019-12-07T19:34:23.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.624+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 46/28 ECSubWriteReply(tid=132, last_complete=46'55, committed=1, applied=1)) v2 -- 0x558c576b4a00 con 0x558c54c5bc00 2019-12-07T19:34:23.624+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 211 ==== MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=133, reqid=client.4540.0:376, at_version=0'0, trim_to=0'0, roll_forward_to=46'55)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c574a5000 con 0x558c54c5bc00 2019-12-07T19:34:23.628+0000 7f8f23c56700 15 osd.7 46 enqueue_op 0x558c57f3e420 prio 127 cost 0 latency 0.000020 epoch 46 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=133, reqid=client.4540.0:376, at_version=0'0, trim_to=0'0, roll_forward_to=46'55)) v2 2019-12-07T19:34:23.628+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=133, reqid=client.4540.0:376, at_version=0'0, trim_to=0'0, roll_forward_to=46'55)) v2) prio 127 cost 0 e46) 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=133, reqid=client.4540.0:376, at_version=0'0, trim_to=0'0, roll_forward_to=46'55)) v2) prio 127 cost 0 e46) queued 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=133, reqid=client.4540.0:376, at_version=0'0, trim_to=0'0, roll_forward_to=46'55)) v2) prio 127 cost 0 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e420 prio 127 cost 0 latency 0.000053 MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=133, reqid=client.4540.0:376, at_version=0'0, trim_to=0'0, roll_forward_to=46'55)) v2 pg pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'52 active mbc={} ps=[c~1,11~1]] 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'52 active mbc={} ps=[c~1,11~1]] _handle_message: MOSDECSubOpWrite(1.6s2 46/28 ECSubWrite(tid=133, reqid=client.4540.0:376, at_version=0'0, trim_to=0'0, roll_forward_to=46'55)) v2 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'52 active mbc={} ps=[c~1,11~1]] append_log log((0'0,46'55], crt=46'52) [] 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] rollforward: entry=46'53 (46'52) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:374 2019-12-07T19:34:23.618359+0000 0 ObjectCleanRegions clean_offsets: [0~851968,1425408~18446744073708126207], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] rollforward: entry=46'54 (46'53) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:375 2019-12-07T19:34:23.619502+0000 0 ObjectCleanRegions clean_offsets: [0~1425408,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] rollforward: entry=46'55 (46'54) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:376 2019-12-07T19:34:23.619518+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] append_log approx pg log length = 55 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] append_log transaction_applied = 1 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:23.628+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57cdec00 seq 88 2019-12-07T19:34:23.628+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cdec00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cdec00 onodes shared_blobs 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cdec00 allocated 0x[] released 0x[] 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdec00 prepare 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cdec00 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdec00 io_done 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e420 finish 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2500cd~471 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2500cd 0x2500cd~471 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x2200cd 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcd 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x53e and padding block with 0xac2 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x25053e 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:23.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000078692s (0.000000100s flush + 0.000078592s kv commit) 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:23.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57cdec00> 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:23.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdec00 kv_submitted 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cdec00 2019-12-07T19:34:23.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cdec00 finishing 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cdec00 onodes 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cdec00 done 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:23.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cdec00 [] 2019-12-07T19:34:23.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:23.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.628+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 46/28 ECSubWriteReply(tid=133, last_complete=46'55, committed=1, applied=1)) v2 -- 0x558c576b5000 con 0x558c54c5bc00 2019-12-07T19:34:23.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 212 ==== MOSDECSubOpRead(1.6s2 46/28 ECSubRead(tid=134, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c58294000 con 0x558c54c5bc00 2019-12-07T19:34:23.628+0000 7f8f23c56700 15 osd.7 46 enqueue_op 0x558c57f3f600 prio 127 cost 0 latency 0.000022 epoch 46 MOSDECSubOpRead(1.6s2 46/28 ECSubRead(tid=134, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:23.628+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 46/28 ECSubRead(tid=134, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e46) 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 46/28 ECSubRead(tid=134, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e46) queued 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 46/28 ECSubRead(tid=134, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e46) pg 0x558c54d5c000 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3f600 prio 127 cost 0 latency 0.000052 MOSDECSubOpRead(1.6s2 46/28 ECSubRead(tid=134, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] _handle_message: MOSDECSubOpRead(1.6s2 46/28 ECSubRead(tid=134, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:23.628+0000 7f8f0197c700 25 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:23.628+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~1000 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x63000 (405504) 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3c8c0 blob([0x63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 0x0~1000 = 4096 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] get_hash_info: Getting attr on 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] get_hash_info: not in cache 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:23.628+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key = 0 2019-12-07T19:34:23.628+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 46/28 ECSubReadReply(tid=134, attrs_read=0)) v2 -- 0x558c5840db80 con 0x558c54c5bc00 2019-12-07T19:34:23.628+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3f600 finish 2019-12-07T19:34:23.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:23.667+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.717+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 10 osd.7 46 tick_without_osd_lock 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f5bf000/0x40000000/0x1680000000, data 0xb86396/0xa3f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 25 osd.7 46 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075394+0000 last_rx_front 2019-12-07T19:34:22.075339+0000 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 25 osd.7 46 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075420+0000 last_rx_front 2019-12-07T19:34:22.075379+0000 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 25 osd.7 46 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075351+0000 last_rx_front 2019-12-07T19:34:22.075363+0000 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 25 osd.7 46 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075252+0000 last_rx_front 2019-12-07T19:34:22.075295+0000 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 25 osd.7 46 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075263+0000 last_rx_front 2019-12-07T19:34:22.075319+0000 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 20 osd.7 46 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 10 osd.7 46 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 20 osd.7 46 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:23.741+0000 7f8f1fbbe700 10 osd.7 46 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:23.768+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.818+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.868+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.918+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:23.968+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.018+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.068+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.118+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.169+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.217+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:24.218+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.269+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.318+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:24.318+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.361+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 9 ==== osd_ping(ping e46 up_from 13 ping_stamp 2019-12-07T19:34:24.362470+0000/47.811696386s send_stamp 47.811696386s delta_ub 0.014016295s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e800 con 0x558c54c65400 2019-12-07T19:34:24.361+0000 7f8f24c58700 20 osd.7 46 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013493774s,0.014016295s]) 2019-12-07T19:34:24.361+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e46 up_from 13 ping_stamp 2019-12-07T19:34:24.362470+0000/47.811696386s send_stamp 47.798202612s delta_ub 18446744073.696057842s) v5 -- 0x558c5841ec00 con 0x558c54c65400 2019-12-07T19:34:24.361+0000 7f8f24c58700 10 osd.7 46 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 45 -> 46 (as per caller) 2019-12-07T19:34:24.361+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 9 ==== osd_ping(ping e46 up_from 13 ping_stamp 2019-12-07T19:34:24.362470+0000/47.811696386s send_stamp 47.811696386s delta_ub 0.014016295s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518800 con 0x558c54c65000 2019-12-07T19:34:24.361+0000 7f8f24c58700 20 osd.7 46 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013400113s,0.014016295s]) 2019-12-07T19:34:24.361+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e46 up_from 13 ping_stamp 2019-12-07T19:34:24.362470+0000/47.811696386s send_stamp 47.798296273s delta_ub 18446744073.696151503s) v5 -- 0x558c5841ec00 con 0x558c54c65000 2019-12-07T19:34:24.368+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.384+0000 7f8f21451700 10 osd.7 46 tick 2019-12-07T19:34:24.384+0000 7f8f21451700 10 osd.7 46 do_waiters -- start 2019-12-07T19:34:24.384+0000 7f8f21451700 10 osd.7 46 do_waiters -- finish 2019-12-07T19:34:24.384+0000 7f8f21451700 20 osd.7 46 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:24.419+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.469+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.519+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.530+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57561800 0x558c552d7600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:24.530+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57561800 0x558c552d7600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:24.530+0000 7f8f24457700 10 osd.7 46 ms_handle_authentication new session 0x558c57404d00 con 0x558c57561800 entity client.admin addr 2019-12-07T19:34:24.530+0000 7f8f24457700 10 osd.7 46 ms_handle_authentication session 0x558c57404d00 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:24.530+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1210169592 conn(0x558c57561800 0x558c552d7600 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4849 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:24.530+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4849 172.21.15.131:0/1210169592 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c5734b0e0 con 0x558c57561800 2019-12-07T19:34:24.530+0000 7f8f164bb700 20 osd.7 46 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:24.530+0000 7f8f164bb700 10 osd.7 46 do_waiters -- start 2019-12-07T19:34:24.530+0000 7f8f164bb700 10 osd.7 46 do_waiters -- finish 2019-12-07T19:34:24.530+0000 7f8f164bb700 20 osd.7 46 _dispatch 0x558c5734b0e0 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:24.530+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1210169592 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce0540 con 0x558c57561800 2019-12-07T19:34:24.551+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4849 172.21.15.131:0/1210169592 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c5734b2c0 con 0x558c57561800 2019-12-07T19:34:24.552+0000 7f8f164bb700 20 osd.7 46 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:24.552+0000 7f8f164bb700 10 osd.7 46 do_waiters -- start 2019-12-07T19:34:24.552+0000 7f8f164bb700 10 osd.7 46 do_waiters -- finish 2019-12-07T19:34:24.552+0000 7f8f164bb700 20 osd.7 46 _dispatch 0x558c5734b2c0 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:24.553+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1210169592 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce1180 con 0x558c57561800 2019-12-07T19:34:24.555+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:24.555+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1210169592 conn(0x558c57561800 msgr2=0x558c552d7600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 69 2019-12-07T19:34:24.555+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1210169592 conn(0x558c57561800 msgr2=0x558c552d7600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:24.555+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1210169592 conn(0x558c57561800 0x558c552d7600 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:24.555+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1210169592 conn(0x558c57561800 0x558c552d7600 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:24.555+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] reap_dead start 2019-12-07T19:34:24.555+0000 7f8f164bb700 2 osd.7 46 ms_handle_reset con 0x558c57561800 session 0x558c57404d00 2019-12-07T19:34:24.570+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1233813 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 360 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:24.615+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.8s5 epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:24.615+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e47) pg 0x558c57380000 2019-12-07T19:34:24.615+0000 7f8f0898a700 20 osd.7 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 47, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:24.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 361 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:24.615+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.9s4 epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:24.615+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 213 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c58124300 con 0x558c54c5bc00 2019-12-07T19:34:24.615+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.1s1 epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:24.615+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e47) pg 0x558c5500a000 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 47, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e47) pg 0x558c5736a000 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 47, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.615+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:24.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 214 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c58124300 con 0x558c54c5bc00 2019-12-07T19:34:24.616+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.fs5 epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:24.616+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 362 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:24.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e47) pg 0x558c57383000 2019-12-07T19:34:24.616+0000 7f8f05183700 20 osd.7 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 47, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.616+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.0s3 epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:24.616+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 363 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:24.616+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.as5 epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:24.616+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e47) pg 0x558c5500d000 2019-12-07T19:34:24.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.616+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 47, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.616+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:24.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e47) pg 0x558c57e8b000 2019-12-07T19:34:24.616+0000 7f8f03980700 20 osd.7 op_wq _add_slot_waiter 1.as5 peering, item epoch is 47, will wait on OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:24.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 364 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:24.616+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.es4 epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:24.616+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 365 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a55100 con 0x558c54c64000 2019-12-07T19:34:24.616+0000 7f8f24457700 15 osd.7 46 enqueue_peering_evt 1.7s4 epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:24.616+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e47) pg 0x558c54d98000 2019-12-07T19:34:24.617+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.es4 peering, item epoch is 47, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e47) pg 0x558c54d5f000 2019-12-07T19:34:24.617+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 47, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.617+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 216 ==== MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=135, reqid=osd.3.0:135, at_version=47'57, trim_to=0'0, roll_forward_to=46'55)) v2 ==== 3458+0+0 (crc 0 0 0) 0x558c574a4800 con 0x558c54c5bc00 2019-12-07T19:34:24.617+0000 7f8f23c56700 15 osd.7 46 enqueue_op 0x558c57f3e580 prio 127 cost 0 latency 0.000043 epoch 47 MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=135, reqid=osd.3.0:135, at_version=47'57, trim_to=0'0, roll_forward_to=46'55)) v2 2019-12-07T19:34:24.617+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=135, reqid=osd.3.0:135, at_version=47'57, trim_to=0'0, roll_forward_to=46'55)) v2) prio 127 cost 0 e47) 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=135, reqid=osd.3.0:135, at_version=47'57, trim_to=0'0, roll_forward_to=46'55)) v2) prio 127 cost 0 e47) queued 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=135, reqid=osd.3.0:135, at_version=47'57, trim_to=0'0, roll_forward_to=46'55)) v2) prio 127 cost 0 e47) pg 0x558c54d5c000 2019-12-07T19:34:24.617+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 240 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c58124300 con 0x558c54c5b400 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e580 prio 127 cost 0 latency 0.000083 MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=135, reqid=osd.3.0:135, at_version=47'57, trim_to=0'0, roll_forward_to=46'55)) v2 pg pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] 2019-12-07T19:34:24.617+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.bs1 epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 46 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 46 -> 47 (as per caller) 2019-12-07T19:34:24.617+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] _handle_message: MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=135, reqid=osd.3.0:135, at_version=47'57, trim_to=0'0, roll_forward_to=46'55)) v2 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6336c6f2:::smithi13140796-16:17 e,13,15 was 2019-12-07T19:34:24.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6336c6f2:::smithi13140796-16:17 e,13,15,17 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6336c6f2:::smithi13140796-16:17 e,13,15 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:24.617+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 215 ==== osd_map(47..47 src has 1..47) v4 ==== 661+0+0 (crc 0 0 0) 0x558c57324b40 con 0x558c54c5bc00 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_0000000000000017_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:24.617+0000 7f8f1199c700 20 osd.7 46 OSD::ms_dispatch: osd_map(47..47 src has 1..47) v4 2019-12-07T19:34:24.617+0000 7f8f1199c700 10 osd.7 46 do_waiters -- start 2019-12-07T19:34:24.617+0000 7f8f1199c700 10 osd.7 46 do_waiters -- finish 2019-12-07T19:34:24.617+0000 7f8f1199c700 20 osd.7 46 _dispatch 0x558c57324b40 osd_map(47..47 src has 1..47) v4 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 46'55 (0'0,46'55] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] append_log log((0'0,46'55], crt=46'55) [47'56 (46'51) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:24.617765+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,47'57 (46'55) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:24.617765+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:24.617+0000 7f8f1199c700 3 osd.7 46 handle_osd_map epochs [47,47], i have 46, src has [1,47] 2019-12-07T19:34:24.617+0000 7f8f1199c700 10 osd.7 46 handle_osd_map got inc map for epoch 47 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 47'56 (0'0,47'56] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'55 crt=46'55 lcod 46'55 active mbc={} ps=[c~1,11~1]] add_log_entry 47'56 (46'51) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:24.617765+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:24.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3328e826:::osdmap.46:0# 0x0~0 2019-12-07T19:34:24.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1474 size 0x1474 (5236) 2019-12-07T19:34:24.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'55 crt=46'55 lcod 46'55 active mbc={} ps=[c~1,11~1]] add_log_entry 47'57 (46'55) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:24.617765+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:24.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55060540 blob([0x9ec000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df8540 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:24.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54df84d0 blob([!~1000,0x9ed000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,474]) SharedBlob(0x558c54df9b20 sbid 0x0)) need 0x1000~474 cache has 0x[1000~474] 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'55 crt=46'55 lcod 46'55 active mbc={} ps=[c~1,11~1]] append_log approx pg log length = 57 2019-12-07T19:34:24.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3328e826:::osdmap.46:0# 0x0~1474 = 5236 2019-12-07T19:34:24.617+0000 7f8f1199c700 10 osd.7 46 add_map_bl 46 5236 bytes 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'55 crt=46'55 lcod 46'55 active mbc={} ps=[c~1,11~1]] append_log transaction_applied = 1 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:24.617+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 47'56, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:24.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57cde900 seq 89 2019-12-07T19:34:24.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys = 0 2019-12-07T19:34:24.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) = 0 2019-12-07T19:34:24.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:34:24.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:34:24.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 osd.7 46 got_full_map 47, nothing requested 2019-12-07T19:34:24.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:24.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000017_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_0000000000000017_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:24.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000047.00000000000000000056' <- 0000000047.00000000000000000056 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000047.00000000000000000057' <- 0000000047.00000000000000000057 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cde900 cost 8429 (1 ios * 4000 + 4429 bytes) 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cde900 onodes 0x558c54ce3900,0x558c57f8e500 shared_blobs 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 osd.7 46 handle_osd_map pg_num_history pg_num_history(e47 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {47={}} 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 osd.7 46 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e47 [1,47] lci=[13,47]) 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1595 (574 bytes onode + 2 bytes spanning blobs + 1019 bytes inline extents) 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e608) update 2#1:6336c6f2:::smithi13140796-16:17# 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:17# is 637 (412 bytes onode + 2 bytes spanning blobs + 223 bytes inline extents) 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cde900 allocated 0x[] released 0x[] 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c5829b200 seq 38 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde900 prepare 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c137972f:::inc_osdmap.47:0# key 0x7f7fffffffffffffffc13797'/!inc_osdmap.47!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cde900 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde900 io_done 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=46'55 crt=46'55 lcod 46'55 active mbc={} ps=[c~1,11~1]] op_applied version 47'57 2019-12-07T19:34:24.618+0000 7f8f0197c700 10 osd.7 46 dequeue_op 0x558c57f3e580 finish 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c137972f:::inc_osdmap.47:0# 0x0~269 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1231 2019-12-07T19:34:24.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e47) pg 0x558c575ef000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c137972f:::inc_osdmap.47:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:24.618+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 47, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:24.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b200 1 blobs 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa41000~1000/1000,1000,0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa41000~1000] 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7ab0 blob([0xa41000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91420 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c580d7ab0) get_ref 0x0~269 Blob(0x558c580d7ab0 blob([0xa41000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91420 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c580d7ab0) get_ref init 0x1000, 1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c580d7ab0 blob([0xa41000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91420 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57f91438 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58093c88) dirty_range mark inline shard dirty 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c137972f:::inc_osdmap.47:0# 0x0~269 = 0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:39a8e826:::osdmap.47:0# key 0x7f7fffffffffffffff39a8e8'&!osdmap.47!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:24.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:39a8e826:::osdmap.47:0# 0x0~1464 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1232 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:39a8e826:::osdmap.47:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b200 2 blobs 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa42000~2000/1000,2000,0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa42000~2000] 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b2bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b31f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b2bd0 blob([0xa42000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b31f0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 241 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c58124300 con 0x558c54c5b400 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c550b2bd0) get_ref 0x0~1000 Blob(0x558c550b2bd0 blob([0xa42000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b31f0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c550b2bd0) get_ref init 0x1000, 1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c550b2bd0 blob([0xa42000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c550b31f0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c550b3208 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5757fea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f911f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:24.618+0000 7f8f23c56700 15 osd.7 46 enqueue_peering_evt 1.3s4 epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5757fea0 blob([!~1000,0xa43000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f911f0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c5757fea0) get_ref 0x1000~464 Blob(0x558c5757fea0 blob([!~1000,0xa43000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57f911f0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c5757fea0) get_ref init 0x2000, 1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c5757fea0 blob([!~1000,0xa43000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57f911f0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57f91208 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58092608) dirty_range mark inline shard dirty 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:39a8e826:::osdmap.47:0# 0x0~1464 = 0 2019-12-07T19:34:24.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:24.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e47) pg 0x558c5513e000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 47, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b200 0 blobs 2019-12-07T19:34:24.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:24.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:24.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:24.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b200 0 blobs 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5829b200 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5829b200 onodes 0x558c53a91900,0x558c55276c80,0x558c58092500,0x558c58093b80 shared_blobs 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58092608) update #-1:39a8e826:::osdmap.47:0# 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58092608) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:39a8e826:::osdmap.47:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58093c88) update #-1:c137972f:::inc_osdmap.47:0# 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58093c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c137972f:::inc_osdmap.47:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5829b200 allocated 0x[a41000~3000] released 0x[] 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 freelist allocate 0xa41000~3000 2019-12-07T19:34:24.618+0000 7f8f1199c700 20 freelist _xor first_key 0xa00000 last_key 0xa00000 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b200 prepare 2019-12-07T19:34:24.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5829b200 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5107 < min_flush_size 524288 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e500 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5130 < min_flush_size 524288 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5130 < min_flush_size 524288 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x25053e~140a to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x25053e 0x25053e~140a to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x22053e 2019-12-07T19:34:24.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b200 aio_wait 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x53e 2019-12-07T19:34:24.618+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5829b200 2019-12-07T19:34:24.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b200 io_done 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x948 and padding block with 0x6b8 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x251948 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000203206s (0.000000171s flush + 0.000203035s kv commit) 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57cde900> 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:24.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde900 kv_submitted 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cde900 2019-12-07T19:34:24.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde900 finishing 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cde900 onodes 0x558c54ce3900,0x558c57f8e500 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cde900 done 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:24.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cde900 [] 2019-12-07T19:34:24.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:24.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:24.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:24.618+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 46/28 ECSubWriteReply(tid=135, last_complete=47'57, committed=1, applied=1)) v2 -- 0x558c576b4400 con 0x558c54c5bc00 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092500 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093b80 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x251948~226c to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x251948 0x251948~226c to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x221948 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x948 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:24.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbb4 and padding block with 0x44c 2019-12-07T19:34:24.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 143 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57ed5100 con 0x558c54c19800 2019-12-07T19:34:24.619+0000 7f8f24c58700 15 osd.7 46 enqueue_peering_evt 1.ds2 epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:24.619+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e47) pg 0x558c55141000 2019-12-07T19:34:24.619+0000 7f8f06185700 20 osd.7 op_wq _add_slot_waiter 1.ds2 peering, item epoch is 47, will wait on OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.619+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 144 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57ed5100 con 0x558c54c19800 2019-12-07T19:34:24.619+0000 7f8f24c58700 15 osd.7 46 enqueue_peering_evt 1.5s4 epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:24.619+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e47) pg 0x558c57522000 2019-12-07T19:34:24.619+0000 7f8f0217d700 20 osd.7 op_wq _add_slot_waiter 1.5s4 peering, item epoch is 47, will wait on OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.619+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x253bb4 2019-12-07T19:34:24.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:24.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:24.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:24.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:24.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001893380s (0.000004668s flush + 0.001888712s kv commit) 2019-12-07T19:34:24.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5829b200> 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:24.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b200 kv_submitted 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5829b200 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c5829b200 osr 0x558c54ea4b40 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 35 0x1b000~1000 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7 46 _committed_osd_maps 47..47 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7 46 advance to epoch 47 (<= last 47 <= newest_map 47) 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 36 0x1b000~1000 crc 1f602716 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 35 0x2000~1000 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 36 0x2000~1000 crc bb4af5b4 2019-12-07T19:34:24.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:24.620+0000 7f8f141a1700 7 osd.7 47 consume_map version 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.8s5 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.0s3 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.1s1 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.9s4 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.as5 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.2s0 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.bs1 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.3s4 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.cs0 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.4s0 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.5s4 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.ds2 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.es4 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.6s2 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.fs5 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 identify_splits_and_merges 1.7s4 e46 to e47 pg_nums {18=16} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 47 prune_pg_created 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:0.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:1.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e47) pg 0x558c5500d000 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:2.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.as5 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:3.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s)) prio 255 cost 10 e47) pg 0x558c57e8b000 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:4.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s)) prio 255 cost 10 e47) pg 0x558c5500a000 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:5.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.5s4 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.ds2 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:6.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7:7.consume_map 47 (was 46) 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 47 <= 47, requeueing 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {47=} 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s)) prio 255 cost 10 e47) pg 0x558c55141000 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s)) prio 255 cost 10 e47) pg 0x558c54d5f000 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 46 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 46 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f141a1700 10 osd.7 47 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.8s5 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.0s3 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.620+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.1s1 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.620+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.069490446s ub 59.069490446s int 16s) 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068413248s ub 59.068413248s int 16s) 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:24.620+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:24.620+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c55122300 seq 40 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:24.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c5740ac00 seq 14 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:24.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 217 ==== MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=136, reqid=osd.3.0:135, at_version=0'0, trim_to=0'0, roll_forward_to=47'57)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748e800 con 0x558c54c5bc00 2019-12-07T19:34:24.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:24.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5511c600 seq 67 2019-12-07T19:34:24.620+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.620+0000 7f8f23c56700 15 osd.7 47 enqueue_op 0x558c57efe580 prio 127 cost 0 latency 0.000028 epoch 47 MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=136, reqid=osd.3.0:135, at_version=0'0, trim_to=0'0, roll_forward_to=47'57)) v2 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55122300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55122300 onodes shared_blobs 2019-12-07T19:34:24.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55122300 allocated 0x[] released 0x[] 2019-12-07T19:34:24.620+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=136, reqid=osd.3.0:135, at_version=0'0, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e47) 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.620+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:24.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:24.620+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:24.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:24.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122300 prepare 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55122300 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.082500675s ub 59.082500675s int 16s) 2019-12-07T19:34:24.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122300 io_done 2019-12-07T19:34:24.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5740ac00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5740ac00 onodes shared_blobs 2019-12-07T19:34:24.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:24.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5740ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:24.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511c600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511c600 onodes shared_blobs 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 46 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:24.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511c600 allocated 0x[] released 0x[] 2019-12-07T19:34:24.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 prepare 2019-12-07T19:34:24.620+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5740ac00 2019-12-07T19:34:24.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 io_done 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s)) prio 255 cost 10 e47) pg 0x558c57380000 2019-12-07T19:34:24.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:24.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c600 prepare 2019-12-07T19:34:24.620+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511c600 2019-12-07T19:34:24.620+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c600 io_done 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5815ec00 seq 43 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s)) prio 255 cost 10 e47) pg 0x558c5736a000 2019-12-07T19:34:24.620+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:24.620+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 46 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:24.620+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815ec00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.620+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815ec00 onodes shared_blobs 2019-12-07T19:34:24.620+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815ec00 allocated 0x[] released 0x[] 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ec00 prepare 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815ec00 2019-12-07T19:34:24.620+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ec00 io_done 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s)) prio 255 cost 10 e47) pg 0x558c57522000 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 46 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:24.620+0000 7f8f05183700 20 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.620+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.7s0( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068577717s ub 59.068577717s int 16s) 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.620+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 46 -> 47 2019-12-07T19:34:24.620+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 46 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.620+0000 7f8f0898a700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.620+0000 7f8f06185700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.620+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c582a8000 seq 65 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.620+0000 7f8f04181700 20 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:24.620+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f06185700 20 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064307871s ub 59.064307871s int 16s) 2019-12-07T19:34:24.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.081729084s ub 59.081729084s int 16s) 2019-12-07T19:34:24.621+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068340106s ub 59.068340106s int 16s) 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a8000 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a8000 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a8000 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8000 prepare 2019-12-07T19:34:24.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a8000 2019-12-07T19:34:24.621+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c5511c900 seq 17 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8000 io_done 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:24.621+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:24.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5815ef00 seq 16 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c55122600 seq 33 2019-12-07T19:34:24.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:24.621+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s)) prio 255 cost 10 e47) pg 0x558c57383000 2019-12-07T19:34:24.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:24.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511c900 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511c900 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511c900 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815ef00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c900 prepare 2019-12-07T19:34:24.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815ef00 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511c900 2019-12-07T19:34:24.621+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c900 io_done 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55122600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815ef00 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55122600 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55122600 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.620+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.9s4 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c5736a000 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122600 prepare 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55122600 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122600 io_done 2019-12-07T19:34:24.621+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 46 -> 47 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.as5 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c5500a000 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 46 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:24.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 218 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e47/47) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c58124300 con 0x558c54c5bc00 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c57380000 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.2s0 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.bs1 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.3s4 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f23c56700 15 osd.7 47 enqueue_peering_evt 1.6s2 epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.cs0 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.4s0 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.5s4 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.ds2 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.es4 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.6s2 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c5500d000 2019-12-07T19:34:24.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.fs5 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c57676000 2019-12-07T19:34:24.621+0000 7f8f141a1700 15 osd.7 47 enqueue_peering_evt 1.7s4 epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) 2019-12-07T19:34:24.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f141a1700 7 osd.7 47 activate_map version 47 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c57e8b000 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.064164481s ub 59.064164481s int 16s) 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 7 submitting 7 deferred done 0 stable 0 2019-12-07T19:34:24.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ef00 prepare 2019-12-07T19:34:24.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815ef00 2019-12-07T19:34:24.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ef00 io_done 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 46 -> 47 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c57e88000 2019-12-07T19:34:24.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c55141000 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c582a8300 seq 19 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2300 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1]] state: Active advmap 2019-12-07T19:34:24.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:24.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3450 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] on_active_advmap new removed_snaps [17~1], snap_trimq now [17~1] 2019-12-07T19:34:24.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c57522000 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4600 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 46 -> 47 2019-12-07T19:34:24.621+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:24.621+0000 7f8f0918b700 20 osd.7 47 agent_entry empty queue 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5750 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a8300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 46 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a8300 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a8300 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1]] state: Active advmap 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] on_active_advmap new removed_snaps [17~1], snap_trimq now [17~1] 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8300 prepare 2019-12-07T19:34:24.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a8300 2019-12-07T19:34:24.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8300 io_done 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.621+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c54d5f000 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] state: Active: handling ActMap 2019-12-07T19:34:24.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c57383000 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.621+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] Active: kicking snap trim 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] state: Active: handling ActMap 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] Active: kicking snap trim 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] exit NotTrimming 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] exit NotTrimming 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming/WaitReservation 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming/WaitReservation 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f03980700 15 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 46:129 2019-12-07T19:34:24.621+0000 7f8f03980700 7 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] state: handle ActMap primary 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f03980700 15 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 46:130 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6900 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8050 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8073 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: NotTrimming: trimming 17 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8073 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] exit Trimming/WaitReservation 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x253bb4~1f89 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x253bb4 0x253bb4~1f89 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x223bb4 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbb4 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:24.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 47:131 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb3d and padding block with 0x4c3 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:24.621+0000 7f8f0b990700 10 osd.7 47 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[17~1] ps=[c~1,11~1]] for snaptrim 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=47) prio 5 cost 1048576 e47) 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 47:132 2019-12-07T19:34:24.621+0000 7f8f06986700 15 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 46:144 2019-12-07T19:34:24.621+0000 7f8f06986700 7 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] state: handle ActMap primary 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s)) prio 255 cost 10 e47) pg 0x558c5513e000 2019-12-07T19:34:24.621+0000 7f8f06986700 15 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 46:145 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 46 -> 47 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c548ab000 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 46 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:24.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: NotTrimming: trimming 17 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] exit Trimming/WaitReservation 2019-12-07T19:34:24.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 46 -> 47 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 46 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:24.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 47:146 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1]] state: Active advmap 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.621+0000 7f8f0b990700 10 osd.7 47 queueing pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean trimq=[17~1] ps=[c~1,11~1]] for snaptrim 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=47) prio 5 cost 1048576 e47) 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] on_active_advmap new removed_snaps [17~1], snap_trimq now [17~1] 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=47) prio 5 cost 1048576 e47) queued 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.056742504s ub 59.056742504s int 16s) 2019-12-07T19:34:24.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:24.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 47:147 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=47) prio 5 cost 1048576 e47) pg 0x558c57676000 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] snap_trimmer posting 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c583c9800 seq 57 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 17 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] state: Active: handling ActMap 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c9800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c9800 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] Active: kicking snap trim 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c9800 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9800 prepare 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c9800 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9800 io_done 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f9c0) upper_bound after SNA_1_0000000000000017_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000017_.0_0000000000000001.C' 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] exit NotTrimming 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s)) prio 255 cost 10 e47) pg 0x558c575ef000 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1f9c0) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming/WaitReservation 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000017_.0_0000000000000001.C) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[c~1,11~1]] SnapTrimmer state: adding snap 17 to purged_snaps 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.621+0000 7f8f06986700 15 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 46:115 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 46 -> 47 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f06986700 7 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] state: handle ActMap primary 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 46 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[c~1,11~1,17~1]] SnapTrimmer state: purged_snaps now [c~1,11~1,17~1], snap_trimq now [] 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f6600 seq 42 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x255b3d 2019-12-07T19:34:24.621+0000 7f8f06986700 15 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 46:116 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:24.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:24.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f6600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f6600 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f6600 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 7 cleaned 0 in 0.000561912s (0.000000144s flush + 0.000561768s kv commit) 2019-12-07T19:34:24.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6600 prepare 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f6600 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6600 io_done 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=47) prio 5 cost 1048576 e47) queued 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55122300,0x558c5740ac00,0x558c5511c600,0x558c5815ec00,0x558c582a8000,0x558c5511c900,0x558c55122600> 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[c~1,11~1,17~1]] share_pg_info 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s)) prio 255 cost 10 e47) pg 0x558c54d98000 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122300 kv_submitted 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55122300 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=47) prio 5 cost 1048576 e47) pg 0x558c57e88000 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122300 finishing 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2300 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55122300 onodes 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55122300 done 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55122300 [] 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e47/47) v1 -- 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.621+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] snap_trimmer posting 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 kv_submitted 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5740ac00 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3450 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 46 -> 47 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5740ac00 finishing 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5740ac00 onodes 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 17 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5740ac00 done 2019-12-07T19:34:24.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5740ac00 [] 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3473 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.057851957s ub 59.057851957s int 16s) 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3473 < min_flush_size 524288 2019-12-07T19:34:24.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:24.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e47/47) v1 -- 0x558c574e7100 con 0x558c54c19800 2019-12-07T19:34:24.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c600 kv_submitted 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x255b3d~d91 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511c600 2019-12-07T19:34:24.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x255b3d 0x255b3d~d91 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c600 finishing 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511c600 onodes 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511c600 done 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e47/47) v1 -- 0x558c58134700 con 0x558c54c5bc00 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c583c9b00 seq 35 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511c600 [] 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ec00 kv_submitted 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57d1fa00) upper_bound after SNA_1_0000000000000017_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000017_.0_0000000000000001.2' 2019-12-07T19:34:24.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815ec00 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ec00 finishing 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x225b3d 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57d1fa00) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815ec00 onodes 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815ec00 done 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb3d 2019-12-07T19:34:24.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e47/47) v1 -- 0x558c58135500 con 0x558c54c5b400 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815ec00 [] 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.621+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000017_.0_0000000000000001.2) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8ce and padding block with 0x732 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8000 kv_submitted 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a8000 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8000 finishing 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a8000 onodes 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a8000 done 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583c9b00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583c9b00 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583c9b00 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a8000 [] 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.621+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[c~1,11~1]] SnapTrimmer state: adding snap 17 to purged_snaps 2019-12-07T19:34:24.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e47/47) v1 -- 0x558c58135c00 con 0x558c54c19c00 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9b00 prepare 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c900 kv_submitted 2019-12-07T19:34:24.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583c9b00 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511c900 2019-12-07T19:34:24.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9b00 io_done 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511c900 finishing 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511c900 onodes 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511c900 done 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511c900 [] 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122600 kv_submitted 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55122600 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122600 finishing 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55122600 onodes 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55122600 done 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[c~1,11~1,17~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55122600 [] 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:24.621+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[c~1,11~1,17~1]] SnapTrimmer state: purged_snaps now [c~1,11~1,17~1], snap_trimq now [] 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 59.068156525s ub 59.068156525s int 16s) 2019-12-07T19:34:24.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean+snaptrim ps=[c~1,11~1,17~1]] exit Trimming 2019-12-07T19:34:24.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c575ef000 2019-12-07T19:34:24.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c582a9b00 seq 23 2019-12-07T19:34:24.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0297e700 15 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:148 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] enter NotTrimming 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2568ce 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c5513e000 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c57cde600 seq 15 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:24.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:24.621+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a9b00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a9b00 onodes shared_blobs 2019-12-07T19:34:24.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a9b00 allocated 0x[] released 0x[] 2019-12-07T19:34:24.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:24.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] snap_trimmer complete 2019-12-07T19:34:24.621+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:24.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000203330s (0.000000136s flush + 0.000203194s kv commit) 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:24.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:24.621+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:24.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9b00 prepare 2019-12-07T19:34:24.622+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a9b00 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:24.622+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9b00 io_done 2019-12-07T19:34:24.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5815ef00,0x558c582a8300,0x558c583c9800> 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cde600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cde600 onodes shared_blobs 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ef00 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815ef00 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cde600 allocated 0x[] released 0x[] 2019-12-07T19:34:24.622+0000 7f8f0b990700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:24.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[c~1,11~1,17~1]] share_pg_info 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde600 prepare 2019-12-07T19:34:24.622+0000 7f8f0b990700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: NotTrimming: trimming 17 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cde600 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde600 io_done 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815ef00 finishing 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815ef00 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815ef00 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815ef00 [] 2019-12-07T19:34:24.622+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[17~1] ps=[c~1,11~1]] exit Trimming/WaitReservation 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2300 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8300 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a8300 2019-12-07T19:34:24.622+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c54e04700 con 0x558c54c64000 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e47) pg 0x558c54d5c000 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8300 finishing 2019-12-07T19:34:24.622+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a8300 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a8300 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a8300 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9800 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c9800 2019-12-07T19:34:24.622+0000 7f8f0b990700 15 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 47:117 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9800 finishing 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c9800 onodes 2019-12-07T19:34:24.622+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c54e04e00 con 0x558c54c19800 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c9800 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2323 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c9800 [] 2019-12-07T19:34:24.622+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2323 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f0b990700 10 osd.7 47 queueing pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[17~1] ps=[c~1,11~1]] for snaptrim 2019-12-07T19:34:24.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=47) prio 5 cost 1048576 e47) 2019-12-07T19:34:24.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] handle_advance_map: 47 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 46 -> 47 2019-12-07T19:34:24.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2568ce~913 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c54e05c00 con 0x558c54c5bc00 2019-12-07T19:34:24.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f0b990700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1] 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2568ce 0x2568ce~913 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 46 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f0b990700 15 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] publish_stats_to_osd 47:118 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x2268ce 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8ce 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1e1 and padding block with 0xe1f 2019-12-07T19:34:24.622+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c54e06300 con 0x558c54c5b400 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] state: Started advmap 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:24.622+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:24.622+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c54e05500 con 0x558c54c19c00 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] plpg_on_pool_change 2019-12-07T19:34:24.622+0000 7f8f07988700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[c~1,11~1,17~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] hit_set_clear 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] agent_stop 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 47 advance_pg new pool opts old pool opts 2019-12-07T19:34:24.622+0000 7f8f07988700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[c~1,11~1,17~1]] exit Trimming 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] handle_activate_map: 47 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=47) prio 5 cost 1048576 e47) queued 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f07988700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] activate_map 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=47) prio 5 cost 1048576 e47) pg 0x558c548ab000 2019-12-07T19:34:24.622+0000 7f8f07988700 15 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:133 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] activate_map: Not dirtying info: last_persisted is 46 while current is 47 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] snap_trimmer posting 2019-12-07T19:34:24.622+0000 7f8f07988700 20 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] enter NotTrimming 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 17 2019-12-07T19:34:24.622+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1]] proc_primary_info updating purged_snaps to [c~1,11~1,17~1] 2019-12-07T19:34:24.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:24.622+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:24.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:24.622+0000 7f8f07988700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] snap_trimmer complete 2019-12-07T19:34:24.622+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c57cde300 seq 90 2019-12-07T19:34:24.622+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57cde300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57cde300 onodes shared_blobs 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57cde300 allocated 0x[] released 0x[] 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57d1e840) upper_bound after SNA_1_0000000000000017_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000017_.0_0000000000000001.4' 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde300 prepare 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57cde300 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57d1e840) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde300 io_done 2019-12-07T19:34:24.622+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000017_.0_0000000000000001.4) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c54d98000 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[17~1] ps=[c~1,11~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[c~1,11~1]] SnapTrimmer state: adding snap 17 to purged_snaps 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) queued 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 47 epoch_requested: 47 NullEvt) prio 255 cost 10 e47) pg 0x558c54d5c000 2019-12-07T19:34:24.622+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 47 epoch_requested: 47 NullEvt 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[c~1,11~1,17~1]] SnapTrimmer state: purged_snaps now [c~1,11~1,17~1], snap_trimq now [] 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=136, reqid=osd.3.0:135, at_version=0'0, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e47) queued 2019-12-07T19:34:24.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=136, reqid=osd.3.0:135, at_version=0'0, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e47) pg 0x558c54d5c000 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c582ac000 seq 38 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 47 dequeue_op 0x558c57efe580 prio 127 cost 0 latency 0.001457 MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=136, reqid=osd.3.0:135, at_version=0'0, trim_to=0'0, roll_forward_to=47'57)) v2 pg pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1,17~1]] 2019-12-07T19:34:24.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWrite(1.6s2 47/28 ECSubWrite(tid=136, reqid=osd.3.0:135, at_version=0'0, trim_to=0'0, roll_forward_to=47'57)) v2 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:24.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:24.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ac000 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=46'55 active mbc={} ps=[c~1,11~1,17~1]] append_log log((0'0,47'57], crt=46'55) [] 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ac000 onodes shared_blobs 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ac000 allocated 0x[] released 0x[] 2019-12-07T19:34:24.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac000 prepare 2019-12-07T19:34:24.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ac000 2019-12-07T19:34:24.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac000 io_done 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] rollforward: entry=47'56 (46'51) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:24.617765+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[c~1,11~1,17~1]] share_pg_info 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] rollforward: entry=47'57 (46'55) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:24.617765+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] append_log approx pg log length = 57 2019-12-07T19:34:24.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c57a55800 con 0x558c54c64000 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] append_log transaction_applied = 1 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:24.622+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:24.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c57379c00 con 0x558c54c19800 2019-12-07T19:34:24.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:24.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582ae000 seq 91 2019-12-07T19:34:24.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e47/47) v1 -- 0x558c57378000 con 0x558c54c19c00 2019-12-07T19:34:24.622+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[c~1,11~1,17~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[c~1,11~1,17~1]] exit Trimming 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ae000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ae000 onodes shared_blobs 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ae000 allocated 0x[] released 0x[] 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:24.622+0000 7f8f06986700 15 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:119 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae000 prepare 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ae000 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] enter NotTrimming 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae000 io_done 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:24.622+0000 7f8f0197c700 10 osd.7 47 dequeue_op 0x558c57efe580 finish 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:24.622+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] snap_trimmer complete 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2571e1 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000450428s (0.000000096s flush + 0.000450332s kv commit) 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585f6600,0x558c583c9b00> 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6600 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f6600 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6600 finishing 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f6600 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6600 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f6600 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9b00 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583c9b00 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:24.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583c9b00 finishing 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2300 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583c9b00 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583c9b00 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583c9b00 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3450 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 2 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3615 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4611 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5725 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5748 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5748 < min_flush_size 524288 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2571e1~1674 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2571e1 0x2571e1~1674 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x2271e1 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1e1 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x855 and padding block with 0x7ab 2019-12-07T19:34:24.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x258855 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:24.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 5 cleaned 0 in 0.000184739s (0.000000081s flush + 0.000184658s kv commit) 2019-12-07T19:34:24.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582a9b00,0x558c57cde600,0x558c57cde300,0x558c582ac000,0x558c582ae000> 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9b00 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a9b00 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9b00 finishing 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a9b00 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a9b00 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a9b00 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde600 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cde600 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde600 finishing 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cde600 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cde600 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:24.622+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cde600 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde300 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57cde300 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57cde300 finishing 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57cde300 onodes 2019-12-07T19:34:24.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57cde300 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ae000 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57cde300 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac000 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ac000 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac000 finishing 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ac000 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ac000 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ac000 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae000 kv_submitted 2019-12-07T19:34:24.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ae000 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae000 finishing 2019-12-07T19:34:24.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ae000 onodes 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ae000 done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ae000 [] 2019-12-07T19:34:24.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:24.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:24.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.622+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 47/28 ECSubWriteReply(tid=136, last_complete=47'57, committed=1, applied=1)) v2 -- 0x558c576b5e00 con 0x558c54c5bc00 2019-12-07T19:34:24.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:24.669+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:24.669+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:24.669+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 36 0x2000~1000 2019-12-07T19:34:24.669+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc bb4af5b4 2019-12-07T19:34:24.669+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 36 0x1b000~1000 2019-12-07T19:34:24.669+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 1f602716 2019-12-07T19:34:24.669+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.669+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:24.669+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:24.672+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:24.719+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 10 osd.7 47 tick_without_osd_lock 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f5bc000/0x40000000/0x1680000000, data 0xb87a63/0xa42000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 25 osd.7 47 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075394+0000 last_rx_front 2019-12-07T19:34:22.075339+0000 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 25 osd.7 47 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075420+0000 last_rx_front 2019-12-07T19:34:22.075379+0000 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 25 osd.7 47 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075351+0000 last_rx_front 2019-12-07T19:34:22.075363+0000 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 25 osd.7 47 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075252+0000 last_rx_front 2019-12-07T19:34:22.075295+0000 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 25 osd.7 47 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075263+0000 last_rx_front 2019-12-07T19:34:22.075319+0000 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 20 osd.7 47 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 10 osd.7 47 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 20 osd.7 47 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:24.744+0000 7f8f1fbbe700 10 osd.7 47 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:24.769+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.820+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.870+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.920+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:24.944+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 14 ==== osd_ping(ping e47 up_from 13 ping_stamp 2019-12-07T19:34:24.945114+0000/48.416833322s send_stamp 48.416833322s delta_ub 0.036448982s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4e00 con 0x558c552d1c00 2019-12-07T19:34:24.944+0000 7f8f24457700 20 osd.7 47 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036081980s,0.036448982s]) 2019-12-07T19:34:24.944+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 14 ==== osd_ping(ping e47 up_from 13 ping_stamp 2019-12-07T19:34:24.945114+0000/48.416833322s send_stamp 48.416833322s delta_ub 0.036448982s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841e800 con 0x558c552d1800 2019-12-07T19:34:24.944+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e47 up_from 13 ping_stamp 2019-12-07T19:34:24.945114+0000/48.416833322s send_stamp 48.380751342s delta_ub 18446744073.673469636s) v5 -- 0x558c57518600 con 0x558c552d1c00 2019-12-07T19:34:24.944+0000 7f8f24457700 10 osd.7 47 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 45 -> 47 (as per caller) 2019-12-07T19:34:24.944+0000 7f8f24c58700 20 osd.7 47 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036057807s,0.036448982s]) 2019-12-07T19:34:24.944+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e47 up_from 13 ping_stamp 2019-12-07T19:34:24.945114+0000/48.416833322s send_stamp 48.380775515s delta_ub 18446744073.673493809s) v5 -- 0x558c5841ec00 con 0x558c552d1800 2019-12-07T19:34:24.970+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.020+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.070+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.120+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.170+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.220+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.271+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.321+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.371+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.380+0000 7f8f21451700 10 osd.7 47 tick 2019-12-07T19:34:25.380+0000 7f8f21451700 10 osd.7 47 do_waiters -- start 2019-12-07T19:34:25.380+0000 7f8f21451700 10 osd.7 47 do_waiters -- finish 2019-12-07T19:34:25.380+0000 7f8f21451700 20 osd.7 47 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:25.421+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.453+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 15 ==== osd_ping(ping e47 up_from 13 ping_stamp 2019-12-07T19:34:25.454679+0000/48.912191653s send_stamp 48.912191653s delta_ub 0.022416728s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b5e00 con 0x558c54c18000 2019-12-07T19:34:25.453+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 15 ==== osd_ping(ping e47 up_from 13 ping_stamp 2019-12-07T19:34:25.454679+0000/48.912191653s send_stamp 48.912191653s delta_ub 0.022416728s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518a00 con 0x558c552d1400 2019-12-07T19:34:25.453+0000 7f8f23c56700 20 osd.7 47 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021910456s,0.022416728s]) 2019-12-07T19:34:25.453+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e47 up_from 13 ping_stamp 2019-12-07T19:34:25.454679+0000/48.912191653s send_stamp 48.890281197s delta_ub 18446744073.687641160s) v5 -- 0x558c576b4400 con 0x558c54c18000 2019-12-07T19:34:25.453+0000 7f8f24457700 20 osd.7 47 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021888915s,0.022416728s]) 2019-12-07T19:34:25.453+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e47 up_from 13 ping_stamp 2019-12-07T19:34:25.454679+0000/48.912191653s send_stamp 48.890302738s delta_ub 18446744073.687662701s) v5 -- 0x558c57518600 con 0x558c552d1400 2019-12-07T19:34:25.471+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.522+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.523+0000 7f8f1219d700 20 osd.7 47 reports for 0 queries 2019-12-07T19:34:25.523+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c55051880 con 0x558c5476d800 2019-12-07T19:34:25.523+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:25.523+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0xb50000/0x9bf000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:25.523+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(3 pgs tid 0 v 0) v2 -- 0x558c55051c00 con 0x558c5476d800 2019-12-07T19:34:25.571+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1237857 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.617+0000 7f8f0998c700 15 osd.7 47 enqueue_peering_evt 1.4s0 epoch_sent: 39 epoch_requested: 39 RenewLease 2019-12-07T19:34:25.617+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 RenewLease) prio 255 cost 10 e39) 2019-12-07T19:34:25.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 RenewLease) prio 255 cost 10 e39) queued 2019-12-07T19:34:25.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 39 epoch_requested: 39 RenewLease) prio 255 cost 10 e39) pg 0x558c548ab000 2019-12-07T19:34:25.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 39 epoch_requested: 39 RenewLease 2019-12-07T19:34:25.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] old_peering_msg reply_epoch 39 query_epoch 39 last_peering_reset 41 2019-12-07T19:34:25.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] discard old epoch_sent: 39 epoch_requested: 39 RenewLease 2019-12-07T19:34:25.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:25.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 21 ==== osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8 ==== 257+0+55 (crc 0 0 0) 0x558c575a02c0 con 0x558c54c9f000 2019-12-07T19:34:25.617+0000 7f8f24457700 15 osd.7 47 enqueue_op 0x558c578d34a0 prio 63 cost 55 latency 0.000020 epoch 48 osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8 2019-12-07T19:34:25.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8) prio 63 cost 55 e48) 2019-12-07T19:34:25.617+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 22 ==== osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8 ==== 225+0+0 (crc 0 0 0) 0x558c575a0580 con 0x558c54c9f000 2019-12-07T19:34:25.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.617+0000 7f8f24457700 15 osd.7 47 enqueue_op 0x558c578d3600 prio 63 cost 0 latency 0.000006 epoch 48 osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8 2019-12-07T19:34:25.617+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) 2019-12-07T19:34:25.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8) prio 63 cost 55 e48) queued 2019-12-07T19:34:25.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8) prio 63 cost 55 e48) pg 0x558c57676000 2019-12-07T19:34:25.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) queued 2019-12-07T19:34:25.617+0000 7f8f06986700 10 osd.7 47 dequeue_op 0x558c578d34a0 prio 63 cost 55 latency 0.000050 osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.617+0000 7f8f06986700 10 osd.7 47 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 42 -> 48 (as per caller) 2019-12-07T19:34:25.617+0000 7f8f06986700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_request min 48, queue on waiting_for_map client.4540 2019-12-07T19:34:25.617+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=48}) v3 -- 0x558c58332a00 con 0x558c5476d000 2019-12-07T19:34:25.617+0000 7f8f06986700 10 osd.7 47 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:25.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.618+0000 7f8f0297e700 10 osd.7 47 dequeue_op 0x558c578d3600 prio 63 cost 0 latency 0.000150 osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:34:25.618+0000 7f8f0297e700 10 osd.7 47 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:25.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.618+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 29 ==== osd_map(48..48 src has 1..48) v4 ==== 633+0+0 (crc 0 0 0) 0x558c57324d80 con 0x558c5476d000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 osd.7 47 OSD::ms_dispatch: osd_map(48..48 src has 1..48) v4 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 osd.7 47 do_waiters -- start 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 osd.7 47 do_waiters -- finish 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 osd.7 47 _dispatch 0x558c57324d80 osd_map(48..48 src has 1..48) v4 2019-12-07T19:34:25.618+0000 7f8f164bb700 3 osd.7 47 handle_osd_map epochs [48,48], i have 47, src has [1,48] 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 osd.7 47 handle_osd_map got inc map for epoch 48 2019-12-07T19:34:25.618+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:39a8e826:::osdmap.47:0# 0x0~0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2bd0 blob([0xa42000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c550b31f0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757fea0 blob([!~1000,0xa43000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57f911f0 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:39a8e826:::osdmap.47:0# 0x0~1464 = 5220 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 osd.7 47 add_map_bl 47 5220 bytes 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 osd.7 47 got_full_map 48, nothing requested 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 osd.7 47 handle_osd_map pg_num_history pg_num_history(e48 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {48={}} 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 osd.7 47 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e48 [1,48] lci=[13,48]) 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c582b6000 seq 39 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:ca37972f:::inc_osdmap.48:0# key 0x7f7fffffffffffffffca3797'/!inc_osdmap.48!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ca37972f:::inc_osdmap.48:0# 0x0~24d 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1233 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:ca37972f:::inc_osdmap.48:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6000 1 blobs 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa44000~1000/1000,1000,0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa44000~1000] 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b2310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b2070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b2310 blob([0xa44000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2070 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c550b2310) get_ref 0x0~24d Blob(0x558c550b2310 blob([0xa44000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2070 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c550b2310) get_ref init 0x1000, 1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c550b2310 blob([0xa44000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c550b2070 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c550b2088 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738da08) dirty_range mark inline shard dirty 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ca37972f:::inc_osdmap.48:0# 0x0~24d = 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:34a8e826:::osdmap.48:0# key 0x7f7fffffffffffffff34a8e8'&!osdmap.48!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:34a8e826:::osdmap.48:0# 0x0~1448 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1234 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:34a8e826:::osdmap.48:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6000 2 blobs 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa45000~2000/1000,2000,0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa45000~2000] 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060a80 blob([0xa45000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d340 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c55060a80) get_ref 0x0~1000 Blob(0x558c55060a80 blob([0xa45000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d340 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c55060a80) get_ref init 0x1000, 1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c55060a80 blob([0xa45000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800d340 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5800d358 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b2540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54677110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b2540 blob([!~1000,0xa46000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677110 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c550b2540) get_ref 0x1000~448 Blob(0x558c550b2540 blob([!~1000,0xa46000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c54677110 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c550b2540) get_ref init 0x2000, 1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c550b2540 blob([!~1000,0xa46000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c54677110 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54677128 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57f35788) dirty_range mark inline shard dirty 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:34a8e826:::osdmap.48:0# 0x0~1448 = 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6000 0 blobs 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6000 0 blobs 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b6000 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b6000 onodes 0x558c53a91900,0x558c55276c80,0x558c5738d900,0x558c57f35680 shared_blobs 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738da08) update #-1:ca37972f:::inc_osdmap.48:0# 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5738da08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:ca37972f:::inc_osdmap.48:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57f35788) update #-1:34a8e826:::osdmap.48:0# 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57f35788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:34a8e826:::osdmap.48:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b6000 allocated 0x[a44000~3000] released 0x[] 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 freelist allocate 0xa44000~3000 2019-12-07T19:34:25.618+0000 7f8f164bb700 20 freelist _xor first_key 0xa00000 last_key 0xa00000 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6000 prepare 2019-12-07T19:34:25.618+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c582b6000 2019-12-07T19:34:25.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6000 aio_wait 2019-12-07T19:34:25.618+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b6000 2019-12-07T19:34:25.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6000 io_done 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738d900 had 1 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35680 had 1 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3858366464 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998152) bluestore 86 GiB free (0.959888), bluefs_ratio 0.0399339 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8848 < min_flush_size 524288 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8848 < min_flush_size 524288 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x258855~2290 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x258855 0x258855~2290 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x228855 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x855 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xae5 and padding block with 0x51b 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x25aae5 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:25.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000199763s (0.000045056s flush + 0.000154707s kv commit) 2019-12-07T19:34:25.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b6000> 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c57f53a40> 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6000 kv_submitted 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b6000 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c582b6000 osr 0x558c54ea4b40 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7 47 _committed_osd_maps 48..48 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7 47 advance to epoch 48 (<= last 48 <= newest_map 48) 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 37 0x1b000~1000 crc 2595bf2c 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 37 0x2000~1000 crc cdbc6182 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2300 deferred_cleanup 2019-12-07T19:34:25.618+0000 7f8f141a1700 7 osd.7 48 consume_map version 48 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d2300 onodes 0x558c53a91900,0x558c55276c80,0x558c574a0780,0x558c57d9a780 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.8s5 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.0s3 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.1s1 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39198 in 0x558c53a59340) _finish_write discard buffer(0x558c579fbef0 space 0x558c57c39198 0x0~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.9s4 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39278 in 0x558c53a59340) _finish_write discard buffer(0x558c579fb710 space 0x558c57c39278 0x1000~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.as5 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.2s0 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91278 in 0x558c53a59340) _finish_write discard buffer(0x558c5813a480 space 0x558c57f91278 0x0~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.bs1 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.3s4 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2300 done 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.cs0 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 deferred_cleanup 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.4s0 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.5s4 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d2300 [] 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.ds2 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.es4 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.6s2 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.fs5 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 identify_splits_and_merges 1.7s4 e47 to e48 pg_nums {18=16} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 48 prune_pg_created 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:0.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:1.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54d58c00 deferred_cleanup 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:2.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:3.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54d58c00 onodes 0x558c53a91900,0x558c55276c80,0x558c5738c000,0x558c5738c280 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:4.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df8558 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4fa70 space 0x558c54df8558 0x0~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:5.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:6.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54df9b38 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4fb00 space 0x558c54df9b38 0x1000~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7:7.consume_map 48 (was 47) 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff40f8 in 0x558c53a59340) _finish_write discard buffer(0x558c57f4f9e0 space 0x558c57ff40f8 0x0~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54d58c00 done 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829b200 deferred_cleanup 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54d58c00 [] 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:25.618+0000 7f8f141a1700 10 osd.7 48 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b200 deferred_cleanup 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5829b200 onodes 0x558c53a91900,0x558c55276c80,0x558c58092500,0x558c58093b80 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.8s5 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3208 in 0x558c53a59340) _finish_write discard buffer(0x558c579fb950 space 0x558c550b3208 0x0~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91208 in 0x558c53a59340) _finish_write discard buffer(0x558c579fb9e0 space 0x558c57f91208 0x1000~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.0s3 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91438 in 0x558c53a59340) _finish_write discard buffer(0x558c579fb8c0 space 0x558c57f91438 0x0~1000 writing nocache) 2019-12-07T19:34:25.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829b200 done 2019-12-07T19:34:25.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6000 deferred_queued 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5829b200 [] 2019-12-07T19:34:25.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:25.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c57380000 2019-12-07T19:34:25.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.1s1 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.9s4 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c5500d000 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.as5 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f04982700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.618+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 47 -> 48 2019-12-07T19:34:25.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.2s0 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.bs1 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c5736a000 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f04982700 10 osd.7 pg_epoch: 47 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:25.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.3s4 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 47 -> 48 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 47 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:25.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c575ef000 2019-12-07T19:34:25.618+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:25.618+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.cs0 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.618+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.618+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.4s0 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.5s4 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.619+0000 7f8f04982700 20 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.ds2 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f04982700 20 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f04982700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.es4 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c57e8b000 2019-12-07T19:34:25.619+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 47 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:25.619+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.6s2 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.fs5 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f0898a700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:25.619+0000 7f8f04982700 20 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f141a1700 15 osd.7 48 enqueue_peering_evt 1.7s4 epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:25.619+0000 7f8f141a1700 7 osd.7 48 activate_map version 48 2019-12-07T19:34:25.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c57676000 2019-12-07T19:34:25.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f0317f700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c57522000 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c5513e000 2019-12-07T19:34:25.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f0317f700 20 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 47 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c5500a000 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 47 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c57383000 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 47 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c57e88000 2019-12-07T19:34:25.619+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:25.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:25.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c55141000 2019-12-07T19:34:25.619+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f06986700 7 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 47 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f06986700 15 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:149 2019-12-07T19:34:25.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 47 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c54d5f000 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] requeue_map_waiters client.4540 0x558c578d34a0,0x558c578d3600 2019-12-07T19:34:25.619+0000 7f8f04181700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8) prio 63 cost 55 e48) 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 47 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c548ab000 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8) prio 63 cost 55 e48) queued 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8) prio 63 cost 55 e48) pg 0x558c57676000 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d34a0 prio 63 cost 55 latency 0.001387 osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0918b700 20 osd.7 48 agent_entry empty queue 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:25.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 47 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: osd_op(client.4540.0:380 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e48) v8 2019-12-07T19:34:25.619+0000 7f8f03980700 7 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op: op osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) v8 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f03980700 15 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:134 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:25.619+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 0=[]:{} 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c54d98000 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f06986700 25 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0)) 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] execute_ctx 0x558c55054300 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] ov 21'11 av 48'31 snapc 19=[19,15,13,e] snapset 0=[]:{} 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:25.619+0000 7f8f0297e700 7 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_osd_op copy-from ver 21 in=55b 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] start_copy 1:33f5bf01:::smithi13140796-2:head from 1:db85c185:::smithi13140796-18:head @1 v21 flags 0 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _copy_some obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0)) 0x558c54c5a010 2019-12-07T19:34:25.619+0000 7f8f0297e700 15 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:120 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) queued 2019-12-07T19:34:25.619+0000 7f8f06986700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6808/36938,v1:172.21.15.109:6809/36938] conn(0x558c58146c00 0x558c576bf700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f06986700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6808/36938,v1:172.21.15.109:6809/36938] -- osd_op(unknown.0.13:3 1.bs0 1:db85c185:::smithi13140796-18:head [assert-version v21,copy-get max 8388608 in=29b] snapc 0=[] ondisk+read+known_if_redirected e48) v8 -- 0x558c582b5340 con 0x558c58146c00 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) queued 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 48 epoch_requested: 48 NullEvt) prio 255 cost 10 e48) pg 0x558c54d5c000 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:25.619+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d34a0 finish 2019-12-07T19:34:25.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 48 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 47 -> 48 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 47 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c578d3600 prio 63 cost 0 latency 0.001542 osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8 pg pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: osd_op(client.4540.0:381 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e48) v8 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:25.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op: op osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 48 advance_pg new pool opts old pool opts 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 48 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 48 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 48 epoch_requested: 48 NullEvt 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] wait_for_blocked_object 1:33f5bf01:::smithi13140796-2:head 0x558c578d3600 2019-12-07T19:34:25.619+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:25.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:25.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.619+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6808/36938,v1:172.21.15.109:6809/36938] conn(0x558c58146c00 0x558c576bf700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:25.619+0000 7f8f24457700 1 --2- 172.21.15.109:0/36939 >> [v2:172.21.15.109:6808/36938,v1:172.21.15.109:6809/36938] conn(0x558c58146c00 0x558c576bf700 crc :-1 s=READY pgs=25 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:25.620+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 242 ==== osd_map(47..48 src has 1..48) v4 ==== 1258+0+0 (unknown 0 0 0) 0x558c576c0b40 con 0x558c54c5b400 2019-12-07T19:34:25.620+0000 7f8f1199c700 20 osd.7 48 OSD::ms_dispatch: osd_map(47..48 src has 1..48) v4 2019-12-07T19:34:25.620+0000 7f8f1199c700 10 osd.7 48 do_waiters -- start 2019-12-07T19:34:25.620+0000 7f8f1199c700 10 osd.7 48 do_waiters -- finish 2019-12-07T19:34:25.620+0000 7f8f1199c700 20 osd.7 48 _dispatch 0x558c576c0b40 osd_map(47..48 src has 1..48) v4 2019-12-07T19:34:25.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 243 ==== MOSDECSubOpRead(1.bs1 48/41 ECSubRead(tid=94, to_read={1:db85c185:::smithi13140796-18:head=0,405504,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c58294280 con 0x558c54c5b400 2019-12-07T19:34:25.620+0000 7f8f1199c700 3 osd.7 48 handle_osd_map epochs [47,48], i have 48, src has [1,48] 2019-12-07T19:34:25.620+0000 7f8f1199c700 10 osd.7 48 no new maps here, dropping 2019-12-07T19:34:25.620+0000 7f8f23c56700 15 osd.7 48 enqueue_op 0x558c57efec60 prio 127 cost 0 latency 0.000019 epoch 48 MOSDECSubOpRead(1.bs1 48/41 ECSubRead(tid=94, to_read={1:db85c185:::smithi13140796-18:head=0,405504,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:25.620+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 48/41 ECSubRead(tid=94, to_read={1:db85c185:::smithi13140796-18:head=0,405504,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e48) 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 48/41 ECSubRead(tid=94, to_read={1:db85c185:::smithi13140796-18:head=0,405504,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e48) queued 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 48/41 ECSubRead(tid=94, to_read={1:db85c185:::smithi13140796-18:head=0,405504,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e48) pg 0x558c575ef000 2019-12-07T19:34:25.620+0000 7f8f0317f700 10 osd.7 48 dequeue_op 0x558c57efec60 prio 127 cost 0 latency 0.000052 MOSDECSubOpRead(1.bs1 48/41 ECSubRead(tid=94, to_read={1:db85c185:::smithi13140796-18:head=0,405504,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.620+0000 7f8f0317f700 10 osd.7 48 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 46 -> 48 (as per caller) 2019-12-07T19:34:25.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpRead(1.bs1 48/41 ECSubRead(tid=94, to_read={1:db85c185:::smithi13140796-18:head=0,405504,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:25.620+0000 7f8f0317f700 25 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:25.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~63000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~63000 size 0x63000 (405504) 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552796c0 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f49b90 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57a06850 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f49ab0 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f499d0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f498f0 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578812d0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f3bab0 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c540 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137340 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551373b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137420 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137500 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551375e0 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551376c0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551377a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137880 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551378f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137960 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551379d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137a40 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137ab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137b20 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137b90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b39d0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3ab0 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3b20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3b90 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3c70 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3d50 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3dc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3e30 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3f10 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3f80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55137110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3030 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b30a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d67e0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6850 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d68c0 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6930 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d69a0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6a10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d6a80 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d6b60 blob([0x973000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6bd0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d6c40 blob([0x975000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6cb0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d6d20 blob([0x977000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c423f0 blob([0x979000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55136fc0 blob([0x97b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55136ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579710a0 blob([0x97d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581eab60 blob([0x97f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581eaa80 blob([0x981000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581ea9a0 blob([0x983000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581ea8c0 blob([0x985000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581ea7e0 blob([0x987000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adf10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581ea700 blob([0x989000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581ea620 blob([0x98b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581ea540 blob([0x98d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c581ea460 blob([0x98f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57970150 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b3180 blob([0x990000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6770 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b20e0 blob([0x992000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3030 blob([0x994000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3030 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b30a0 sbid 0x0)) need {<0x0, 0x2000> : [0x36000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x36000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3180 blob([0x990000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6770 sbid 0x0)) need {<0x0, 0x2000> : [0x5d000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5d000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b39d0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3a40 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3ab0 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3b20 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3b90 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3c00 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3c70 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ce0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3d50 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3dc0 sbid 0x0)) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3e30 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ea0 sbid 0x0)) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b3f10 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3f80 sbid 0x0)) need {<0x0, 0x2000> : [0x32000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55136fc0 blob([0x97b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55136ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x48000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x48000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2fc0 sbid 0x0)) need {<0x0, 0x2000> : [0x34000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x34000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137340 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551373b0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137420 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137490 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137500 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137570 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c551375e0 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137650 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c551376c0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137730 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c551377a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137810 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137880 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551378f0 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137960 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551379d0 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137a40 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137ab0 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55137b20 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137b90 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c552796c0 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578812d0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c540 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579710a0 blob([0x97d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3c00 sbid 0x0)) need {<0x0, 0x2000> : [0x4a000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4a000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b20e0 blob([0x992000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3110 sbid 0x0)) need {<0x0, 0x2000> : [0x5f000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5f000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3030 blob([0x994000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) need {<0x0, 0x2000> : [0x61000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x61000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57a06850 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57c423f0 blob([0x979000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2690 sbid 0x0)) need {<0x0, 0x2000> : [0x46000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x46000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f3bab0 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f498f0 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f499d0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f49ab0 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f49b90 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d67e0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6850 sbid 0x0)) need {<0x0, 0x2000> : [0x38000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x38000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d68c0 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6930 sbid 0x0)) need {<0x0, 0x2000> : [0x3a000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3a000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d69a0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6a10 sbid 0x0)) need {<0x0, 0x2000> : [0x3c000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3c000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d6a80 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6af0 sbid 0x0)) need {<0x0, 0x2000> : [0x3e000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3e000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d6b60 blob([0x973000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6bd0 sbid 0x0)) need {<0x0, 0x2000> : [0x40000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x40000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d6c40 blob([0x975000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6cb0 sbid 0x0)) need {<0x0, 0x2000> : [0x42000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x42000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d6d20 blob([0x977000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6d90 sbid 0x0)) need {<0x0, 0x2000> : [0x44000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x44000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581ea460 blob([0x98f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57970150 sbid 0x0)) need {<0x0, 0x1000> : [0x5c000:0~1000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5c000: 0x0 reading 0x0~1000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581ea540 blob([0x98d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971c00 sbid 0x0)) need {<0x0, 0x2000> : [0x5a000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x5a000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581ea620 blob([0x98b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad730 sbid 0x0)) need {<0x0, 0x2000> : [0x58000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x58000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581ea700 blob([0x989000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) need {<0x0, 0x2000> : [0x56000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x56000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581ea7e0 blob([0x987000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adf10 sbid 0x0)) need {<0x0, 0x2000> : [0x54000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x54000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581ea8c0 blob([0x985000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need {<0x0, 0x2000> : [0x52000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x52000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581ea9a0 blob([0x983000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1960 sbid 0x0)) need {<0x0, 0x2000> : [0x50000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x50000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581eaa80 blob([0x981000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) need {<0x0, 0x2000> : [0x4e000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4e000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c581eab60 blob([0x97f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adea0 sbid 0x0)) need {<0x0, 0x2000> : [0x4c000:0~2000]} 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4c000: 0x0 reading 0x0~2000 2019-12-07T19:34:25.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3030 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b30a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x36000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b30b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57b8e870 space 0x558c550b30b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3180 blob([0x990000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6770 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x5d000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57b8e900 space 0x558c580d6788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b39d0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57b8e990 space 0x558c550b3a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3ab0 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3b20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57b8ea20 space 0x558c550b3b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3b90 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57b8eab0 space 0x558c550b3c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3c70 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3cf8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57b8eb40 space 0x558c550b3cf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3d50 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3dc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3dd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3200 space 0x558c550b3dd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3e30 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3ea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3eb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3290 space 0x558c550b3eb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b3f10 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3f80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x32000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b3f98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3320 space 0x558c550b3f98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55136fc0 blob([0x97b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55136ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x48000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55136ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c33b0 space 0x558c55136ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x34000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c550b2fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3440 space 0x558c550b2fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137340 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551373b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551373c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c34d0 space 0x558c551373c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137420 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551374a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3560 space 0x558c551374a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137500 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137588 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c35f0 space 0x558c55137588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c551375e0 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137668 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3680 space 0x558c55137668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c551376c0 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137748 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3710 space 0x558c55137748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c551377a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c37a0 space 0x558c55137828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137880 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551378f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137908 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3830 space 0x558c55137908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137960 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551379d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551379e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c38c0 space 0x558c551379e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137a40 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137ab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137ac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3950 space 0x558c55137ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55137b20 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55137b90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55137ba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c39e0 space 0x558c55137ba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c552796c0 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57a07e48 in 0x558c53a59c00) _discard 0x1000~1000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3a70 space 0x558c57a07e48 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578812d0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57c38a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3b00 space 0x558c57c38a28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c540 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5809b2e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3b90 space 0x558c5809b2e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579710a0 blob([0x97d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4a000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3c20 space 0x558c579b3c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b20e0 blob([0x992000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x5f000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3128 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3cb0 space 0x558c579b3128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3030 blob([0x994000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b30a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x61000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b30b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3d40 space 0x558c579b30b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57a06850 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57f49b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3dd0 space 0x558c57f49b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57c423f0 blob([0x979000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x46000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c577f26a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3e60 space 0x558c577f26a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f3bab0 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57971eb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c586c3ef0 space 0x558c57971eb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f498f0 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57f49898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba000 space 0x558c57f49898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f499d0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57f49978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba090 space 0x558c57f49978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f49ab0 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57f49a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba120 space 0x558c57f49a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f49b90 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57a075f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba1b0 space 0x558c57a075f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d67e0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6850 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x38000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba240 space 0x558c580d6868 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d68c0 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6930 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3a000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba2d0 space 0x558c580d6948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d69a0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6a10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3c000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba360 space 0x558c580d6a28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d6a80 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3e000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6b08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba3f0 space 0x558c580d6b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d6b60 blob([0x973000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6bd0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x40000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6be8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba480 space 0x558c580d6be8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d6c40 blob([0x975000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6cb0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x42000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6cc8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba510 space 0x558c580d6cc8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d6d20 blob([0x977000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d6d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x44000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580d6da8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba5a0 space 0x558c580d6da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581ea460 blob([0x98f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57970150 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x5c000:0~1000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57970168 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba630 space 0x558c57970168 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581ea540 blob([0x98d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x5a000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57971c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba6c0 space 0x558c57971c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581ea620 blob([0x98b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x58000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad748 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba750 space 0x558c576ad748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581ea700 blob([0x989000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x56000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba7e0 space 0x558c576ad828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581ea7e0 blob([0x987000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adf10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x54000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adf28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba870 space 0x558c576adf28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581ea8c0 blob([0x985000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x52000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57971198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba900 space 0x558c57971198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581ea9a0 blob([0x983000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x50000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574f1978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582ba990 space 0x558c574f1978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581eaa80 blob([0x981000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f11f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4e000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574f1208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582baa20 space 0x558c574f1208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c581eab60 blob([0x97f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4c000:0~2000]} 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576adeb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582baab0 space 0x558c576adeb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~63000 = 405504 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_sub_read read request=405504 r=405504 len=405504 2019-12-07T19:34:25.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] get_hash_info: Getting attr on 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:25.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] get_hash_info: not in cache 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:25.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:25.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:25.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key 2019-12-07T19:34:25.621+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1248541 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key = 0 2019-12-07T19:34:25.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_sub_read: Checking hash of 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:25.621+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 48/41 ECSubReadReply(tid=94, attrs_read=0)) v2 -- 0x558c55113400 con 0x558c54c5b400 2019-12-07T19:34:25.621+0000 7f8f0317f700 10 osd.7 48 dequeue_op 0x558c57efec60 finish 2019-12-07T19:34:25.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:25.623+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6808/36938 1 ==== osd_op_reply(3 smithi13140796-18 [assert-version v21,copy-get max 8388608 out=1622405b] v0'0 uv21 ondisk = 1622016) v8 ==== 203+0+1622405 (crc 0 0 0) 0x558c57573900 con 0x558c58146c00 2019-12-07T19:34:25.623+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] process_copy_chunk 1:33f5bf01:::smithi13140796-2:head tid 3 (0) Success 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] process_copy_chunk got digest: rx data 0x15460b02 omap 0xffffffff, source: data 0x15460b02 omap 0xffffffff flags 1 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] process_copy_chunk success; committing 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] process_copy_chunk complete r = (0) Success 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] execute_ctx 0x558c55054300 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] ov 21'11 av 48'31 snapc 19=[19,15,13,e] snapset 0=[]:{} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_osd_op copy-from ver 21 in=55b 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] finish_copyfrom on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] finish_copyfrom: exists, removing 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] fill_in_final_tx: writing directly to final object 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _write_copy_chunk 0x558c54c5a010 0 attrs 1622016 bytes 0 omap header bytes 0 omap data bytes 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] finish_copyfrom clearing omap flag on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=0=[]:{} snapc=19=[19,15,13,e] 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] got greedy write on clone_obc obc(1:33f5bf01:::smithi13140796-2:19 rwstate(write n=1 w=0)) 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] cloning v 21'11 to 1:33f5bf01:::smithi13140796-2:19 v 48'31 snaps=[19,15,13,e] snapset=0=[]:{19=[19,15,13,e]} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=19=[]:{19=[19,15,13,e]} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x558c55054300 op modify 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] mtime unchanged at 2019-12-07T19:34:21.628654+0000 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] final snapset 19=[]:{19=[19,15,13,e]} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] finish_ctx extra_reqids [client.4540.0:360,21,client.4540.0:359,20,client.4540.0:358,19,client.4540.0:357,18,client.4540.0:53,2,client.4540.0:52,1] {} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] execute_ctx alloc reply 0x558c58092280 result 0 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] calc_trim_to_aggressive limit = 45'30 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] op order client.4540 tid 380 (first) 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] new_repop rep_tid 36 on osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) v8 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] new_repop: repgather(0x558c576b6d80 0'0 rep_tid=36 committed?=0 r=0) 2019-12-07T19:34:25.624+0000 7f8f0b18f700 7 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] issue_repop rep_tid 36 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=48'32 tt=0'0 tid=36 reqid=client.4540.0:380 client_op=osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:19# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0x0000000000000019ffffffffffffffff'o' 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:25.624+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 projected size 901120 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=48'32 tt=0'0 tid=36 reqid=client.4540.0:380 client_op=osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[0~1622016]}) 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_state_to_reads: invalidating cache after this op 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=48'32 tt=0'0 tid=36 reqid=client.4540.0:380 client_op=osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[0~1622016]}) 2019-12-07T19:34:25.624+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=48'32 tt=0'0 tid=36 reqid=client.4540.0:380 client_op=osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[0~1622016]}) 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~1622016:7) ) 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:19, current size is 0 buffers are {} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): new_size start 901120 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): to_overwrite: {} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): to_append: {} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 resetting hinfo to logical size 901120 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 901120 buffers are {0~1622016(1622016)} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): new_size start 0 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): adding buffer_update 0,1622016 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): to_overwrite: {} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): to_append: {0~1622016(1622016)} 2019-12-07T19:34:25.624+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): appending 0~1622016 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] encode_and_write: 1:33f5bf01:::smithi13140796-2:head new_size 1622016 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] operator(): marking append 0 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~1622016:7) ) 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:19={},1:33f5bf01:::smithi13140796-2:head={0~1622016(1622016)}} 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=48'32 tt=0'0 tid=36 reqid=client.4540.0:380 client_op=osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[0~1622016]}) 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[0~1622016]} 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:19,{} 2019-12-07T19:34:25.627+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{0~1622016(1622016)} 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 send_incremental_map 45 -> 48 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:25.628+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cff7972f:::inc_osdmap.46:0# 0x0~0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~289 size 0x289 (649) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55060a10 blob([0x9eb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x289) SharedBlob(0x558c57ff40e0 sbid 0x0)) need 0x0~289 cache has 0x[] 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55060a10 blob([0x9eb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x289) SharedBlob(0x558c57ff40e0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~289]} 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55060a10 blob([0x9eb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x289) SharedBlob(0x558c57ff40e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~289]} 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57ff40f8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 _add level 1 near 0 on buffer(0x558c582bb200 space 0x558c57ff40f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cff7972f:::inc_osdmap.46:0# 0x0~289 = 649 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 add_map_inc_bl 46 649 bytes 2019-12-07T19:34:25.628+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c137972f:::inc_osdmap.47:0# 0x0~0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d7ab0 blob([0xa41000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91420 sbid 0x0)) need 0x0~269 cache has 0x[] 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580d7ab0 blob([0xa41000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91420 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~269]} 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580d7ab0 blob([0xa41000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~269]} 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c57f91438 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 _add level 1 near 0 on buffer(0x558c582bb290 space 0x558c57f91438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c137972f:::inc_osdmap.47:0# 0x0~269 = 617 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 add_map_inc_bl 47 617 bytes 2019-12-07T19:34:25.628+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ca37972f:::inc_osdmap.48:0# 0x0~0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2310 blob([0xa44000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c550b2070 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ca37972f:::inc_osdmap.48:0# 0x0~24d = 589 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 add_map_inc_bl 48 589 bytes 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(46..48 src has 1..48) v4 -- 0x558c54d2efc0 con 0x558c54c64000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 45 -> 48 (shared) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 48/28 ECSubWrite(tid=36, reqid=client.4540.0:380, at_version=48'32, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c54e40800 con 0x558c54c64000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 send_incremental_map 47 -> 48 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(48..48 src has 1..48) v4 -- 0x558c5781c6c0 con 0x558c54c19800 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 47 -> 48 (shared) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 48/28 ECSubWrite(tid=36, reqid=client.4540.0:380, at_version=48'32, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c54e3e800 con 0x558c54c19800 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 send_incremental_map 47 -> 48 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(48..48 src has 1..48) v4 -- 0x558c576b3200 con 0x558c54c5bc00 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 47 -> 48 (shared) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 48/28 ECSubWrite(tid=36, reqid=client.4540.0:380, at_version=48'32, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c54e41800 con 0x558c54c5bc00 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 48/28 ECSubWrite(tid=36, reqid=client.4540.0:380, at_version=48'32, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c54e3f000 con 0x558c54c5b400 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 send_incremental_map 45 -> 48 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(46..48 src has 1..48) v4 -- 0x558c57e03d40 con 0x558c54c19c00 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 48 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 45 -> 48 (shared) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 48/28 ECSubWrite(tid=36, reqid=client.4540.0:380, at_version=48'32, trim_to=0'0, roll_forward_to=45'30)) v2 -- 0x558c574a2000 con 0x558c54c19c00 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.add_oid 1:33f5bf01:::smithi13140796-2:19 e,13,15,19 2019-12-07T19:34:25.628+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.get_snaps 1:33f5bf01:::smithi13140796-2:19 got.empty() 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.set_snaps 1:33f5bf01:::smithi13140796-2:19 e,13,15,19 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 snap_mapper.add_oid set SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 earliest_dup_version = 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 trim 48'31 (21'11) clone 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [19,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 trim 48'32 (21'11) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:380 0.000000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 45'30 (0'0,45'30] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] append_log log((0'0,45'30], crt=45'30) [48'31 (21'11) clone 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [19,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,48'32 (21'11) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:380 0.000000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 1, new_object: 0] 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'31 (0'0,48'31] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 lua=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] add_log_entry 48'31 (21'11) clone 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [19,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 lua=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] add_log_entry 48'32 (21'11) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:380 0.000000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 lua=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] append_log approx pg log length = 32 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 lua=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] append_log transaction_applied = 1 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 48'31, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57ef8f00 seq 43 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:19# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0x0000000000000019ffffffffffffffff'o' 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1235 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.onode(0x558c574a1400).flush flush done 2019-12-07T19:34:25.628+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 0x0 2019-12-07T19:34:25.628+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:19# 0x0~37000 -> 0x0~37000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x0~2000: 0x0~2000 Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10566 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57396380 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57396380 blob([0x2a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78700 blob([0x2a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78700) get_ref 0x0~2000 Blob(0x558c58b78700 blob([0x2a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78700) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x0~2000: 0x0~2000 Blob(0x558c58b78700 blob([0x2a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10567 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57396460 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57396460 blob([0x2a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78690 blob([0x2a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78690) get_ref 0x0~2000 Blob(0x558c58b78690 blob([0x2a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78690) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c58b78690 blob([0x2a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x4000~2000: 0x0~2000 Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10568 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578804d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578804d0 blob([0x2aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b785b0 blob([0x2aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b785b0) get_ref 0x0~2000 Blob(0x558c58b785b0 blob([0x2aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b785b0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c58b785b0 blob([0x2aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x6000~2000: 0x0~2000 Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10569 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578805b0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578805b0 blob([0x2ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78620 blob([0x2ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78620) get_ref 0x0~2000 Blob(0x558c58b78620 blob([0x2ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78620) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c58b78620 blob([0x2ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10570 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57880690 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57880690 blob([0x2ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78770 blob([0x2ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78770) get_ref 0x0~2000 Blob(0x558c58b78770 blob([0x2ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78770) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c58b78770 blob([0x2ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10571 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57880770 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57880770 blob([0x2b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b787e0 blob([0x2b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b787e0) get_ref 0x0~2000 Blob(0x558c58b787e0 blob([0x2b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b787e0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c58b787e0 blob([0x2b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10572 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57880850 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57880850 blob([0x2b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b788c0 blob([0x2b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b788c0) get_ref 0x0~2000 Blob(0x558c58b788c0 blob([0x2b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b788c0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c58b788c0 blob([0x2b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10573 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57880930 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57880930 blob([0x2b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78930 blob([0x2b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78930) get_ref 0x0~2000 Blob(0x558c58b78930 blob([0x2b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78930) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c58b78930 blob([0x2b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x10000~2000: 0x0~2000 Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10574 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c574d3340 blob([0x350000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c574d3340 blob([0x350000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b789a0 blob([0x350000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b789a0) get_ref 0x0~2000 Blob(0x558c58b789a0 blob([0x350000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b789a0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c58b789a0 blob([0x350000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10575 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57880d20 blob([0x352000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57880d20 blob([0x352000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78a10 blob([0x352000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78a10) get_ref 0x0~2000 Blob(0x558c58b78a10 blob([0x352000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78a10) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c58b78a10 blob([0x352000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10576 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57880c40 blob([0x354000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57880c40 blob([0x354000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=1)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78a80 blob([0x354000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78a80) get_ref 0x0~2000 Blob(0x558c58b78a80 blob([0x354000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78a80) get_ref init 0x2000, 1000 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c58b78a80 blob([0x354000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=2)))) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x16000~2000: 0x0~2000 Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10577 2019-12-07T19:34:25.628+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c54d75420 blob([0x356000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 sbid 0x0)) 2019-12-07T19:34:25.628+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c54d75420 blob([0x356000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78af0 blob([0x356000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78af0) get_ref 0x0~2000 Blob(0x558c58b78af0 blob([0x356000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78af0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c58b78af0 blob([0x356000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x18000~2000: 0x0~2000 Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10578 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c575bd8f0 blob([0x358000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c575bd8f0 blob([0x358000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78b60 blob([0x358000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78b60) get_ref 0x0~2000 Blob(0x558c58b78b60 blob([0x358000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78b60) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c58b78b60 blob([0x358000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10579 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578de540 blob([0x35a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578de540 blob([0x35a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78bd0 blob([0x35a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78bd0) get_ref 0x0~2000 Blob(0x558c58b78bd0 blob([0x35a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c58b78bd0 blob([0x35a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10580 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578de380 blob([0x35c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578de380 blob([0x35c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78c40 blob([0x35c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78c40) get_ref 0x0~2000 Blob(0x558c58b78c40 blob([0x35c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78c40) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c58b78c40 blob([0x35c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10581 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578de1c0 blob([0x35e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578de1c0 blob([0x35e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78cb0 blob([0x35e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78cb0) get_ref 0x0~2000 Blob(0x558c58b78cb0 blob([0x35e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c58b78cb0 blob([0x35e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x20000~2000: 0x0~2000 Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10582 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578de000 blob([0x360000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578de000 blob([0x360000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78d20 blob([0x360000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78d20) get_ref 0x0~2000 Blob(0x558c58b78d20 blob([0x360000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78d20) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c58b78d20 blob([0x360000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10583 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57819e30 blob([0x362000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57819e30 blob([0x362000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78d90 blob([0x362000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78d90) get_ref 0x0~2000 Blob(0x558c58b78d90 blob([0x362000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78d90) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c58b78d90 blob([0x362000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x24000~2000: 0x0~2000 Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10584 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c54cf8620 blob([0x364000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c54cf8620 blob([0x364000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78e00 blob([0x364000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78e00) get_ref 0x0~2000 Blob(0x558c58b78e00 blob([0x364000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78e00) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c58b78e00 blob([0x364000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x26000~2000: 0x0~2000 Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10585 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c578f1f10 blob([0x366000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c578f1f10 blob([0x366000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78e70 blob([0x366000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78e70) get_ref 0x0~2000 Blob(0x558c58b78e70 blob([0x366000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78e70) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c58b78e70 blob([0x366000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x28000~2000: 0x0~2000 Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10586 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794c4d0 blob([0x368000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794c4d0 blob([0x368000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78ee0 blob([0x368000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78ee0) get_ref 0x0~2000 Blob(0x558c58b78ee0 blob([0x368000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c58b78ee0 blob([0x368000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10587 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794c3f0 blob([0x36a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794c3f0 blob([0x36a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78f50 blob([0x36a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78f50) get_ref 0x0~2000 Blob(0x558c58b78f50 blob([0x36a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78f50) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c58b78f50 blob([0x36a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10588 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794c310 blob([0x36c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794c310 blob([0x36c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b78fc0 blob([0x36c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78fc0) get_ref 0x0~2000 Blob(0x558c58b78fc0 blob([0x36c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b78fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c58b78fc0 blob([0x36c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10589 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794c1c0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794c1c0 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b79030 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79030) get_ref 0x0~2000 Blob(0x558c58b79030 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79030) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c58b79030 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x30000~2000: 0x0~2000 Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10590 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794c0e0 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794c0e0 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b790a0 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b790a0) get_ref 0x0~2000 Blob(0x558c58b790a0 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b790a0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c58b790a0 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x32000~2000: 0x0~2000 Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10591 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c5794cd90 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c5794cd90 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b79110 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79110) get_ref 0x0~2000 Blob(0x558c58b79110 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79110) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c58b79110 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x34000~2000: 0x0~2000 Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10592 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c574d3810 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c574d3810 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b79180 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79180) get_ref 0x0~2000 Blob(0x558c58b79180 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79180) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c58b79180 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup src 0x36000~1000: 0x0~1000 Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10593 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c574d3260 blob([0x376000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c574d3260 blob([0x376000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=1)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup new Blob(0x558c58b791f0 blob([0x376000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b791f0) get_ref 0x0~1000 Blob(0x558c58b791f0 blob([0x376000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b791f0) get_ref init 0x1000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dup dst 0x36000~1000: 0x0~1000 Blob(0x558c58b791f0 blob([0x376000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=2)))) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) dirty_range mark inline shard dirty 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58295288) dirty_range mark inline shard dirty 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:19# = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# snapset 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# snapset = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:head#20 key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0xfffffffffffffffe0000000000000020'o' 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:head#20 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:head#20 = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1236 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 3 keys 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 3 keys = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~63000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 0#1:33f5bf01:::smithi13140796-2:head# 0x0~63000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~63000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57ef8f00 50 blobs 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x63000/1000,63000,0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xa47000~63000/1000,63000,0 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xa47000~63000] 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b792d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b792d0 blob([0xa47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79340 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b792d0) get_ref 0x0~2000 Blob(0x558c58b792d0 blob([0xa47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79340 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b792d0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c58b792d0 blob([0xa47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79358 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b793b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b793b0 blob([0xa49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79420 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b793b0) get_ref 0x0~2000 Blob(0x558c58b793b0 blob([0xa49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79420 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b793b0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c58b793b0 blob([0xa49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79438 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79490 blob([0xa4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79500 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79490) get_ref 0x0~2000 Blob(0x558c58b79490 blob([0xa4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79500 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79490) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c58b79490 blob([0xa4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79518 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b795e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79570 blob([0xa4d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b795e0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79570) get_ref 0x0~2000 Blob(0x558c58b79570 blob([0xa4d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b795e0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79570) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c58b79570 blob([0xa4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b795f8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b796c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79650 blob([0xa4f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b796c0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79650) get_ref 0x0~2000 Blob(0x558c58b79650 blob([0xa4f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b796c0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79650) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c58b79650 blob([0xa4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b796d8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b797a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79730 blob([0xa51000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b797a0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79730) get_ref 0x0~2000 Blob(0x558c58b79730 blob([0xa51000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b797a0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79730) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c58b79730 blob([0xa51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b797b8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79810 blob([0xa53000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79880 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79810) get_ref 0x0~2000 Blob(0x558c58b79810 blob([0xa53000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79880 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79810) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c58b79810 blob([0xa53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79898 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b798f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b798f0 blob([0xa55000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79960 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b798f0) get_ref 0x0~2000 Blob(0x558c58b798f0 blob([0xa55000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79960 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b798f0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c58b798f0 blob([0xa55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79978 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b799d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b799d0 blob([0xa57000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79a40 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b799d0) get_ref 0x0~2000 Blob(0x558c58b799d0 blob([0xa57000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79a40 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b799d0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c58b799d0 blob([0xa57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79a58 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79ab0 blob([0xa59000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79b20 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79ab0) get_ref 0x0~2000 Blob(0x558c58b79ab0 blob([0xa59000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79b20 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c58b79ab0 blob([0xa59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79b38 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79b90 blob([0xa5b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79c00 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79b90) get_ref 0x0~2000 Blob(0x558c58b79b90 blob([0xa5b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79c00 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79b90) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c58b79b90 blob([0xa5b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79c18 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79c70 blob([0xa5d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ce0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79c70) get_ref 0x0~2000 Blob(0x558c58b79c70 blob([0xa5d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ce0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79c70) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c58b79c70 blob([0xa5d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79cf8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79d50 blob([0xa5f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79dc0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79d50) get_ref 0x0~2000 Blob(0x558c58b79d50 blob([0xa5f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79dc0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79d50) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c58b79d50 blob([0xa5f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79dd8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79e30 blob([0xa61000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ea0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79e30) get_ref 0x0~2000 Blob(0x558c58b79e30 blob([0xa61000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ea0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79e30) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c58b79e30 blob([0xa61000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79eb8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b79f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79f10 blob([0xa63000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79f80 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79f10) get_ref 0x0~2000 Blob(0x558c58b79f10 blob([0xa63000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79f80 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b79f10) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c58b79f10 blob([0xa63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b79f98 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80000 blob([0xa65000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80070 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80000) get_ref 0x0~2000 Blob(0x558c58b80000 blob([0xa65000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80070 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80000) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c58b80000 blob([0xa65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80088 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b800e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b800e0 blob([0xa67000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80150 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b800e0) get_ref 0x0~2000 Blob(0x558c58b800e0 blob([0xa67000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80150 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b800e0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c58b800e0 blob([0xa67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80168 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b801c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b801c0 blob([0xa69000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80230 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b801c0) get_ref 0x0~2000 Blob(0x558c58b801c0 blob([0xa69000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80230 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b801c0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c58b801c0 blob([0xa69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80248 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b802a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b802a0 blob([0xa6b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80310 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b802a0) get_ref 0x0~2000 Blob(0x558c58b802a0 blob([0xa6b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80310 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b802a0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c58b802a0 blob([0xa6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80328 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b803f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80380 blob([0xa6d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b803f0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80380) get_ref 0x0~2000 Blob(0x558c58b80380 blob([0xa6d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b803f0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80380) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c58b80380 blob([0xa6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80408 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b804d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80460 blob([0xa6f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b804d0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80460) get_ref 0x0~2000 Blob(0x558c58b80460 blob([0xa6f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b804d0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80460) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c58b80460 blob([0xa6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b804e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b805b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80540 blob([0xa71000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b805b0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80540) get_ref 0x0~2000 Blob(0x558c58b80540 blob([0xa71000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b805b0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80540) get_ref init 0x2000, 1000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c58b80540 blob([0xa71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 sbid 0x0)) 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b805c8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.629+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80620 blob([0xa73000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80690 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80620) get_ref 0x0~2000 Blob(0x558c58b80620 blob([0xa73000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80690 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80620) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c58b80620 blob([0xa73000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b806a8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80700 blob([0xa75000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80770 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80700) get_ref 0x0~2000 Blob(0x558c58b80700 blob([0xa75000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80770 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80700) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c58b80700 blob([0xa75000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80788 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b807e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b807e0 blob([0xa77000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80850 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b807e0) get_ref 0x0~2000 Blob(0x558c58b807e0 blob([0xa77000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80850 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b807e0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c58b807e0 blob([0xa77000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80868 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b808c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b808c0 blob([0xa79000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80930 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b808c0) get_ref 0x0~2000 Blob(0x558c58b808c0 blob([0xa79000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80930 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b808c0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c58b808c0 blob([0xa79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80948 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b809a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b809a0 blob([0xa7b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80a10 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b809a0) get_ref 0x0~2000 Blob(0x558c58b809a0 blob([0xa7b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80a10 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b809a0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c58b809a0 blob([0xa7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80a28 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80a80 blob([0xa7d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80af0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80a80) get_ref 0x0~2000 Blob(0x558c58b80a80 blob([0xa7d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80af0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80a80) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c58b80a80 blob([0xa7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80b08 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80b60 blob([0xa7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80bd0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80b60) get_ref 0x0~2000 Blob(0x558c58b80b60 blob([0xa7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80bd0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80b60) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c58b80b60 blob([0xa7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80be8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80c40 blob([0xa81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80cb0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80c40) get_ref 0x0~2000 Blob(0x558c58b80c40 blob([0xa81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80cb0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80c40) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c58b80c40 blob([0xa81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80cc8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80d20 blob([0xa83000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80d90 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80d20) get_ref 0x0~2000 Blob(0x558c58b80d20 blob([0xa83000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80d90 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80d20) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c58b80d20 blob([0xa83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80da8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80e00 blob([0xa85000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80e70 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80e00) get_ref 0x0~2000 Blob(0x558c58b80e00 blob([0xa85000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80e70 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80e00) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c58b80e00 blob([0xa85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80e88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b80f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80ee0 blob([0xa87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80f50 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80ee0) get_ref 0x0~2000 Blob(0x558c58b80ee0 blob([0xa87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80f50 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c58b80ee0 blob([0xa87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b80f68 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b80fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b80fc0 blob([0xa89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81030 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80fc0) get_ref 0x0~2000 Blob(0x558c58b80fc0 blob([0xa89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81030 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b80fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c58b80fc0 blob([0xa89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81048 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b810a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b810a0 blob([0xa8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81110 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b810a0) get_ref 0x0~2000 Blob(0x558c58b810a0 blob([0xa8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81110 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b810a0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c58b810a0 blob([0xa8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81128 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b811f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81180 blob([0xa8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b811f0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81180) get_ref 0x0~2000 Blob(0x558c58b81180 blob([0xa8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b811f0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81180) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c58b81180 blob([0xa8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81208 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b812d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81260 blob([0xa8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b812d0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81260) get_ref 0x0~2000 Blob(0x558c58b81260 blob([0xa8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b812d0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81260) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c58b81260 blob([0xa8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b812e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b813b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81340 blob([0xa91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b813b0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81340) get_ref 0x0~2000 Blob(0x558c58b81340 blob([0xa91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b813b0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81340) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c58b81340 blob([0xa91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b813c8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81420 blob([0xa93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81490 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81420) get_ref 0x0~2000 Blob(0x558c58b81420 blob([0xa93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81490 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81420) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c58b81420 blob([0xa93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b814a8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81500 blob([0xa95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81570 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81500) get_ref 0x0~2000 Blob(0x558c58b81500 blob([0xa95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81570 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81500) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c58b81500 blob([0xa95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81588 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b815e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b815e0 blob([0xa97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81650 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b815e0) get_ref 0x0~2000 Blob(0x558c58b815e0 blob([0xa97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81650 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b815e0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c58b815e0 blob([0xa97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81668 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b816c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b816c0 blob([0xa99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81730 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b816c0) get_ref 0x0~2000 Blob(0x558c58b816c0 blob([0xa99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81730 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b816c0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c58b816c0 blob([0xa99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81748 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b817a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b817a0 blob([0xa9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81810 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b817a0) get_ref 0x0~2000 Blob(0x558c58b817a0 blob([0xa9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81810 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b817a0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c58b817a0 blob([0xa9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81828 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b818f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81880 blob([0xa9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b818f0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81880) get_ref 0x0~2000 Blob(0x558c58b81880 blob([0xa9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b818f0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81880) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c58b81880 blob([0xa9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81908 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b819d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81960 blob([0xa9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b819d0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81960) get_ref 0x0~2000 Blob(0x558c58b81960 blob([0xa9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b819d0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81960) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c58b81960 blob([0xa9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b819e8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81a40 blob([0xaa1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81ab0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81a40) get_ref 0x0~2000 Blob(0x558c58b81a40 blob([0xaa1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81ab0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81a40) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c58b81a40 blob([0xaa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81ac8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81b20 blob([0xaa3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81b90 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81b20) get_ref 0x0~2000 Blob(0x558c58b81b20 blob([0xaa3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81b90 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81b20) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~2000: 0x0~2000 Blob(0x558c58b81b20 blob([0xaa3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81ba8 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81c00 blob([0xaa5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81c70 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81c00) get_ref 0x0~2000 Blob(0x558c58b81c00 blob([0xaa5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81c70 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81c00) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5e000~2000: 0x0~2000 Blob(0x558c58b81c00 blob([0xaa5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81c88 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81ce0 blob([0xaa7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81d50 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81ce0) get_ref 0x0~2000 Blob(0x558c58b81ce0 blob([0xaa7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81d50 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x60000~2000: 0x0~2000 Blob(0x558c58b81ce0 blob([0xaa7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81d68 in 0x558c53a59ce0) _discard 0x0~2000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b81dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b81e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b81dc0 blob([0xaa9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81e30 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81dc0) get_ref 0x0~1000 Blob(0x558c58b81dc0 blob([0xaa9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81e30 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.blob(0x558c58b81dc0) get_ref init 0x1000, 1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x62000~1000: 0x0~1000 Blob(0x558c58b81dc0 blob([0xaa9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 sbid 0x0)) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.BufferSpace(0x558c58b81e48 in 0x558c53a59ce0) _discard 0x0~1000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x63000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58295508) dirty_range mark inline shard dirty 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~63000 = 0 2019-12-07T19:34:25.630+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:25.630+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:25.630+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:25.630+0000 7f8f0b18f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000048.00000000000000000031' <- 0000000048.00000000000000000031 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000048.00000000000000000032' <- 0000000048.00000000000000000032 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ef8f00 cost 686425 (69 ios * 4000 + 410425 bytes) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ef8f00 onodes 0x558c574a1400,0x558c58295180,0x558c58295400 shared_blobs 0x558c54cf8700,0x558c54d3df80,0x558c54d75180,0x558c54d75340,0x558c57396150,0x558c573963f0,0x558c574d32d0,0x558c574d3730,0x558c574d3880,0x558c574d38f0,0x558c57819d50,0x558c57819f10,0x558c57880540,0x558c57880620,0x558c57880700,0x558c578807e0,0x558c578808c0,0x558c578809a0,0x558c57880e70,0x558c578de0e0,0x558c578de2a0,0x558c578de460,0x558c578de620,0x558c5794c150,0x558c5794c230,0x558c5794c2a0,0x558c5794c460,0x558c5794ce00 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) update 0#1:33f5bf01:::smithi13140796-2:head#20 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c574a1508) update inline shard 756 bytes from 28 extents 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head#20 is 1245 (483 bytes onode + 2 bytes spanning blobs + 760 bytes inline extents) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58295288) update 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58295288) update inline shard 756 bytes from 28 extents 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:19# is 1172 (410 bytes onode + 2 bytes spanning blobs + 760 bytes inline extents) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore.extentmap(0x558c58295508) update inline shard 950 bytes from 50 extents 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1521 (565 bytes onode + 2 bytes spanning blobs + 954 bytes inline extents) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2958 is 11 SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2959 is 11 SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2950 is 11 SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2951 is 11 SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2947 is 11 SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2946 is 11 SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295f is 11 SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2961 is 11 SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2960 is 11 SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294e is 11 SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2957 is 11 SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2956 is 11 SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2948 is 11 SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2949 is 11 SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294a is 11 SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294b is 11 SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294c is 11 SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294d is 11 SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294f is 11 SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2955 is 11 SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2954 is 11 SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2953 is 11 SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2952 is 11 SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295d is 11 SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295c is 11 SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295b is 11 SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295a is 11 SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295e is 11 SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=2))) 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ef8f00 allocated 0x[a47000~63000] released 0x[] 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 freelist allocate 0xa47000~63000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 20 freelist _xor first_key 0xa00000 last_key 0xa80000 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8f00 prepare 2019-12-07T19:34:25.630+0000 7f8f0b18f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57ef8f00 2019-12-07T19:34:25.631+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 lua=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] op_applied version 48'32 2019-12-07T19:34:25.631+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] eval_repop repgather(0x558c576b6d80 48'32 rep_tid=36 committed?=0 r=0) 2019-12-07T19:34:25.631+0000 7f8f0b18f700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] kick_object_context_blocked 1:33f5bf01:::smithi13140796-2:head requeuing 1 requests 2019-12-07T19:34:25.631+0000 7f8f0b18f700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] requeue_op 0x558c578d3600 2019-12-07T19:34:25.631+0000 7f8f0b18f700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) 2019-12-07T19:34:25.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) queued 2019-12-07T19:34:25.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.631+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8f00 aio_wait 2019-12-07T19:34:25.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.631+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ef8f00 2019-12-07T19:34:25.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.631+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3600 prio 63 cost 0 latency 0.014095 osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.632+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8f00 io_done 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op: op osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[19,15,13,e]} 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:25.632+0000 7f8f06986700 25 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] do_op waiting for rw locks 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7662 < min_flush_size 524288 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1400 had 1 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295180 had 1 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7685 < min_flush_size 524288 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7685 < min_flush_size 524288 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x25aae5~1e05 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x25aae5 0x25aae5~1e05 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x22aae5 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xae5 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8ea and padding block with 0x716 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x25c8ea 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:25.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000234275s (0.000050702s flush + 0.000183573s kv commit) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:25.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ef8f00> 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:25.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8f00 kv_submitted 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ef8f00 2019-12-07T19:34:25.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ef8f00 finishing 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ef8f00 onodes 0x558c574a1400,0x558c58295180,0x558c58295400 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79358 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb3b0 space 0x558c58b79358 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79438 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb440 space 0x558c58b79438 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79518 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb4d0 space 0x558c58b79518 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b795f8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb560 space 0x558c58b795f8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b796d8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb5f0 space 0x558c58b796d8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b797b8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb680 space 0x558c58b797b8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79898 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb710 space 0x558c58b79898 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79978 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb7a0 space 0x558c58b79978 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79a58 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb830 space 0x558c58b79a58 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79b38 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb8c0 space 0x558c58b79b38 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79c18 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb950 space 0x558c58b79c18 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79cf8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bb9e0 space 0x558c58b79cf8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79dd8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bba70 space 0x558c58b79dd8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79eb8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbb00 space 0x558c58b79eb8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b79f98 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbb90 space 0x558c58b79f98 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80088 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbc20 space 0x558c58b80088 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80168 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbcb0 space 0x558c58b80168 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80248 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbd40 space 0x558c58b80248 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80328 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbdd0 space 0x558c58b80328 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80408 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbe60 space 0x558c58b80408 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b804e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c582bbef0 space 0x558c58b804e8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b805c8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84000 space 0x558c58b805c8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b806a8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84090 space 0x558c58b806a8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80788 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84120 space 0x558c58b80788 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80868 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b841b0 space 0x558c58b80868 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80948 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84240 space 0x558c58b80948 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80a28 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b842d0 space 0x558c58b80a28 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80b08 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84360 space 0x558c58b80b08 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80be8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b843f0 space 0x558c58b80be8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80cc8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84480 space 0x558c58b80cc8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80da8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84510 space 0x558c58b80da8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80e88 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b845a0 space 0x558c58b80e88 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b80f68 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84630 space 0x558c58b80f68 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81048 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b846c0 space 0x558c58b81048 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81128 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84750 space 0x558c58b81128 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81208 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b847e0 space 0x558c58b81208 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b812e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84870 space 0x558c58b812e8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b813c8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84900 space 0x558c58b813c8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b814a8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84990 space 0x558c58b814a8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81588 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84a20 space 0x558c58b81588 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81668 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84ab0 space 0x558c58b81668 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81748 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84b40 space 0x558c58b81748 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81828 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84bd0 space 0x558c58b81828 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81908 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84c60 space 0x558c58b81908 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b819e8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84cf0 space 0x558c58b819e8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81ac8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84d80 space 0x558c58b81ac8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81ba8 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84e10 space 0x558c58b81ba8 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81c88 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84ea0 space 0x558c58b81c88 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81d68 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84f30 space 0x558c58b81d68 0x0~2000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b81e48 in 0x558c53a59ce0) _finish_write discard buffer(0x558c58b84fc0 space 0x558c58b81e48 0x0~1000 writing nocache) 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ef8f00 done 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:25.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ef8f00 [] 2019-12-07T19:34:25.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:25.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:25.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.632+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 219 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c576b5000 con 0x558c54c5bc00 2019-12-07T19:34:25.632+0000 7f8f23c56700 15 osd.7 48 enqueue_op 0x558c57efeb00 prio 196 cost 0 latency 0.000018 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.632+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.632+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c57efeb00 prio 196 cost 0 latency 0.000039 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.632+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:25.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 366 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c576b4e00 con 0x558c54c64000 2019-12-07T19:34:25.632+0000 7f8f24457700 15 osd.7 48 enqueue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000019 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.632+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.632+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 244 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c576b5e00 con 0x558c54c5b400 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.632+0000 7f8f23c56700 15 osd.7 48 enqueue_op 0x558c57eff8c0 prio 196 cost 0 latency 0.000015 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.632+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000084 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.632+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.633+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c57eff8c0 prio 196 cost 0 latency 0.000064 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.633+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.633+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c57eff8c0 finish 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 93 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518a00 con 0x558c54c19c00 2019-12-07T19:34:25.633+0000 7f8f24457700 15 osd.7 48 enqueue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000016 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.633+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.633+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3a20 prio 196 cost 0 latency 0.000041 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.633+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.633+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3a20 finish 2019-12-07T19:34:25.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.634+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 145 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841e800 con 0x558c54c19800 2019-12-07T19:34:25.634+0000 7f8f24c58700 15 osd.7 48 enqueue_op 0x558c5511e840 prio 196 cost 0 latency 0.000014 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.634+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c5511e840 prio 196 cost 0 latency 0.000032 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=36, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=48'32 tt=0'0 tid=36 reqid=client.4540.0:380 client_op=osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[0~1622016]}) 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=45'30 crt=45'30 lcod 45'28 mlcod 45'28 active+clean ps=[c~1,11~1,17~1]] repop_all_committed: repop tid 36 all committed 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] eval_repop repgather(0x558c576b6d80 48'32 rep_tid=36 committed?=1 r=0) 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] commit: repgather(0x558c576b6d80 48'32 rep_tid=36 committed?=1 r=0) 2019-12-07T19:34:25.634+0000 7f8f0297e700 15 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] log_op_stats osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) v8 inb 1622016 outb 0 lat 0.016291 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] sending reply on osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) v8 0x558c58092280 2019-12-07T19:34:25.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(380 smithi13140796-2 [copy-from ver 21] v48'32 uv21 ondisk = 0) v8 -- 0x558c58092280 con 0x558c54c9f000 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:25.634+0000 7f8f0297e700 15 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:150 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] removing repgather(0x558c576b6d80 48'32 rep_tid=36 committed?=1 r=0) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] q front is repgather(0x558c576b6d80 48'32 rep_tid=36 committed?=1 r=0) 2019-12-07T19:34:25.634+0000 7f8f0297e700 15 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] remove_repop repgather(0x558c576b6d80 48'32 rep_tid=36 committed?=1 r=0) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] requeue_op 0x558c578d3600 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=48'32 tt=0'0 tid=36 reqid=client.4540.0:380 client_op=osd_op(client.4540.0:380 1.cs0 1:33f5bf01:::smithi13140796-2:head [copy-from ver 21 in=55b] snapc 19=[19,15,13,e] ondisk+write+known_if_redirected e48) roll_forward_to=45'30 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[0~1622016]}) 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) queued 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_finish_rmw: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~1622016:7) ) 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=37 reqid=client.4540.0:380 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=37 reqid=client.4540.0:380 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:25.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 48/28 ECSubWrite(tid=37, reqid=client.4540.0:380, at_version=0'0, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c54e3f800 con 0x558c54c64000 2019-12-07T19:34:25.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 48/28 ECSubWrite(tid=37, reqid=client.4540.0:380, at_version=0'0, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c575f4000 con 0x558c54c19800 2019-12-07T19:34:25.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 48/28 ECSubWrite(tid=37, reqid=client.4540.0:380, at_version=0'0, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c57e7c800 con 0x558c54c5bc00 2019-12-07T19:34:25.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 48/28 ECSubWrite(tid=37, reqid=client.4540.0:380, at_version=0'0, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c57e7b000 con 0x558c54c5b400 2019-12-07T19:34:25.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 48/28 ECSubWrite(tid=37, reqid=client.4540.0:380, at_version=0'0, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c57e7d800 con 0x558c54c19c00 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=45'30 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] append_log log((0'0,48'32], crt=45'30) [] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] rollforward: entry=48'31 (21'11) clone 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [19,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] rollforward: entry=48'32 (21'11) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:380 0.000000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] append_log approx pg log length = 32 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] append_log transaction_applied = 1 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:25.634+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f6900 seq 44 2019-12-07T19:34:25.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head#20 onode 0x558c574a1400 txc 0x558c585f6900 2019-12-07T19:34:25.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head#20 0x0 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57396380) put_ref 0x0~2000 Blob(0x558c57396380 blob([0x2a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57396460) put_ref 0x0~2000 Blob(0x558c57396460 blob([0x2a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578804d0) put_ref 0x0~2000 Blob(0x558c578804d0 blob([0x2aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578805b0) put_ref 0x0~2000 Blob(0x558c578805b0 blob([0x2ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880690) put_ref 0x0~2000 Blob(0x558c57880690 blob([0x2ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880770) put_ref 0x0~2000 Blob(0x558c57880770 blob([0x2b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880850) put_ref 0x0~2000 Blob(0x558c57880850 blob([0x2b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880930) put_ref 0x0~2000 Blob(0x558c57880930 blob([0x2b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3340) put_ref 0x0~2000 Blob(0x558c574d3340 blob([0x350000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880d20) put_ref 0x0~2000 Blob(0x558c57880d20 blob([0x352000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880c40) put_ref 0x0~2000 Blob(0x558c57880c40 blob([0x354000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c54d75420) put_ref 0x0~2000 Blob(0x558c54d75420 blob([0x356000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c575bd8f0) put_ref 0x0~2000 Blob(0x558c575bd8f0 blob([0x358000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578de540) put_ref 0x0~2000 Blob(0x558c578de540 blob([0x35a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578de380) put_ref 0x0~2000 Blob(0x558c578de380 blob([0x35c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578de1c0) put_ref 0x0~2000 Blob(0x558c578de1c0 blob([0x35e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578de000) put_ref 0x0~2000 Blob(0x558c578de000 blob([0x360000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57819e30) put_ref 0x0~2000 Blob(0x558c57819e30 blob([0x362000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c54cf8620) put_ref 0x0~2000 Blob(0x558c54cf8620 blob([0x364000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1f10) put_ref 0x0~2000 Blob(0x558c578f1f10 blob([0x366000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c4d0) put_ref 0x0~2000 Blob(0x558c5794c4d0 blob([0x368000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c3f0) put_ref 0x0~2000 Blob(0x558c5794c3f0 blob([0x36a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c310) put_ref 0x0~2000 Blob(0x558c5794c310 blob([0x36c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c1c0) put_ref 0x0~2000 Blob(0x558c5794c1c0 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c5794c0e0) put_ref 0x0~2000 Blob(0x558c5794c0e0 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c5794cd90) put_ref 0x0~2000 Blob(0x558c5794cd90 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3810) put_ref 0x0~2000 Blob(0x558c574d3810 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.blob(0x558c574d3260) put_ref 0x0~1000 Blob(0x558c574d3260 blob([0x376000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore.extentmap(0x558c574a1508) dirty_range mark inline shard dirty 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57396380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a6000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57396460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a8000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c578804d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2aa000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c578805b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ac000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57880690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ae000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57880770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b0000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57880850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b2000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57880930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b4000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c574d3340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x350000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57880d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x352000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57880c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x354000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c54d75420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x356000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c575bd8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x358000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c578de540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x35a000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c578de380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x35c000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c578de1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x35e000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c578de000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x360000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57819e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x362000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c54cf8620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x364000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c578f1f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x366000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c5794c4d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x368000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c5794c3f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x36a000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c5794c310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x36c000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c5794c1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x36e000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c5794c0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x370000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c5794cd90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x372000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c574d3810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x374000~2000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~1000: 0x0~1000 Blob(0x558c574d3260 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=2)))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x376000~1000] 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf8700,0x558c54d3df80,0x558c54d75180,0x558c54d75340,0x558c57396150,0x558c573963f0,0x558c574d32d0,0x558c574d3730,0x558c574d3880,0x558c574d38f0,0x558c57819d50,0x558c57819f10,0x558c57880540,0x558c57880620,0x558c57880700,0x558c578807e0,0x558c578808c0,0x558c578809a0,0x558c57880e70,0x558c578de0e0,0x558c578de2a0,0x558c578de460,0x558c578de620,0x558c5794c150,0x558c5794c230,0x558c5794c2a0,0x558c5794c460,0x558c5794ce00 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c58295400 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head#20 = 0 2019-12-07T19:34:25.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f6900 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f6900 onodes shared_blobs 0x558c54cf8700,0x558c54d3df80,0x558c54d75180,0x558c54d75340,0x558c57396150,0x558c573963f0,0x558c574d32d0,0x558c574d3730,0x558c574d3880,0x558c574d38f0,0x558c57819d50,0x558c57819f10,0x558c57880540,0x558c57880620,0x558c57880700,0x558c578807e0,0x558c578808c0,0x558c578809a0,0x558c57880e70,0x558c578de0e0,0x558c578de2a0,0x558c578de460,0x558c578de620,0x558c5794c150,0x558c5794c230,0x558c5794c2a0,0x558c5794c460,0x558c5794ce00 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2958 is 11 SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2959 is 11 SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2950 is 11 SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2951 is 11 SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2947 is 11 SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2946 is 11 SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295f is 11 SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2961 is 11 SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2960 is 11 SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294e is 11 SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2957 is 11 SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2956 is 11 SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2948 is 11 SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2949 is 11 SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294a is 11 SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294b is 11 SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294c is 11 SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294d is 11 SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294f is 11 SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2955 is 11 SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2954 is 11 SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2953 is 11 SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2952 is 11 SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295d is 11 SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295c is 11 SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295b is 11 SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295a is 11 SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295e is 11 SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=1))) 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f6900 allocated 0x[] released 0x[] 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 prepare 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f6900 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 io_done 2019-12-07T19:34:25.634+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c5511e840 finish 2019-12-07T19:34:25.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8) prio 63 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3600 prio 63 cost 0 latency 0.016807 osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] _handle_message: osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_op: op osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_op osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[19,15,13,e]} 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:25.634+0000 7f8f06986700 25 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(excl n=1 w=0)) 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] execute_ctx 0x558c574e7800 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [stat] ov 48'32 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [stat] 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_osd_op stat 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] stat oi has 1622016 2019-12-07T19:34:21.628654+0000 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] execute_ctx alloc reply 0x558c58092f00 result 0 2019-12-07T19:34:25.634+0000 7f8f06986700 15 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:25.634+0000 7f8f06986700 15 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] log_op_stats osd_op(client.4540.0:381 1.cs0 1:33f5bf01:::smithi13140796-2:head [stat out=16b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e48) v8 inb 0 outb 16 lat 0.016902 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:25.634+0000 7f8f06986700 15 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 48:151 2019-12-07T19:34:25.634+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(381 smithi13140796-2 [stat out=16b] v0'0 uv21 ondisk = 0) v8 -- 0x558c58092f00 con 0x558c54c9f000 2019-12-07T19:34:25.634+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d3600 finish 2019-12-07T19:34:25.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1892 < min_flush_size 524288 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1400 had 1 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1915 < min_flush_size 524288 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1915 < min_flush_size 524288 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x25c8ea~77b to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x25c8ea 0x25c8ea~77b to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x22c8ea 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8ea 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x65 and padding block with 0xf9b 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x25d065 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:25.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107555s (0.000000096s flush + 0.000107459s kv commit) 2019-12-07T19:34:25.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585f6900> 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:25.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 kv_submitted 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f6900 2019-12-07T19:34:25.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 finishing 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f6900 onodes 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6900 done 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:25.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f6900 [] 2019-12-07T19:34:25.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:25.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 94 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518600 con 0x558c54c19c00 2019-12-07T19:34:25.635+0000 7f8f24457700 15 osd.7 48 enqueue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000018 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 245 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c576b4400 con 0x558c54c5b400 2019-12-07T19:34:25.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.635+0000 7f8f23c56700 15 osd.7 48 enqueue_op 0x558c57effb80 prio 196 cost 0 latency 0.000016 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.635+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 146 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841ec00 con 0x558c54c19800 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f24c58700 15 osd.7 48 enqueue_op 0x558c5511e9a0 prio 196 cost 0 latency 0.000015 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000056 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.635+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 367 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332400 con 0x558c54c64000 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.635+0000 7f8f24457700 15 osd.7 48 enqueue_op 0x558c578d31e0 prio 196 cost 0 latency 0.000012 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 220 ==== MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332a00 con 0x558c54c5bc00 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.635+0000 7f8f23c56700 15 osd.7 48 enqueue_op 0x558c57334c60 prio 196 cost 0 latency 0.000011 epoch 48 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) 2019-12-07T19:34:25.635+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c57effb80 prio 196 cost 0 latency 0.000082 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.635+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c57effb80 finish 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c5511e9a0 prio 196 cost 0 latency 0.000083 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c5511e9a0 finish 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) queued 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.635+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d31e0 prio 196 cost 0 latency 0.000100 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.635+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f06986700 10 osd.7 48 dequeue_op 0x558c578d31e0 finish 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2) prio 196 cost 0 e48) pg 0x558c57676000 2019-12-07T19:34:25.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c57334c60 prio 196 cost 0 latency 0.000101 MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 pg pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 48/28 ECSubWriteReply(tid=37, last_complete=48'32, committed=1, applied=1)) v2 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=37 reqid=client.4540.0:380 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:25.635+0000 7f8f0297e700 10 osd.7 48 dequeue_op 0x558c57334c60 finish 2019-12-07T19:34:25.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:25.654+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 13 ==== osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:25.655920+0000/49.112843376s send_stamp 49.112843376s delta_ub 0.021654510s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b5400 con 0x558c54c18800 2019-12-07T19:34:25.654+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021327357s,0.021654510s]) 2019-12-07T19:34:25.654+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:25.655920+0000/49.112843376s send_stamp 49.091516019s delta_ub 18446744073.688224259s) v5 -- 0x558c5795c800 con 0x558c54c18800 2019-12-07T19:34:25.654+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 13 ==== osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:25.655920+0000/49.112843376s send_stamp 49.112843376s delta_ub 0.021654510s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4a00 con 0x558c54c19000 2019-12-07T19:34:25.654+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021247687s,0.021654510s]) 2019-12-07T19:34:25.654+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:25.655920+0000/49.112843376s send_stamp 49.091595689s delta_ub 18446744073.688303929s) v5 -- 0x558c5795c800 con 0x558c54c19000 2019-12-07T19:34:25.672+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 10 osd.7 48 tick_without_osd_lock 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f556000/0x40000000/0x1680000000, data 0xbec0f8/0xaa8000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075394+0000 last_rx_front 2019-12-07T19:34:22.075339+0000 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075420+0000 last_rx_front 2019-12-07T19:34:22.075379+0000 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075351+0000 last_rx_front 2019-12-07T19:34:22.075363+0000 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075252+0000 last_rx_front 2019-12-07T19:34:22.075295+0000 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:22.075017+0000 last_rx_back 2019-12-07T19:34:22.075263+0000 last_rx_front 2019-12-07T19:34:22.075319+0000 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 20 osd.7 48 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 10 osd.7 48 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 20 osd.7 48 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:25.707+0000 7f8f1fbbe700 10 osd.7 48 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:25.722+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.772+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.822+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.873+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.923+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:25.973+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.002+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:26.023+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.072+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.123+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.173+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.174+0000 7f8f0097a700 5 osd.7 48 heartbeat osd_stat(store_statfs(0x163f556000/0x40000000/0x1680000000, data 0xbec0f8/0xaa8000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5] op hist []) 2019-12-07T19:34:26.174+0000 7f8f0097a700 20 osd.7 48 check_full_status cur ratio 0.000115712, physical ratio 0.000115712, new state none 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.688303929s) v5 -- 0x558c58333200 con 0x558c54c9e000 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.688303929s) v5 -- 0x558c58333400 con 0x558c54c9e400 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.673493809s) v5 -- 0x558c58333600 con 0x558c551b7800 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.673493809s) v5 -- 0x558c58333800 con 0x558c551b7c00 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.687662701s) v5 -- 0x558c58333a00 con 0x558c552c2000 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.687662701s) v5 -- 0x558c58333c00 con 0x558c552c2400 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.696151503s) v5 -- 0x558c58333e00 con 0x558c552c2800 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.696151503s) v5 -- 0x558c5841e800 con 0x558c552c2c00 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.678770454s) v5 -- 0x558c5841ec00 con 0x558c54c65800 2019-12-07T19:34:26.174+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.610693766s delta_ub 18446744073.678770454s) v5 -- 0x558c5841e400 con 0x558c54c65c00 2019-12-07T19:34:26.174+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 14 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.624635499s delta_ub 0.013941733s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c552c2800 2019-12-07T19:34:26.174+0000 7f8f24c58700 25 osd.7 48 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:22.075252+0000 -> 2019-12-07T19:34:26.175478+0000 last_rx_front 2019-12-07T19:34:22.075295+0000 2019-12-07T19:34:26.174+0000 7f8f24c58700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013717992s,0.013941733s]) 2019-12-07T19:34:26.174+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 10 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.641753889s delta_ub 0.031060123s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4a00 con 0x558c54c65c00 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:22.075263+0000 last_rx_front 2019-12-07T19:34:22.075319+0000 -> 2019-12-07T19:34:26.175507+0000 2019-12-07T19:34:26.174+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030807045s,0.031060123s]) 2019-12-07T19:34:26.174+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 14 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.624692599s delta_ub 0.013998833s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518c00 con 0x558c552c2c00 2019-12-07T19:34:26.174+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 14 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.647248547s delta_ub 0.036554781s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c551b7c00 2019-12-07T19:34:26.174+0000 7f8f24457700 25 osd.7 48 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175478+0000 last_rx_front 2019-12-07T19:34:22.075295+0000 -> 2019-12-07T19:34:26.175520+0000 2019-12-07T19:34:26.174+0000 7f8f24457700 25 osd.7 48 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:26.175254+0000) and older pending ping(s) 2019-12-07T19:34:26.174+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 10 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.632438527s delta_ub 0.021744761s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4a00 con 0x558c54c9e000 2019-12-07T19:34:26.174+0000 7f8f24457700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013732779s,0.013998833s]) 2019-12-07T19:34:26.174+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 14 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.633066488s delta_ub 0.022372722s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518c00 con 0x558c552c2000 2019-12-07T19:34:26.174+0000 7f8f24c58700 25 osd.7 48 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:22.075420+0000 last_rx_front 2019-12-07T19:34:22.075379+0000 -> 2019-12-07T19:34:26.175543+0000 2019-12-07T19:34:26.174+0000 7f8f24c58700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036266352s,0.036554781s]) 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:22.075394+0000 -> 2019-12-07T19:34:26.175569+0000 last_rx_front 2019-12-07T19:34:22.075339+0000 2019-12-07T19:34:26.174+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021429505s,0.021744761s]) 2019-12-07T19:34:26.174+0000 7f8f24457700 25 osd.7 48 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:22.075351+0000 -> 2019-12-07T19:34:26.175579+0000 last_rx_front 2019-12-07T19:34:22.075363+0000 2019-12-07T19:34:26.174+0000 7f8f24457700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022047462s,0.022372722s]) 2019-12-07T19:34:26.174+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 10 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.632504497s delta_ub 0.021810731s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c54c9e400 2019-12-07T19:34:26.174+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 10 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.641770821s delta_ub 0.031077055s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4a00 con 0x558c54c65800 2019-12-07T19:34:26.174+0000 7f8f24c58700 25 osd.7 48 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175569+0000 last_rx_front 2019-12-07T19:34:22.075339+0000 -> 2019-12-07T19:34:26.175589+0000 2019-12-07T19:34:26.174+0000 7f8f24c58700 25 osd.7 48 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:26.175254+0000) and older pending ping(s) 2019-12-07T19:34:26.174+0000 7f8f24c58700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021475436s,0.021810731s]) 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:22.075263+0000 -> 2019-12-07T19:34:26.175607+0000 last_rx_front 2019-12-07T19:34:26.175507+0000 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:26.175254+0000) and older pending ping(s) 2019-12-07T19:34:26.174+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030724271s,0.031077055s]) 2019-12-07T19:34:26.174+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 14 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.633040501s delta_ub 0.022346735s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4a00 con 0x558c552c2400 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175579+0000 last_rx_front 2019-12-07T19:34:22.075363+0000 -> 2019-12-07T19:34:26.175631+0000 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:26.175254+0000) and older pending ping(s) 2019-12-07T19:34:26.174+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021969639s,0.022346735s]) 2019-12-07T19:34:26.174+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 14 ==== osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.175254+0000/49.610693766s send_stamp 49.647278986s delta_ub 0.036585220s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4a00 con 0x558c551b7800 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:22.075420+0000 -> 2019-12-07T19:34:26.175651+0000 last_rx_front 2019-12-07T19:34:26.175543+0000 2019-12-07T19:34:26.174+0000 7f8f23c56700 25 osd.7 48 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:26.175254+0000) and older pending ping(s) 2019-12-07T19:34:26.174+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036187987s,0.036585220s]) 2019-12-07T19:34:26.223+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.273+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.323+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.346+0000 7f8f21451700 10 osd.7 48 tick 2019-12-07T19:34:26.346+0000 7f8f21451700 10 osd.7 48 do_waiters -- start 2019-12-07T19:34:26.346+0000 7f8f21451700 10 osd.7 48 do_waiters -- finish 2019-12-07T19:34:26.346+0000 7f8f21451700 20 osd.7 48 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:26.373+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.404+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:26.423+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.474+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.524+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.573+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.612+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:26.624+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 15 ==== osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.645406+0000/50.117124807s send_stamp 50.117124807s delta_ub 0.036585220s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518c00 con 0x558c552d1c00 2019-12-07T19:34:26.644+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 15 ==== osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:26.645406+0000/50.117124807s send_stamp 50.117124807s delta_ub 0.036585220s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c552d1800 2019-12-07T19:34:26.644+0000 7f8f24457700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036160731s,0.036585220s]) 2019-12-07T19:34:26.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.645406+0000/50.117124807s send_stamp 50.080964076s delta_ub 18446744073.673390885s) v5 -- 0x558c5841e800 con 0x558c552d1c00 2019-12-07T19:34:26.644+0000 7f8f24c58700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036143956s,0.036585220s]) 2019-12-07T19:34:26.644+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:26.645406+0000/50.117124807s send_stamp 50.080980851s delta_ub 18446744073.673407660s) v5 -- 0x558c58333e00 con 0x558c552d1800 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:26.674+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0259231 2019-12-07T19:34:26.674+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.724+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 10 osd.7 48 tick_without_osd_lock 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f556000/0x40000000/0x1680000000, data 0xbec0f8/0xaa8000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175569+0000 last_rx_front 2019-12-07T19:34:26.175589+0000 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175651+0000 last_rx_front 2019-12-07T19:34:26.175543+0000 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175579+0000 last_rx_front 2019-12-07T19:34:26.175631+0000 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175478+0000 last_rx_front 2019-12-07T19:34:26.175520+0000 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 25 osd.7 48 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175607+0000 last_rx_front 2019-12-07T19:34:26.175507+0000 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 20 osd.7 48 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 10 osd.7 48 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 20 osd.7 48 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:26.735+0000 7f8f1fbbe700 10 osd.7 48 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:26.775+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.814+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:26.825+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.875+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.924+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.951+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57be8800 0x558c54c70680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:26.951+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c57be8800 0x558c54c70680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:26.951+0000 7f8f24457700 10 osd.7 48 ms_handle_authentication new session 0x558c574049c0 con 0x558c57be8800 entity client.admin addr 2019-12-07T19:34:26.951+0000 7f8f24457700 10 osd.7 48 ms_handle_authentication session 0x558c574049c0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:26.951+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/162291428 conn(0x558c57be8800 0x558c54c70680 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4502 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:26.951+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4502 172.21.15.131:0/162291428 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c5734b4a0 con 0x558c57be8800 2019-12-07T19:34:26.951+0000 7f8f164bb700 20 osd.7 48 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:26.951+0000 7f8f164bb700 10 osd.7 48 do_waiters -- start 2019-12-07T19:34:26.951+0000 7f8f164bb700 10 osd.7 48 do_waiters -- finish 2019-12-07T19:34:26.951+0000 7f8f164bb700 20 osd.7 48 _dispatch 0x558c5734b4a0 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:26.951+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/162291428 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce01c0 con 0x558c57be8800 2019-12-07T19:34:26.974+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:26.993+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4502 172.21.15.131:0/162291428 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c5734b680 con 0x558c57be8800 2019-12-07T19:34:26.993+0000 7f8f164bb700 20 osd.7 48 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:26.993+0000 7f8f164bb700 10 osd.7 48 do_waiters -- start 2019-12-07T19:34:26.993+0000 7f8f164bb700 10 osd.7 48 do_waiters -- finish 2019-12-07T19:34:26.993+0000 7f8f164bb700 20 osd.7 48 _dispatch 0x558c5734b680 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:26.994+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/162291428 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce1a40 con 0x558c57be8800 2019-12-07T19:34:26.997+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/162291428 conn(0x558c57be8800 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 72 2019-12-07T19:34:26.997+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/162291428 conn(0x558c57be8800 msgr2=0x558c54c70680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:26.997+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/162291428 conn(0x558c57be8800 0x558c54c70680 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:26.997+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/162291428 conn(0x558c57be8800 0x558c54c70680 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:26.997+0000 7f8f164bb700 2 osd.7 48 ms_handle_reset con 0x558c57be8800 session 0x558c574049c0 2019-12-07T19:34:27.025+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.075+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.125+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.145+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 11 ==== osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:27.145613+0000/50.611936679s send_stamp 50.611936679s delta_ub 0.031077055s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b5400 con 0x558c54c5ac00 2019-12-07T19:34:27.145+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 11 ==== osd_ping(ping e48 up_from 13 ping_stamp 2019-12-07T19:34:27.145613+0000/50.611936679s send_stamp 50.611936679s delta_ub 0.031077055s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518800 con 0x558c54c5a400 2019-12-07T19:34:27.145+0000 7f8f23c56700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030693263s,0.031077055s]) 2019-12-07T19:34:27.145+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:27.145613+0000/50.611936679s send_stamp 50.581243416s delta_ub 18446744073.678858353s) v5 -- 0x558c5841e400 con 0x558c54c5ac00 2019-12-07T19:34:27.145+0000 7f8f24c58700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030677246s,0.031077055s]) 2019-12-07T19:34:27.145+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:27.145613+0000/50.611936679s send_stamp 50.581259433s delta_ub 18446744073.678874370s) v5 -- 0x558c58333e00 con 0x558c54c5a400 2019-12-07T19:34:27.176+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.225+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1271908 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.262+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 10 ==== osd_ping(ping e49 up_from 13 ping_stamp 2019-12-07T19:34:27.262911+0000/50.712137541s send_stamp 50.712137541s delta_ub 0.013998833s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c54c65400 2019-12-07T19:34:27.262+0000 7f8f24c58700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013597266s,0.013998833s]) 2019-12-07T19:34:27.262+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:27.262911+0000/50.712137541s send_stamp 50.698540275s delta_ub 18446744073.695954350s) v5 -- 0x558c58333e00 con 0x558c54c65400 2019-12-07T19:34:27.262+0000 7f8f24c58700 10 osd.7 48 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 48 -> 49 (as per caller) 2019-12-07T19:34:27.262+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 10 ==== osd_ping(ping e49 up_from 13 ping_stamp 2019-12-07T19:34:27.262911+0000/50.712137541s send_stamp 50.712137541s delta_ub 0.013998833s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518800 con 0x558c54c65000 2019-12-07T19:34:27.262+0000 7f8f24c58700 20 osd.7 48 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013551348s,0.013998833s]) 2019-12-07T19:34:27.262+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e48 up_from 13 ping_stamp 2019-12-07T19:34:27.262911+0000/50.712137541s send_stamp 50.698586193s delta_ub 18446744073.696000268s) v5 -- 0x558c58333e00 con 0x558c54c65000 2019-12-07T19:34:27.262+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 246 ==== osd_map(49..49 src has 1..49) v4 ==== 633+0+0 (unknown 0 0 0) 0x558c576b3200 con 0x558c54c5b400 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 osd.7 48 OSD::ms_dispatch: osd_map(49..49 src has 1..49) v4 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 osd.7 48 do_waiters -- start 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 osd.7 48 do_waiters -- finish 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 osd.7 48 _dispatch 0x558c576b3200 osd_map(49..49 src has 1..49) v4 2019-12-07T19:34:27.262+0000 7f8f1199c700 3 osd.7 48 handle_osd_map epochs [49,49], i have 48, src has [1,49] 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 osd.7 48 handle_osd_map got inc map for epoch 49 2019-12-07T19:34:27.262+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:34a8e826:::osdmap.48:0# 0x0~0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55060a80 blob([0xa45000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800d340 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2540 blob([!~1000,0xa46000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c54677110 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:34a8e826:::osdmap.48:0# 0x0~1448 = 5192 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 osd.7 48 add_map_bl 48 5192 bytes 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 osd.7 48 got_full_map 49, nothing requested 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 osd.7 48 handle_osd_map pg_num_history pg_num_history(e49 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {49={}} 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 osd.7 48 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e49 [1,49] lci=[13,49]) 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c5829b500 seq 40 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c7b7972f:::inc_osdmap.49:0# key 0x7f7fffffffffffffffc7b797'/!inc_osdmap.49!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c7b7972f:::inc_osdmap.49:0# 0x0~24d 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1237 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c7b7972f:::inc_osdmap.49:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b500 1 blobs 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xaaa000~1000/1000,1000,0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xaaa000~1000] 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b25b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580d77a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b25b0 blob([0xaaa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d77a0 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c550b25b0) get_ref 0x0~24d Blob(0x558c550b25b0 blob([0xaaa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d77a0 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c550b25b0) get_ref init 0x1000, 1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c550b25b0 blob([0xaaa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c580d77a0 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c580d77b8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57573008) dirty_range mark inline shard dirty 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c7b7972f:::inc_osdmap.49:0# 0x0~24d = 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3f54e826:::osdmap.49:0# key 0x7f7fffffffffffffff3f54e8'&!osdmap.49!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3f54e826:::osdmap.49:0# 0x0~1448 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1238 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3f54e826:::osdmap.49:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b500 2 blobs 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xaab000~2000/1000,2000,0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xaab000~2000] 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7810 blob([0xaab000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3960 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c580d7810) get_ref 0x0~1000 Blob(0x558c580d7810 blob([0xaab000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3960 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c580d7810) get_ref init 0x1000, 1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c580d7810 blob([0xaab000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c550b3960 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c550b3978 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c393b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c393b0 blob([!~1000,0xaac000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39420 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c57c393b0) get_ref 0x1000~448 Blob(0x558c57c393b0 blob([!~1000,0xaac000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57c39420 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c57c393b0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c57c393b0 blob([!~1000,0xaac000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57c39420 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c39438 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57e8f788) dirty_range mark inline shard dirty 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3f54e826:::osdmap.49:0# 0x0~1448 = 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b500 0 blobs 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b500 0 blobs 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5829b500 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5829b500 onodes 0x558c53a91900,0x558c55276c80,0x558c57572f00,0x558c57e8f680 shared_blobs 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57573008) update #-1:c7b7972f:::inc_osdmap.49:0# 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57573008) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c7b7972f:::inc_osdmap.49:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57e8f788) update #-1:3f54e826:::osdmap.49:0# 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57e8f788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3f54e826:::osdmap.49:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5829b500 allocated 0x[aaa000~3000] released 0x[] 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 freelist allocate 0xaaa000~3000 2019-12-07T19:34:27.262+0000 7f8f1199c700 20 freelist _xor first_key 0xa80000 last_key 0xa80000 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b500 prepare 2019-12-07T19:34:27.262+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5829b500 2019-12-07T19:34:27.262+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b500 aio_wait 2019-12-07T19:34:27.262+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5829b500 2019-12-07T19:34:27.262+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b500 io_done 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572f00 had 1 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f680 had 1 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3858366464 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998152) bluestore 86 GiB free (0.959884), bluefs_ratio 0.0399341 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x25d065~226c to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x25d065 0x25d065~226c to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~230000 x_off 0x22d065 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x65 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2d1 and padding block with 0xd2f 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x25f2d1 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:27.262+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000166027s (0.000048561s flush + 0.000117466s kv commit) 2019-12-07T19:34:27.262+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5829b500> 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:27.262+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b500 kv_submitted 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5829b500 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c5829b500 osr 0x558c54ea4b40 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 37 0x1b000~1000 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7 48 _committed_osd_maps 49..49 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 38 0x1b000~1000 crc 3f4a13d1 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7 48 advance to epoch 49 (<= last 49 <= newest_map 49) 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 37 0x2000~1000 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 38 0x2000~1000 crc 768775bf 2019-12-07T19:34:27.262+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:27.262+0000 7f8f141a1700 7 osd.7 49 consume_map version 49 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.8s5 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.0s3 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.1s1 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.9s4 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.as5 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.2s0 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.bs1 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.3s4 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.cs0 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.4s0 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.5s4 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.ds2 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.es4 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.6s2 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.fs5 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 identify_splits_and_merges 1.7s4 e48 to e49 pg_nums {18=16} 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 49 prune_pg_created 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:0.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:1.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:2.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:3.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:4.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:5.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:6.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7:7.consume_map 49 (was 48) 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:27.262+0000 7f8f141a1700 10 osd.7 49 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.8s5 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.0s3 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.1s1 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.262+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.9s4 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.262+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c57380000 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.as5 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.262+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.262+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.2s0 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c5736a000 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.262+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.262+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.262+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c57e8b000 2019-12-07T19:34:27.262+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.262+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 48 -> 49 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.bs1 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f08189700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.262+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 48 -> 49 2019-12-07T19:34:27.262+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.3s4 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:27.262+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.262+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.cs0 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.262+0000 7f8f08189700 10 osd.7 pg_epoch: 48 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.4s0 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.262+0000 7f8f07988700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.5s4 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c575ef000 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.ds2 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c57676000 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c57522000 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.es4 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.6s2 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c54d98000 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 48 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:27.263+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c55141000 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c5500d000 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c5500a000 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 48 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 48 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 48 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 48 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:27.263+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c5513e000 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 48 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:27.263+0000 7f8f06986700 7 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:27.263+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:27.263+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:27.263+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f06986700 15 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 48:152 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.fs5 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 48 while current is 49 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f141a1700 15 osd.7 49 enqueue_peering_evt 1.7s4 epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c54d5c000 2019-12-07T19:34:27.263+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c57383000 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f141a1700 7 osd.7 49 activate_map version 49 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c548ab000 2019-12-07T19:34:27.263+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 48 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:27.263+0000 7f8f0918b700 20 osd.7 49 agent_entry empty queue 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c54d5f000 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 48 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 48 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 48 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:27.263+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:27.263+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:27.263+0000 7f8f06986700 7 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:27.263+0000 7f8f06986700 15 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:121 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 NullEvt) prio 255 cost 10 e49) pg 0x558c57e88000 2019-12-07T19:34:27.263+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f07988700 10 osd.7 pg_epoch: 48 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:27.263+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:27.263+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 49 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 48 -> 49 2019-12-07T19:34:27.263+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f07988700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 48 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:27.263+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.263+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 49 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 49 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:27.263+0000 7f8f03980700 7 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:27.263+0000 7f8f03980700 15 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 47:135 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 49 2019-12-07T19:34:27.263+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 NullEvt 2019-12-07T19:34:27.263+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.263+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.276+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1275144 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.301+0000 7f8f21451700 10 osd.7 49 tick 2019-12-07T19:34:27.301+0000 7f8f21451700 10 osd.7 49 do_waiters -- start 2019-12-07T19:34:27.301+0000 7f8f21451700 10 osd.7 49 do_waiters -- finish 2019-12-07T19:34:27.301+0000 7f8f21451700 20 osd.7 49 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:27.325+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1275144 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.355+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 14 ==== osd_ping(ping e49 up_from 13 ping_stamp 2019-12-07T19:34:27.356295+0000/50.813218500s send_stamp 50.813218500s delta_ub 0.021810731s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b5400 con 0x558c54c19000 2019-12-07T19:34:27.355+0000 7f8f23c56700 20 osd.7 49 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021220503s,0.021810731s]) 2019-12-07T19:34:27.355+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e49 up_from 13 ping_stamp 2019-12-07T19:34:27.356295+0000/50.813218500s send_stamp 50.791997997s delta_ub 18446744073.688331113s) v5 -- 0x558c5841e400 con 0x558c54c19000 2019-12-07T19:34:27.355+0000 7f8f23c56700 10 osd.7 49 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 48 -> 49 (as per caller) 2019-12-07T19:34:27.355+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 14 ==== osd_ping(ping e49 up_from 13 ping_stamp 2019-12-07T19:34:27.356295+0000/50.813218500s send_stamp 50.813218500s delta_ub 0.021810731s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b4a00 con 0x558c54c18800 2019-12-07T19:34:27.355+0000 7f8f23c56700 20 osd.7 49 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021109461s,0.021810731s]) 2019-12-07T19:34:27.355+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e49 up_from 13 ping_stamp 2019-12-07T19:34:27.356295+0000/50.813218500s send_stamp 50.792109039s delta_ub 18446744073.688442155s) v5 -- 0x558c5841e400 con 0x558c54c18800 2019-12-07T19:34:27.376+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1275144 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.425+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1275144 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.476+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1275144 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.526+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1275144 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.576+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1275144 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.605+0000 7f8f0998c700 15 osd.7 49 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:27.605+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) 2019-12-07T19:34:27.605+0000 7f8f0998c700 15 osd.7 49 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f0998c700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) queued 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) queued 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041434037s (sent 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.4s2 pg_lease(ru 59.041434037s ub 67.041757061s int 16s) e49/49) v1 -- 0x558c54dfae00 con 0x558c54c64000 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.4s4 pg_lease(ru 59.041434037s ub 67.041757061s int 16s) e49/49) v1 -- 0x558c5795d800 con 0x558c54c19800 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041594632s (sent 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.4s1 pg_lease(ru 59.041434037s ub 67.041757061s int 16s) e49/49) v1 -- 0x558c55271200 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.4s5 pg_lease(ru 59.041434037s ub 67.041757061s int 16s) e49/49) v1 -- 0x558c55270600 con 0x558c54c5b400 2019-12-07T19:34:27.605+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.2s2 pg_lease(ru 59.041594632s ub 67.041802321s int 16s) e49/49) v1 -- 0x558c5795d200 con 0x558c54c64000 2019-12-07T19:34:27.605+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.4s3 pg_lease(ru 59.041434037s ub 67.041757061s int 16s) e49/49) v1 -- 0x558c57612200 con 0x558c54c19c00 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.2s4 pg_lease(ru 59.041594632s ub 67.041802321s int 16s) e49/49) v1 -- 0x558c580b8000 con 0x558c54c19800 2019-12-07T19:34:27.605+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.2s5 pg_lease(ru 59.041594632s ub 67.041802321s int 16s) e49/49) v1 -- 0x558c580b8200 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.2s1 pg_lease(ru 59.041594632s ub 67.041802321s int 16s) e49/49) v1 -- 0x558c580b8400 con 0x558c54c5b400 2019-12-07T19:34:27.605+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.2s3 pg_lease(ru 59.041594632s ub 67.041802321s int 16s) e49/49) v1 -- 0x558c580b8600 con 0x558c54c19c00 2019-12-07T19:34:27.605+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 95 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 67.041757061s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5734b860 con 0x558c54c19c00 2019-12-07T19:34:27.605+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.4s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) pg 0x558c548ab000 2019-12-07T19:34:27.605+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 96 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 67.041802321s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5734b860 con 0x558c54c19c00 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.2s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) pg 0x558c57e88000 2019-12-07T19:34:27.605+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.5 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 247 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 67.041802321s) e49/49) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c57edef00 con 0x558c54c5b400 2019-12-07T19:34:27.605+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 147 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 67.041757061s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54d9ef00 con 0x558c54c19800 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.2s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f24c58700 15 osd.7 49 enqueue_peering_evt 1.4s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 248 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 67.041757061s) e49/49) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c57edef00 con 0x558c54c5b400 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.4s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041594632s (sent 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 368 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 67.041757061s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5734b860 con 0x558c54c64000 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.4s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) pg 0x558c57e88000 2019-12-07T19:34:27.605+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 221 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 67.041757061s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c57edef00 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 369 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 67.041802321s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5734b860 con 0x558c54c64000 2019-12-07T19:34:27.605+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.4s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.2s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 222 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 67.041802321s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c57edef00 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 148 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 67.041802321s) e49/49) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54d9ef00 con 0x558c54c19800 2019-12-07T19:34:27.605+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.2s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f24c58700 15 osd.7 49 enqueue_peering_evt 1.2s0 epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041594632s (sent 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041434037s (sent 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) pg 0x558c57e88000 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) pg 0x558c548ab000 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041594632s (sent 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 223 ==== pg_lease(1.fs5 pg_lease(ru 59.064164481s ub 67.064394266s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8200 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.fs5 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064164481s ub 67.064394266s int 16s) 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064164481s ub 67.064394266s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041434037s (sent 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) pg 0x558c57e88000 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 224 ==== pg_lease(1.1s1 pg_lease(ru 59.064307871s ub 67.064402266s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8200 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064164481s ub 67.064394266s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.1s1 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064307871s ub 67.064402266s int 16s) 2019-12-07T19:34:27.605+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064307871s ub 67.064402266s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064164481s ub 67.064394266s int 16s)) prio 255 cost 10 e49) pg 0x558c57383000 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) pg 0x558c548ab000 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.4 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064164481s ub 67.064394266s int 16s) 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064307871s ub 67.064402266s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064307871s ub 67.064402266s int 16s)) prio 255 cost 10 e49) pg 0x558c5736a000 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 59.041594632s 2019-12-07T19:34:27.605+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.064164481s ub 67.064394266s int 16s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f04181700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.3 pg_lease(ru 59.064307871s ub 67.064402266s int 16s) 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.022346735s -> ru 59.041817746s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041594632s (sent 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f04181700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.064307871s ub 67.064402266s int 16s) 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.041817746s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.022346735s -> ru 59.041961136s 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.021969639s -> ruub 67.042424627s 2019-12-07T19:34:27.605+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.041961136s 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.042424627s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041802321s)) prio 255 cost 10 e49) pg 0x558c57e88000 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.021969639s -> ruub 67.042432627s 2019-12-07T19:34:27.605+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.fs0 pg_lease_ack(ruub 67.064394266s) e49/49) v1 -- 0x558c54d9e5a0 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041434037s (sent 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.2 pg_lease_ack(ruub 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.042432627s 2019-12-07T19:34:27.605+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.1s0 pg_lease_ack(ruub 67.064402266s) e49/49) v1 -- 0x558c581e94a0 con 0x558c54c5bc00 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.605+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) pg 0x558c548ab000 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.1 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 67.041802321s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 59.041434037s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 67.041802321s (sent 67.041802321s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 59.041434037s (sent 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041757061s)) prio 255 cost 10 e49) pg 0x558c548ab000 2019-12-07T19:34:27.605+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLeaseAck epoch 49 from osd.3 pg_lease_ack(ruub 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.3 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.1 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.5 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until peer osd.4 ruub 67.041757061s 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recalc_readable_until readable_until[_ub] 67.041757061s (sent 67.041757061s) 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:27.605+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.605+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:27.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 249 ==== pg_lease(1.3s4 pg_lease(ru 59.056742504s ub 67.056879179s int 16s) e49/49) v1 ==== 58+0+0 (unknown 0 0 0) 0x558c580b8200 con 0x558c54c5b400 2019-12-07T19:34:27.606+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.3s4 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.056742504s ub 67.056879179s int 16s) 2019-12-07T19:34:27.606+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.056742504s ub 67.056879179s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.056742504s ub 67.056879179s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.056742504s ub 67.056879179s int 16s)) prio 255 cost 10 e49) pg 0x558c5513e000 2019-12-07T19:34:27.606+0000 7f8f0317f700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.056742504s ub 67.056879179s int 16s) 2019-12-07T19:34:27.606+0000 7f8f0317f700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.056742504s ub 67.056879179s int 16s) 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.013998833s -> ru 59.042743671s 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.042743671s 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.013551348s -> ruub 67.043327831s 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.043327831s 2019-12-07T19:34:27.606+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.3s0 pg_lease_ack(ruub 67.056879179s) e49/49) v1 -- 0x558c54d9f0e0 con 0x558c54c5b400 2019-12-07T19:34:27.606+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:27.606+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:27.608+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 250 ==== pg_lease(1.bs1 pg_lease(ru 59.057851957s ub 67.057987122s int 16s) e49/49) v1 ==== 58+0+0 (unknown 0 0 0) 0x558c580b8200 con 0x558c54c5b400 2019-12-07T19:34:27.608+0000 7f8f23c56700 15 osd.7 49 enqueue_peering_evt 1.bs1 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.057851957s ub 67.057987122s int 16s) 2019-12-07T19:34:27.608+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.057851957s ub 67.057987122s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.057851957s ub 67.057987122s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.057851957s ub 67.057987122s int 16s)) prio 255 cost 10 e49) pg 0x558c575ef000 2019-12-07T19:34:27.608+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.4 pg_lease(ru 59.057851957s ub 67.057987122s int 16s) 2019-12-07T19:34:27.608+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.057851957s ub 67.057987122s int 16s) 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.013998833s -> ru 59.043853124s 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.043853124s 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.013551348s -> ruub 67.044435774s 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.044435774s 2019-12-07T19:34:27.608+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.bs0 pg_lease_ack(ruub 67.057987122s) e49/49) v1 -- 0x558c576dc1e0 con 0x558c54c5b400 2019-12-07T19:34:27.608+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:27.609+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 149 ==== pg_lease(1.5s4 pg_lease(ru 59.081729084s ub 67.081869114s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c54c19800 2019-12-07T19:34:27.609+0000 7f8f24c58700 15 osd.7 49 enqueue_peering_evt 1.5s4 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.081729084s ub 67.081869114s int 16s) 2019-12-07T19:34:27.609+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.081729084s ub 67.081869114s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.081729084s ub 67.081869114s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.081729084s ub 67.081869114s int 16s)) prio 255 cost 10 e49) pg 0x558c57522000 2019-12-07T19:34:27.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.081729084s ub 67.081869114s int 16s) 2019-12-07T19:34:27.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.081729084s ub 67.081869114s int 16s) 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.036585220s -> ru 59.045143864s 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.045143864s 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.036143956s -> ruub 67.045725158s 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.045725158s 2019-12-07T19:34:27.609+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.5s0 pg_lease_ack(ruub 67.081869114s) e49/49) v1 -- 0x558c57f112c0 con 0x558c54c19800 2019-12-07T19:34:27.609+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:27.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:27.609+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 150 ==== pg_lease(1.ds2 pg_lease(ru 59.082500675s ub 67.082631240s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c54c19800 2019-12-07T19:34:27.609+0000 7f8f24c58700 15 osd.7 49 enqueue_peering_evt 1.ds2 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.082500675s ub 67.082631240s int 16s) 2019-12-07T19:34:27.609+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.082500675s ub 67.082631240s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.082500675s ub 67.082631240s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.082500675s ub 67.082631240s int 16s)) prio 255 cost 10 e49) pg 0x558c55141000 2019-12-07T19:34:27.609+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.2 pg_lease(ru 59.082500675s ub 67.082631240s int 16s) 2019-12-07T19:34:27.609+0000 7f8f06185700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.082500675s ub 67.082631240s int 16s) 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.036585220s -> ru 59.045915455s 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.045915455s 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.036143956s -> ruub 67.046487284s 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.046487284s 2019-12-07T19:34:27.609+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.ds0 pg_lease_ack(ruub 67.082631240s) e49/49) v1 -- 0x558c57f11e00 con 0x558c54c19800 2019-12-07T19:34:27.609+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:27.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:27.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 370 ==== pg_lease(1.es4 pg_lease(ru 59.068156525s ub 67.068340184s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.es4 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068156525s ub 67.068340184s int 16s) 2019-12-07T19:34:27.610+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068156525s ub 67.068340184s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068156525s ub 67.068340184s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068156525s ub 67.068340184s int 16s)) prio 255 cost 10 e49) pg 0x558c54d98000 2019-12-07T19:34:27.610+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068156525s ub 67.068340184s int 16s) 2019-12-07T19:34:27.610+0000 7f8f05984700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.068156525s ub 67.068340184s int 16s) 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.021810731s -> ru 59.046345794s 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.046345794s 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.021109461s -> ruub 67.047230723s 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.047230723s 2019-12-07T19:34:27.610+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.es0 pg_lease_ack(ruub 67.068340184s) e49/49) v1 -- 0x558c585f8f00 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:27.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:27.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 371 ==== pg_lease(1.8s5 pg_lease(ru 59.068340106s ub 67.068518371s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8600 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.8s5 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068340106s ub 67.068518371s int 16s) 2019-12-07T19:34:27.610+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068340106s ub 67.068518371s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068340106s ub 67.068518371s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068340106s ub 67.068518371s int 16s)) prio 255 cost 10 e49) pg 0x558c57380000 2019-12-07T19:34:27.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068340106s ub 67.068518371s int 16s) 2019-12-07T19:34:27.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.068340106s ub 67.068518371s int 16s) 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.021810731s -> ru 59.046529375s 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.046529375s 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.021109461s -> ruub 67.047408910s 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.047408910s 2019-12-07T19:34:27.610+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.8s0 pg_lease_ack(ruub 67.068518371s) e49/49) v1 -- 0x558c585f81e0 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:27.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:27.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 372 ==== pg_lease(1.9s4 pg_lease(ru 59.068413248s ub 67.068600079s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.9s4 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068413248s ub 67.068600079s int 16s) 2019-12-07T19:34:27.610+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068413248s ub 67.068600079s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068413248s ub 67.068600079s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068413248s ub 67.068600079s int 16s)) prio 255 cost 10 e49) pg 0x558c5500a000 2019-12-07T19:34:27.610+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068413248s ub 67.068600079s int 16s) 2019-12-07T19:34:27.610+0000 7f8f08189700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.068413248s ub 67.068600079s int 16s) 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.021810731s -> ru 59.046602517s 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.046602517s 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.021109461s -> ruub 67.047490618s 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.047490618s 2019-12-07T19:34:27.610+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.9s0 pg_lease_ack(ruub 67.068600079s) e49/49) v1 -- 0x558c576dc960 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:27.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:27.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 373 ==== pg_lease(1.7s4 pg_lease(ru 59.068577717s ub 67.068728225s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.7s4 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068577717s ub 67.068728225s int 16s) 2019-12-07T19:34:27.610+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068577717s ub 67.068728225s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068577717s ub 67.068728225s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068577717s ub 67.068728225s int 16s)) prio 255 cost 10 e49) pg 0x558c54d5f000 2019-12-07T19:34:27.610+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.068577717s ub 67.068728225s int 16s) 2019-12-07T19:34:27.610+0000 7f8f05183700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.068577717s ub 67.068728225s int 16s) 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.021810731s -> ru 59.046766986s 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.046766986s 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.021109461s -> ruub 67.047618764s 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.047618764s 2019-12-07T19:34:27.610+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.7s0 pg_lease_ack(ruub 67.068728225s) e49/49) v1 -- 0x558c575cc5a0 con 0x558c54c64000 2019-12-07T19:34:27.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:27.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 374 ==== pg_lease(1.as5 pg_lease(ru 59.069490446s ub 67.069633703s int 16s) e49/49) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c54c64000 2019-12-07T19:34:27.611+0000 7f8f24457700 15 osd.7 49 enqueue_peering_evt 1.as5 epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.069490446s ub 67.069633703s int 16s) 2019-12-07T19:34:27.611+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.069490446s ub 67.069633703s int 16s)) prio 255 cost 10 e49) 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.069490446s ub 67.069633703s int 16s)) prio 255 cost 10 e49) queued 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.069490446s ub 67.069633703s int 16s)) prio 255 cost 10 e49) pg 0x558c57e8b000 2019-12-07T19:34:27.611+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 49 epoch_requested: 49 MLease epoch 49 from osd.1 pg_lease(ru 59.069490446s ub 67.069633703s int 16s) 2019-12-07T19:34:27.611+0000 7f8f07988700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] proc_lease pg_lease(ru 59.069490446s ub 67.069633703s int 16s) 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_ub 0.021810731s -> ru 59.047679715s 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until now 59.047679715s 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] peer_clock_delta_lb 0.021109461s -> ruub 67.048524242s 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] proc_lease readable_until_ub now 67.048524242s 2019-12-07T19:34:27.611+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.as0 pg_lease_ack(ruub 67.069633703s) e49/49) v1 -- 0x558c54d39860 con 0x558c54c64000 2019-12-07T19:34:27.611+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:27.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.625+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 251 ==== osd_map(50..50 src has 1..50) v4 ==== 633+0+0 (unknown 0 0 0) 0x558c576c0d80 con 0x558c54c5b400 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 osd.7 49 OSD::ms_dispatch: osd_map(50..50 src has 1..50) v4 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 osd.7 49 do_waiters -- start 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 osd.7 49 do_waiters -- finish 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 osd.7 49 _dispatch 0x558c576c0d80 osd_map(50..50 src has 1..50) v4 2019-12-07T19:34:27.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 252 ==== MOSDECSubOpRead(1.bs1 50/41 ECSubRead(tid=95, to_read={1:dce63cc3:::smithi13140796-50:head=0,208896,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c55113400 con 0x558c54c5b400 2019-12-07T19:34:27.625+0000 7f8f23c56700 15 osd.7 49 enqueue_op 0x558c578d29a0 prio 127 cost 0 latency 0.000019 epoch 50 MOSDECSubOpRead(1.bs1 50/41 ECSubRead(tid=95, to_read={1:dce63cc3:::smithi13140796-50:head=0,208896,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:27.625+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 50/41 ECSubRead(tid=95, to_read={1:dce63cc3:::smithi13140796-50:head=0,208896,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e50) 2019-12-07T19:34:27.625+0000 7f8f1199c700 3 osd.7 49 handle_osd_map epochs [50,50], i have 49, src has [1,50] 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 osd.7 49 handle_osd_map got inc map for epoch 50 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 50/41 ECSubRead(tid=95, to_read={1:dce63cc3:::smithi13140796-50:head=0,208896,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e50) queued 2019-12-07T19:34:27.625+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3f54e826:::osdmap.49:0# 0x0~0 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 50/41 ECSubRead(tid=95, to_read={1:dce63cc3:::smithi13140796-50:head=0,208896,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e50) pg 0x558c575ef000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d7810 blob([0xaab000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c550b3960 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c393b0 blob([!~1000,0xaac000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57c39420 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3f54e826:::osdmap.49:0# 0x0~1448 = 5192 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 osd.7 49 add_map_bl 49 5192 bytes 2019-12-07T19:34:27.625+0000 7f8f0317f700 10 osd.7 49 dequeue_op 0x558c578d29a0 prio 127 cost 0 latency 0.000054 MOSDECSubOpRead(1.bs1 50/41 ECSubRead(tid=95, to_read={1:dce63cc3:::smithi13140796-50:head=0,208896,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] 2019-12-07T19:34:27.625+0000 7f8f0317f700 10 osd.7 49 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 49 -> 50 (as per caller) 2019-12-07T19:34:27.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpRead(1.bs1 50/41 ECSubRead(tid=95, to_read={1:dce63cc3:::smithi13140796-50:head=0,208896,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:27.625+0000 7f8f0317f700 25 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:27.625+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~33000 size 0x33000 (208896) 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2f50 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57897ea0 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57566cb0 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1dc0 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2cb0 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2d20 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ad960 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3650 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b36c0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579704d0 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57971030 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3570 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b35e0 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970460 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b2070 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3490 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 osd.7 49 got_full_map 50, nothing requested 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3500 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579703f0 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576adab0 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b33b0 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3420 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970380 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ada40 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b32d0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579b3340 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970310 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b2cb0 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b2d20 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b2f50 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 osd.7 49 handle_osd_map pg_num_history pg_num_history(e50 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574f1dc0 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {50={}} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57566cb0 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ad960 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 osd.7 49 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e50 [1,50] lci=[13,50]) 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ada40 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576adab0 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57897ea0 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c5829b800 seq 41 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970310 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) need {<0x0, 0x1000> : [0x32000:0~1000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32000: 0x0 reading 0x0~1000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970380 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579703f0 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c247972f:::inc_osdmap.50:0# key 0x7f7fffffffffffffffc24797'/!inc_osdmap.50!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970460 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579704d0 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57971030 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b2070 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b32d0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3340 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b33b0 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c247972f:::inc_osdmap.50:0# 0x0~24d 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1239 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3420 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c247972f:::inc_osdmap.50:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3490 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3500 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3570 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b800 1 blobs 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b35e0 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xaad000~1000/1000,1000,0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xaad000~1000] 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b3650 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809a850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c39570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579b36c0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809a850 blob([0xaad000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39570 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c5809a850) get_ref 0x0~24d Blob(0x558c5809a850 blob([0xaad000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c39570 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c5809a850) get_ref init 0x1000, 1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c5809a850 blob([0xaad000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c57c39570 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c39588 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35788) dirty_range mark inline shard dirty 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c247972f:::inc_osdmap.50:0# 0x0~24d = 0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3ad4e826:::osdmap.50:0# key 0x7f7fffffffffffffff3ad4e8'&!osdmap.50!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:27.625+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ad4e826:::osdmap.50:0# 0x0~1448 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1240 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3ad4e826:::osdmap.50:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b800 2 blobs 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xaae000~2000/1000,2000,0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xaae000~2000] 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c396c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39650 blob([0xaae000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c396c0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c57c39650) get_ref 0x0~1000 Blob(0x558c57c39650 blob([0xaae000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c396c0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c57c39650) get_ref init 0x1000, 1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c57c39650 blob([0xaae000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c396c0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c396d8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c39730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c397a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c39730 blob([!~1000,0xaaf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c397a0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c57c39730) get_ref 0x1000~448 Blob(0x558c57c39730 blob([!~1000,0xaaf000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57c397a0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c57c39730) get_ref init 0x2000, 1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c57c39730 blob([!~1000,0xaaf000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57c397a0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57c397b8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295a08) dirty_range mark inline shard dirty 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ad4e826:::osdmap.50:0# 0x0~1448 = 0 2019-12-07T19:34:27.625+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b800 0 blobs 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:27.625+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:27.625+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:27.625+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:27.625+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829b800 0 blobs 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:27.625+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5829b800 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5829b800 onodes 0x558c53a91900,0x558c55276c80,0x558c57a35680,0x558c58295900 shared_blobs 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:27.625+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35788) update #-1:c247972f:::inc_osdmap.50:0# 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore.extentmap(0x558c57a35788) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c247972f:::inc_osdmap.50:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295a08) update #-1:3ad4e826:::osdmap.50:0# 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295a08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3ad4e826:::osdmap.50:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5829b800 allocated 0x[aad000~3000] released 0x[] 2019-12-07T19:34:27.626+0000 7f8f1199c700 10 freelist allocate 0xaad000~3000 2019-12-07T19:34:27.626+0000 7f8f1199c700 20 freelist _xor first_key 0xa80000 last_key 0xa80000 2019-12-07T19:34:27.626+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b800 prepare 2019-12-07T19:34:27.626+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5829b800 2019-12-07T19:34:27.626+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b800 aio_wait 2019-12-07T19:34:27.626+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5829b800 2019-12-07T19:34:27.626+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b800 io_done 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3375 < min_flush_size 524288 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35680 had 1 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295900 had 1 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x25f2d1~2277 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x25f2d1 0x25f2d1~2277 to file(ino 16 size 0x260000 mtime 2019-12-07T19:34:23.625498+0000 allocated 260000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~230000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _allocate len 0x1548 from 0 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _allocate len 0x1548 from 1 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd390000 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd390000~10000/10000,10000,acd390000 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x270000 to include allocated 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 55 (was clean) 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x270000 mtime 2019-12-07T19:34:27.627439+0000 allocated 270000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~240000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~240000 x_off 0x22f2d1 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2d1 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x548 and padding block with 0xab8 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x270000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b2cb0 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1278380 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57881828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f76480 space 0x558c57881828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b2d20 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57efc088 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582bab40 space 0x558c57efc088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b2f50 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57819dd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582babd0 space 0x558c57819dd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574f1dc0 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574fd518 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582bac60 space 0x558c574fd518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57566cb0 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57a073c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c582bacf0 space 0x558c57a073c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ad960 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x261548 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57e63358 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b85710 space 0x558c57e63358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ada40 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b857a0 space 0x558c581ea948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576adab0 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea5c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b85830 space 0x558c581ea5c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57897ea0 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55279828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b858c0 space 0x558c55279828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970310 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) need 0x{<0x0, 0x1000> : [0x32000:0~1000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581eabe8 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b85950 space 0x558c581eabe8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970380 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b859e0 space 0x558c581ea868 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579703f0 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea4e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b85a70 space 0x558c581ea4e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970460 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea168 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b85b00 space 0x558c581ea168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579704d0 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (55) on file(ino 16 size 0x270000 mtime 2019-12-07T19:34:27.627439+0000 allocated 270000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~240000]), flushing log 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5742bb38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c58b85b90 space 0x558c5742bb38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x270000 mtime 2019-12-07T19:34:27.627439+0000 allocated 270000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~240000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 55 len 0x4a crc 0x27cf8fcf) 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57971030 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c577f3898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x36000~1000 to file(ino 1 size 0x36000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c5813b200 space 0x558c577f3898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x36000 0x36000~1000 to file(ino 1 size 0x36000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x37000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x36000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b2070 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea248 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57f76ab0 space 0x558c581ea248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b32d0 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581eaa28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c57a08fc0 space 0x558c581eaa28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3340 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581eab08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cbdd0 space 0x558c581eab08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b33b0 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea6a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cbd40 space 0x558c581ea6a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3420 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cbcb0 space 0x558c581ea788 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3490 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cbc20 space 0x558c581ea328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3500 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea408 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cbb90 space 0x558c581ea408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3570 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57b97f28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cbb00 space 0x558c57b97f28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b35e0 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c581ea088 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x37000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cba70 space 0x558c581ea088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b3650 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574fd6d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cb9e0 space 0x558c574fd6d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579b36c0 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5742af68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 _add level 1 near 0 on buffer(0x558c583cb950 space 0x558c5742af68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 = 208896 2019-12-07T19:34:27.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000020 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_sub_read read request=208896 r=208896 len=208896 2019-12-07T19:34:27.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] get_hash_info: Getting attr on 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:27.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] get_hash_info: not in cache 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:27.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:27.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] get_hash_info: found on disk, size 208896 2019-12-07T19:34:27.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key 2019-12-07T19:34:27.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key = 0 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_sub_read: Checking hash of 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 55 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x270000 mtime 2019-12-07T19:34:27.627439+0000 allocated 270000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~240000]) 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000396356s (0.000042716s flush + 0.000353640s kv commit) 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:27.626+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 49/41 ECSubReadReply(tid=95, attrs_read=0)) v2 -- 0x558c57573b80 con 0x558c54c5b400 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5829b800> 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:27.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b800 kv_submitted 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5829b800 2019-12-07T19:34:27.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c5829b800 osr 0x558c54ea4b40 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 38 0x1b000~1000 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7 49 _committed_osd_maps 50..50 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7 49 advance to epoch 50 (<= last 50 <= newest_map 50) 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 39 0x1b000~1000 crc 102ae6d6 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 38 0x2000~1000 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 39 0x2000~1000 crc be263f09 2019-12-07T19:34:27.626+0000 7f8f0317f700 10 osd.7 50 dequeue_op 0x558c578d29a0 finish 2019-12-07T19:34:27.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:27.626+0000 7f8f141a1700 7 osd.7 50 consume_map version 50 2019-12-07T19:34:27.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.8s5 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.0s3 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.1s1 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.9s4 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.as5 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.2s0 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.bs1 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.3s4 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.cs0 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.4s0 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.5s4 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.ds2 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.es4 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.6s2 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.fs5 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 identify_splits_and_merges 1.7s4 e49 to e50 pg_nums {18=16} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 50 prune_pg_created 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:0.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:1.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:2.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:3.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:4.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:5.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:6.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7:7.consume_map 50 (was 49) 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:27.626+0000 7f8f141a1700 10 osd.7 50 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.8s5 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.0s3 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c57380000 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.1s1 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.9s4 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.as5 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.2s0 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.bs1 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c5500d000 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.3s4 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.cs0 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.4s0 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c57e8b000 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c5736a000 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c57676000 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.5s4 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.ds2 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.es4 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 49 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.6s2 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.fs5 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 49 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 49 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:27.626+0000 7f8f141a1700 15 osd.7 50 enqueue_peering_evt 1.7s4 epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) 2019-12-07T19:34:27.626+0000 7f8f141a1700 7 osd.7 50 activate_map version 50 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:27.626+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 49 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c54d98000 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c57522000 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c548ab000 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c57383000 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 49 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:27.626+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c5500a000 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c57e88000 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c54d5c000 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c55141000 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.626+0000 7f8f06986700 7 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 49 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c54d5f000 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 49 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f06986700 15 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 48:122 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 49 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:27.626+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 49 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 49 -> 50 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 49 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:27.626+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:27.626+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:27.626+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f03980700 7 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f03980700 15 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 48:136 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f04982700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: Active: handling ActMap 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:27.626+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] Active: kicking snap trim 2019-12-07T19:34:27.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:27.626+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:27.626+0000 7f8f0918b700 20 osd.7 50 agent_entry empty queue 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.626+0000 7f8f0297e700 7 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: handle ActMap primary 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] prepare_stats_for_publish reporting purged_snaps [c~1,11~1,17~1] 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f0297e700 15 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] publish_stats_to_osd 48:153 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f0197c700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 48 while current is 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.626+0000 7f8f0898a700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.626+0000 7f8f04181700 20 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.626+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:27.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.627+0000 7f8f0197c700 20 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.627+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:27.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.626+0000 7f8f04181700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.627+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f0197c700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.627+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.627+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f0898a700 20 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.627+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c575ef000 2019-12-07T19:34:27.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:27.627+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.627+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 49 -> 50 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.627+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.627+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.627+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) queued 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 50 epoch_requested: 50 NullEvt) prio 255 cost 10 e50) pg 0x558c5513e000 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 50 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 49 -> 50 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 49 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:27.627+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 50 advance_pg new pool opts old pool opts 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 50 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 50 2019-12-07T19:34:27.627+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f07187700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 50 epoch_requested: 50 NullEvt 2019-12-07T19:34:27.627+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:27.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:27.632+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 375 ==== osd_map(50..50 src has 1..50) v4 ==== 633+0+0 (crc 0 0 0) 0x558c57e03d40 con 0x558c54c64000 2019-12-07T19:34:27.632+0000 7f8f1199c700 20 osd.7 50 OSD::ms_dispatch: osd_map(50..50 src has 1..50) v4 2019-12-07T19:34:27.632+0000 7f8f1199c700 10 osd.7 50 do_waiters -- start 2019-12-07T19:34:27.632+0000 7f8f1199c700 10 osd.7 50 do_waiters -- finish 2019-12-07T19:34:27.632+0000 7f8f1199c700 20 osd.7 50 _dispatch 0x558c57e03d40 osd_map(50..50 src has 1..50) v4 2019-12-07T19:34:27.632+0000 7f8f1199c700 3 osd.7 50 handle_osd_map epochs [50,50], i have 50, src has [1,50] 2019-12-07T19:34:27.632+0000 7f8f1199c700 10 osd.7 50 no new maps here, dropping 2019-12-07T19:34:27.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 376 ==== MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=232, reqid=client.4540.0:384, at_version=50'25, trim_to=0'0, roll_forward_to=45'23)) v2 ==== 212378+0+0 (crc 0 0 0) 0x558c57e7d800 con 0x558c54c64000 2019-12-07T19:34:27.632+0000 7f8f24457700 15 osd.7 50 enqueue_op 0x558c57335e40 prio 127 cost 0 latency 0.000032 epoch 50 MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=232, reqid=client.4540.0:384, at_version=50'25, trim_to=0'0, roll_forward_to=45'23)) v2 2019-12-07T19:34:27.632+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=232, reqid=client.4540.0:384, at_version=50'25, trim_to=0'0, roll_forward_to=45'23)) v2) prio 127 cost 0 e50) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=232, reqid=client.4540.0:384, at_version=50'25, trim_to=0'0, roll_forward_to=45'23)) v2) prio 127 cost 0 e50) queued 2019-12-07T19:34:27.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=232, reqid=client.4540.0:384, at_version=50'25, trim_to=0'0, roll_forward_to=45'23)) v2) prio 127 cost 0 e50) pg 0x558c54d5f000 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 50 dequeue_op 0x558c57335e40 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=232, reqid=client.4540.0:384, at_version=50'25, trim_to=0'0, roll_forward_to=45'23)) v2 pg pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 50 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 49 -> 50 (as per caller) 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=232, reqid=client.4540.0:384, at_version=50'25, trim_to=0'0, roll_forward_to=45'23)) v2 2019-12-07T19:34:27.632+0000 7f8f05183700 20 snap_mapper.add_oid 1:ecc72bdb:::smithi13140796-10:1b 19,1a,1b 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:1b got.empty() 2019-12-07T19:34:27.632+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:1b 19,1a,1b 2019-12-07T19:34:27.632+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.632+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.632+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.632+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 45'23 (0'0,45'23] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] append_log log((0'0,45'23], crt=45'23) [50'24 (45'21) clone 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:21.618915+0000 0 snaps [1b,1a,19] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,50'25 (45'21) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:384 0.000000 0 ObjectCleanRegions clean_offsets: [835584~18446744073708716031], clean_omap: 1, new_object: 0] 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'24 (0'0,50'24] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'23 crt=45'23 lcod 45'23 active mbc={} ps=[c~1,11~1,17~1]] add_log_entry 50'24 (45'21) clone 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:21.618915+0000 0 snaps [1b,1a,19] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'23 crt=45'23 lcod 45'23 active mbc={} ps=[c~1,11~1,17~1]] add_log_entry 50'25 (45'21) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:384 0.000000 0 ObjectCleanRegions clean_offsets: [835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'23 crt=45'23 lcod 45'23 active mbc={} ps=[c~1,11~1,17~1]] append_log approx pg log length = 25 2019-12-07T19:34:27.632+0000 7f8f05183700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'23 crt=45'23 lcod 45'23 active mbc={} ps=[c~1,11~1,17~1]] append_log transaction_applied = 1 2019-12-07T19:34:27.632+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:27.632+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 50'24, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c582a8600 seq 66 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ecc72bdb:::smithi13140796-10:1b# key 0x848000000000000001ecc72bdb21736d'ithi13140796-10!='0x000000000000001bffffffffffffffff'o' 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:1b# 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1241 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.onode(0x558c54d97180).flush flush done 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# 0x0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:1b# 0x0~a000 -> 0x0~a000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d74cb0 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c5800d420 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c5800d420) get_ref 0x0~2000 Blob(0x558c5800d420 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c5800d420) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x0~2000: 0x0~2000 Blob(0x558c5800d420 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d74a80 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c574d2ee0 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c574d2ee0) get_ref 0x0~2000 Blob(0x558c574d2ee0 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c574d2ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c574d2ee0 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d74bd0 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c5829da40 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c5829da40) get_ref 0x0~2000 Blob(0x558c5829da40 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c5829da40) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c5829da40 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54d74d20 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c57f91030 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c57f91030) get_ref 0x0~2000 Blob(0x558c57f91030 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c57f91030) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57f91030 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup src 0x8000~2000: 0x0~2000 Blob(0x558c54d749a0 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup new Blob(0x558c54df88c0 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c54df88c0) get_ref 0x0~2000 Blob(0x558c54df88c0 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c54df88c0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c54df88c0 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=3)))) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2888) dirty_range mark inline shard dirty 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:1b# = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# snapset 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# snapset = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# 1 keys 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# 1 keys = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# hinfo_key (42 bytes) 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ecc72bdb:::smithi13140796-10:head#19 key 0x848000000000000001ecc72bdb21736d'ithi13140796-10!='0xfffffffffffffffe0000000000000019'o' 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:head#19 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:head#19 = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1242 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 3 keys 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 3 keys = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:27.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~33000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~33000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:27.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~33000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582a8600 26 blobs 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:27.632+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate 0x33000/1000,33000,0 2019-12-07T19:34:27.632+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xab0000~33000/1000,33000,0 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xab0000~33000] 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eb6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eb7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eb6c0 blob([0xab0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb7a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c583eb6c0) get_ref 0x0~2000 Blob(0x558c583eb6c0 blob([0xab0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb7a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c583eb6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c583eb6c0 blob([0xab0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb7a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c583eb7b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eb810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eb880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eb810 blob([0xab2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb880 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c583eb810) get_ref 0x0~2000 Blob(0x558c583eb810 blob([0xab2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb880 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c583eb810) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c583eb810 blob([0xab2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb880 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c583eb898 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eb8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b83570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eb8f0 blob([0xab4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83570 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c583eb8f0) get_ref 0x0~2000 Blob(0x558c583eb8f0 blob([0xab4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83570 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c583eb8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c583eb8f0 blob([0xab4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83570 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58b83588 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b83490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83500 blob([0xab6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83490 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83500) get_ref 0x0~2000 Blob(0x558c58b83500 blob([0xab6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83490 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83500) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c58b83500 blob([0xab6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83490 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58b834a8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b83c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83c00 blob([0xab8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83c70 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83c00) get_ref 0x0~2000 Blob(0x558c58b83c00 blob([0xab8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83c70 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83c00) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c58b83c00 blob([0xab8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83c70 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58b83c88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b83d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83ce0 blob([0xaba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83d50 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83ce0) get_ref 0x0~2000 Blob(0x558c58b83ce0 blob([0xaba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83d50 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c58b83ce0 blob([0xaba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83d50 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58b83d68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b83e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83dc0 blob([0xabc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83e30 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83dc0) get_ref 0x0~2000 Blob(0x558c58b83dc0 blob([0xabc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83e30 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c58b83dc0 blob([0xabc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83e30 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58b83e48 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b83f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83ea0 blob([0xabe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83f10 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83ea0) get_ref 0x0~2000 Blob(0x558c58b83ea0 blob([0xabe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83f10 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c58b83ea0 blob([0xabe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83f10 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58b83f28 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83f80 blob([0xac0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba000 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83f80) get_ref 0x0~2000 Blob(0x558c58b83f80 blob([0xac0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba000 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c58b83f80) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c58b83f80 blob([0xac0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba000 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba018 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba070 blob([0xac2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba0e0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba070) get_ref 0x0~2000 Blob(0x558c580ba070 blob([0xac2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba0e0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba070) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c580ba070 blob([0xac2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba0e0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba0f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba150 blob([0xac4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba1c0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba150) get_ref 0x0~2000 Blob(0x558c580ba150 blob([0xac4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba1c0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba150) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c580ba150 blob([0xac4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba1c0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba1d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba230 blob([0xac6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba2a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba230) get_ref 0x0~2000 Blob(0x558c580ba230 blob([0xac6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba2a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba230) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c580ba230 blob([0xac6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba2a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba2b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba310 blob([0xac8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba380 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba310) get_ref 0x0~2000 Blob(0x558c580ba310 blob([0xac8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba380 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba310) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c580ba310 blob([0xac8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba380 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba398 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba3f0 blob([0xaca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba460 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba3f0) get_ref 0x0~2000 Blob(0x558c580ba3f0 blob([0xaca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba460 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c580ba3f0 blob([0xaca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba460 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba478 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba4d0 blob([0xacc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba540 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba4d0) get_ref 0x0~2000 Blob(0x558c580ba4d0 blob([0xacc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba540 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c580ba4d0 blob([0xacc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba540 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba558 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba5b0 blob([0xace000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba620 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba5b0) get_ref 0x0~2000 Blob(0x558c580ba5b0 blob([0xace000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba620 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c580ba5b0 blob([0xace000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba620 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba638 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba690 blob([0xad0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba700 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba690) get_ref 0x0~2000 Blob(0x558c580ba690 blob([0xad0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba700 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba690) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c580ba690 blob([0xad0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba700 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba718 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba770 blob([0xad2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba7e0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba770) get_ref 0x0~2000 Blob(0x558c580ba770 blob([0xad2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba7e0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba770) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c580ba770 blob([0xad2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba7e0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba7f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba850 blob([0xad4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba8c0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba850) get_ref 0x0~2000 Blob(0x558c580ba850 blob([0xad4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba8c0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba850) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c580ba850 blob([0xad4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba8c0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba8d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ba930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ba9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ba930 blob([0xad6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba9a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba930) get_ref 0x0~2000 Blob(0x558c580ba930 blob([0xad6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba9a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580ba930) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c580ba930 blob([0xad6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba9a0 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580ba9b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580baa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580baa80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580baa10 blob([0xad8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baa80 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580baa10) get_ref 0x0~2000 Blob(0x558c580baa10 blob([0xad8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baa80 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580baa10) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c580baa10 blob([0xad8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580baa80 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580baa98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580baaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bab60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580baaf0 blob([0xada000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bab60 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580baaf0) get_ref 0x0~2000 Blob(0x558c580baaf0 blob([0xada000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bab60 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580baaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c580baaf0 blob([0xada000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bab60 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580bab78 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580babd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bac40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580babd0 blob([0xadc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bac40 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580babd0) get_ref 0x0~2000 Blob(0x558c580babd0 blob([0xadc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bac40 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580babd0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c580babd0 blob([0xadc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bac40 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580bac58 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580bacb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bad20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580bacb0 blob([0xade000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bad20 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580bacb0) get_ref 0x0~2000 Blob(0x558c580bacb0 blob([0xade000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bad20 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.blob(0x558c580bacb0) get_ref init 0x2000, 1000 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c580bacb0 blob([0xade000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bad20 sbid 0x0)) 2019-12-07T19:34:27.632+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580bad38 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580bad90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bae00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580bad90 blob([0xae0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bae00 sbid 0x0)) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.blob(0x558c580bad90) get_ref 0x0~2000 Blob(0x558c580bad90 blob([0xae0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bae00 sbid 0x0)) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.blob(0x558c580bad90) get_ref init 0x2000, 1000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c580bad90 blob([0xae0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bae00 sbid 0x0)) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580bae18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580bae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580baee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580bae70 blob([0xae2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baee0 sbid 0x0)) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.blob(0x558c580bae70) get_ref 0x0~1000 Blob(0x558c580bae70 blob([0xae2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baee0 sbid 0x0)) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.blob(0x558c580bae70) get_ref init 0x1000, 1000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~1000: 0x0~1000 Blob(0x558c580bae70 blob([0xae2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580baee0 sbid 0x0)) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580baef8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x33000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~33000 = 0 2019-12-07T19:34:27.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:27.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:27.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- SNA_1_000000000000001B_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:27.633+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000050.00000000000000000024' <- 0000000050.00000000000000000024 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000050.00000000000000000025' <- 0000000050.00000000000000000025 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a8600 cost 321745 (27 ios * 4000 + 213745 bytes) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a8600 onodes 0x558c54d97180,0x558c575a2780,0x558c575a2a00 shared_blobs 0x558c54d3ce70,0x558c54d74930,0x558c54d74a10,0x558c54d74af0,0x558c54d74c40 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c54d97288) update 4#1:ecc72bdb:::smithi13140796-10:head#19 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head#19 is 58064 (57920 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2888) update 4#1:ecc72bdb:::smithi13140796-10:1b# 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2888) update inline shard 138 bytes from 5 extents 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:1b# is 57903 (57759 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2b08) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2b08) update inline shard 494 bytes from 26 extents 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 1145 (645 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293c is 11 SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=3))) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293d is 11 SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=3))) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2939 is 11 SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=3))) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293a is 11 SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=3))) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293b is 11 SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=3))) 2019-12-07T19:34:27.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a8600 allocated 0x[ab0000~33000] released 0x[] 2019-12-07T19:34:27.633+0000 7f8f05183700 10 freelist allocate 0xab0000~33000 2019-12-07T19:34:27.633+0000 7f8f05183700 20 freelist _xor first_key 0xa80000 last_key 0xa80000 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8600 prepare 2019-12-07T19:34:27.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c582a8600 2019-12-07T19:34:27.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.633+0000 7f8f05183700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=45'23 crt=45'23 lcod 45'23 active mbc={} ps=[c~1,11~1,17~1]] op_applied version 50'25 2019-12-07T19:34:27.633+0000 7f8f05183700 10 osd.7 50 dequeue_op 0x558c57335e40 finish 2019-12-07T19:34:27.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.633+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8600 aio_wait 2019-12-07T19:34:27.633+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a8600 2019-12-07T19:34:27.633+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8600 io_done 2019-12-07T19:34:27.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:27.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:27.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:27.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 27320 < min_flush_size 524288 2019-12-07T19:34:27.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 60088 < min_flush_size 524288 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 92856 < min_flush_size 524288 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 120124 < min_flush_size 524288 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97180 had 1 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2780 had 1 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 120147 < min_flush_size 524288 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 120147 < min_flush_size 524288 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x270000 mtime 2019-12-07T19:34:27.627439+0000 allocated 270000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~240000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x261548~1d553 to file(ino 16 size 0x270000 mtime 2019-12-07T19:34:27.627439+0000 allocated 270000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~240000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x261548 0x261548~1d553 to file(ino 16 size 0x270000 mtime 2019-12-07T19:34:27.627439+0000 allocated 270000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~240000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _allocate len 0xea9b from 0 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _allocate len 0xea9b from 1 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd3a0000 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd3a0000~10000/10000,10000,acd3a0000 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x280000 to include allocated 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 56 (was clean) 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~250000 x_off 0x231548 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x548 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa9b and padding block with 0x565 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x280000 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x27ea9b 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (56) on file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]), flushing log 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 56 len 0x4a crc 0x6d3d1243) 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x37000~1000 to file(ino 1 size 0x37000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x37000 0x37000~1000 to file(ino 1 size 0x37000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x38000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x37000 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x38000 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:27.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000022 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 56 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000513803s (0.000041643s flush + 0.000472160s kv commit) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:27.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582a8600> 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:27.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8600 kv_submitted 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a8600 2019-12-07T19:34:27.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8600 finishing 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a8600 onodes 0x558c54d97180,0x558c575a2780,0x558c575a2a00 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba018 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc1b0 space 0x558c580ba018 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba0f8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc240 space 0x558c580ba0f8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba1d8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc2d0 space 0x558c580ba1d8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba2b8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc360 space 0x558c580ba2b8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba398 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc3f0 space 0x558c580ba398 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba478 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc480 space 0x558c580ba478 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba558 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc510 space 0x558c580ba558 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba638 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc5a0 space 0x558c580ba638 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba718 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc630 space 0x558c580ba718 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba7f8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc6c0 space 0x558c580ba7f8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba8d8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc750 space 0x558c580ba8d8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ba9b8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc7e0 space 0x558c580ba9b8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 50/41 ECSubWriteReply(tid=232, last_complete=50'25, committed=1, applied=1)) v2 -- 0x558c55045800 con 0x558c54c64000 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580baa98 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc870 space 0x558c580baa98 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bab78 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc900 space 0x558c580bab78 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bac58 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc990 space 0x558c580bac58 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bad38 in 0x558c53a59880) _finish_write discard buffer(0x558c580bca20 space 0x558c580bad38 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bae18 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcab0 space 0x558c580bae18 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580baef8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcb40 space 0x558c580baef8 0x0~1000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eb7b8 in 0x558c53a59880) _finish_write discard buffer(0x558c58b85cb0 space 0x558c583eb7b8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eb898 in 0x558c53a59880) _finish_write discard buffer(0x558c58b85d40 space 0x558c583eb898 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b834a8 in 0x558c53a59880) _finish_write discard buffer(0x558c58b85e60 space 0x558c58b834a8 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b83588 in 0x558c53a59880) _finish_write discard buffer(0x558c58b85dd0 space 0x558c58b83588 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b83c88 in 0x558c53a59880) _finish_write discard buffer(0x558c58b85ef0 space 0x558c58b83c88 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b83d68 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc000 space 0x558c58b83d68 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b83e48 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc090 space 0x558c58b83e48 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b83f28 in 0x558c53a59880) _finish_write discard buffer(0x558c580bc120 space 0x558c58b83f28 0x0~2000 writing nocache) 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a8600 done 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:27.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a8600 [] 2019-12-07T19:34:27.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:27.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:27.636+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 377 ==== MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=233, reqid=client.4540.0:384, at_version=0'0, trim_to=0'0, roll_forward_to=50'25)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e3f800 con 0x558c54c64000 2019-12-07T19:34:27.636+0000 7f8f24457700 15 osd.7 50 enqueue_op 0x558c573354a0 prio 127 cost 0 latency 0.000021 epoch 50 MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=233, reqid=client.4540.0:384, at_version=0'0, trim_to=0'0, roll_forward_to=50'25)) v2 2019-12-07T19:34:27.636+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=233, reqid=client.4540.0:384, at_version=0'0, trim_to=0'0, roll_forward_to=50'25)) v2) prio 127 cost 0 e50) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=233, reqid=client.4540.0:384, at_version=0'0, trim_to=0'0, roll_forward_to=50'25)) v2) prio 127 cost 0 e50) queued 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=233, reqid=client.4540.0:384, at_version=0'0, trim_to=0'0, roll_forward_to=50'25)) v2) prio 127 cost 0 e50) pg 0x558c54d5f000 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 osd.7 50 dequeue_op 0x558c573354a0 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=233, reqid=client.4540.0:384, at_version=0'0, trim_to=0'0, roll_forward_to=50'25)) v2 pg pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] _handle_message: MOSDECSubOpWrite(1.7s4 50/41 ECSubWrite(tid=233, reqid=client.4540.0:384, at_version=0'0, trim_to=0'0, roll_forward_to=50'25)) v2 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=45'23 active mbc={} ps=[c~1,11~1,17~1]] append_log log((0'0,50'25], crt=45'23) [] 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] rollforward: entry=50'24 (45'21) clone 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:21.618915+0000 0 snaps [1b,1a,19] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] rollforward: entry=50'25 (45'21) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:384 0.000000 0 ObjectCleanRegions clean_offsets: [835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] append_log approx pg log length = 25 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] append_log transaction_applied = 1 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:27.636+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b4300 seq 67 2019-12-07T19:34:27.636+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head#19 onode 0x558c54d97180 txc 0x558c586b4300 2019-12-07T19:34:27.636+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head#19 0x0 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74cb0) put_ref 0x0~2000 Blob(0x558c54d74cb0 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74a80) put_ref 0x0~2000 Blob(0x558c54d74a80 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74bd0) put_ref 0x0~2000 Blob(0x558c54d74bd0 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74d20) put_ref 0x0~2000 Blob(0x558c54d74d20 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore.blob(0x558c54d749a0) put_ref 0x0~2000 Blob(0x558c54d749a0 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54d97288) dirty_range mark inline shard dirty 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d74cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3e000~2000] 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d74a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x40000~2000] 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54d74bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x42000~2000] 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c54d74d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x44000~2000] 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c54d749a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=3)))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x46000~2000] 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head#19 = 0 2019-12-07T19:34:27.636+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b4300 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b4300 onodes shared_blobs 0x558c54d3ce70,0x558c54d74930,0x558c54d74a10,0x558c54d74af0,0x558c54d74c40 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293c is 11 SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293d is 11 SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2939 is 11 SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293a is 11 SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293b is 11 SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2))) 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b4300 allocated 0x[] released 0x[] 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4300 prepare 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b4300 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4300 io_done 2019-12-07T19:34:27.636+0000 7f8f0117b700 10 osd.7 50 dequeue_op 0x558c573354a0 finish 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1341 < min_flush_size 524288 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d97180 had 1 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1364 < min_flush_size 524288 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1364 < min_flush_size 524288 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x27ea9b~554 to file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x27ea9b 0x27ea9b~554 to file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~250000 x_off 0x24ea9b 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa9b 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfef and padding block with 0x11 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x27efef 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:27.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000091592s (0.000000113s flush + 0.000091479s kv commit) 2019-12-07T19:34:27.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586b4300> 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:27.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4300 kv_submitted 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b4300 2019-12-07T19:34:27.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4300 finishing 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b4300 onodes 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b4300 done 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:27.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b4300 [] 2019-12-07T19:34:27.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:27.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:27.636+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 50/41 ECSubWriteReply(tid=233, last_complete=50'25, committed=1, applied=1)) v2 -- 0x558c54c59800 con 0x558c54c64000 2019-12-07T19:34:27.636+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:27.676+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:27.676+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:27.676+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 39 0x2000~1000 2019-12-07T19:34:27.676+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc be263f09 2019-12-07T19:34:27.676+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 39 0x1b000~1000 2019-12-07T19:34:27.676+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 102ae6d6 2019-12-07T19:34:27.676+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.676+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:27.676+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 10 osd.7 50 tick_without_osd_lock 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f51d000/0x40000000/0x1680000000, data 0xc21e22/0xae1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 25 osd.7 50 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175569+0000 last_rx_front 2019-12-07T19:34:26.175589+0000 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 25 osd.7 50 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175651+0000 last_rx_front 2019-12-07T19:34:26.175543+0000 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 25 osd.7 50 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175579+0000 last_rx_front 2019-12-07T19:34:26.175631+0000 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 25 osd.7 50 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175478+0000 last_rx_front 2019-12-07T19:34:26.175520+0000 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 25 osd.7 50 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:26.175254+0000 last_rx_back 2019-12-07T19:34:26.175607+0000 last_rx_front 2019-12-07T19:34:26.175507+0000 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 20 osd.7 50 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 10 osd.7 50 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 20 osd.7 50 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:27.709+0000 7f8f1fbbe700 10 osd.7 50 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:27.726+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.777+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.826+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.874+0000 7f8f0097a700 5 osd.7 50 heartbeat osd_stat(store_statfs(0x163f51d000/0x40000000/0x1680000000, data 0xc21e22/0xae1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5] op hist []) 2019-12-07T19:34:27.874+0000 7f8f0097a700 20 osd.7 50 check_full_status cur ratio 0.000118128, physical ratio 0.000118128, new state none 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.688442155s) v5 -- 0x558c5841f200 con 0x558c54c9e000 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.688442155s) v5 -- 0x558c55250800 con 0x558c54c9e400 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.673407660s) v5 -- 0x558c55251200 con 0x558c551b7800 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.673407660s) v5 -- 0x558c575b8400 con 0x558c551b7c00 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.687581977s) v5 -- 0x558c576b4e00 con 0x558c552c2000 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.687581977s) v5 -- 0x558c576b5e00 con 0x558c552c2400 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.696000268s) v5 -- 0x558c576b4400 con 0x558c552c2800 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.696000268s) v5 -- 0x558c576b5000 con 0x558c552c2c00 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.678874370s) v5 -- 0x558c57518a00 con 0x558c54c65800 2019-12-07T19:34:27.874+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.311001494s delta_ub 18446744073.678874370s) v5 -- 0x558c57518600 con 0x558c54c65c00 2019-12-07T19:34:27.874+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 15 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.324927642s delta_ub 0.013926148s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c552c2800 2019-12-07T19:34:27.874+0000 7f8f24c58700 25 osd.7 50 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:26.175478+0000 -> 2019-12-07T19:34:27.875766+0000 last_rx_front 2019-12-07T19:34:26.175520+0000 2019-12-07T19:34:27.874+0000 7f8f24c58700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013721654s,0.013926148s]) 2019-12-07T19:34:27.874+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 11 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.342041745s delta_ub 0.031040251s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8200 con 0x558c54c65800 2019-12-07T19:34:27.874+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 15 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.324978073s delta_ub 0.013976579s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c552c2c00 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:26.175607+0000 -> 2019-12-07T19:34:27.875796+0000 last_rx_front 2019-12-07T19:34:26.175507+0000 2019-12-07T19:34:27.874+0000 7f8f23c56700 10 osd.7 50 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 48 -> 50 (as per caller) 2019-12-07T19:34:27.874+0000 7f8f23c56700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030805829s,0.031040251s]) 2019-12-07T19:34:27.874+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 11 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.342058784s delta_ub 0.031057290s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8200 con 0x558c54c65c00 2019-12-07T19:34:27.874+0000 7f8f24457700 25 osd.7 50 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875766+0000 last_rx_front 2019-12-07T19:34:26.175520+0000 -> 2019-12-07T19:34:27.875813+0000 2019-12-07T19:34:27.874+0000 7f8f24457700 25 osd.7 50 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:27.875562+0000) and older pending ping(s) 2019-12-07T19:34:27.874+0000 7f8f24457700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013725199s,0.013976579s]) 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875796+0000 last_rx_front 2019-12-07T19:34:26.175507+0000 -> 2019-12-07T19:34:27.875837+0000 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:27.875562+0000) and older pending ping(s) 2019-12-07T19:34:27.874+0000 7f8f23c56700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030782261s,0.031057290s]) 2019-12-07T19:34:27.874+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 15 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.347589547s delta_ub 0.036588053s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c551b7c00 2019-12-07T19:34:27.874+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 11 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.332754981s delta_ub 0.021753487s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8200 con 0x558c54c9e000 2019-12-07T19:34:27.874+0000 7f8f24c58700 25 osd.7 50 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:26.175651+0000 last_rx_front 2019-12-07T19:34:26.175543+0000 -> 2019-12-07T19:34:27.875870+0000 2019-12-07T19:34:27.874+0000 7f8f24c58700 10 osd.7 50 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 48 -> 50 (as per caller) 2019-12-07T19:34:27.874+0000 7f8f24c58700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036279602s,0.036588053s]) 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:26.175569+0000 -> 2019-12-07T19:34:27.875883+0000 last_rx_front 2019-12-07T19:34:26.175589+0000 2019-12-07T19:34:27.874+0000 7f8f23c56700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021431864s,0.021753487s]) 2019-12-07T19:34:27.874+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 15 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.333422209s delta_ub 0.022420715s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59800 con 0x558c552c2000 2019-12-07T19:34:27.874+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 11 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.332793658s delta_ub 0.021792164s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c54c9e400 2019-12-07T19:34:27.874+0000 7f8f24457700 25 osd.7 50 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:26.175579+0000 -> 2019-12-07T19:34:27.875893+0000 last_rx_front 2019-12-07T19:34:26.175631+0000 2019-12-07T19:34:27.874+0000 7f8f24457700 10 osd.7 50 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 48 -> 50 (as per caller) 2019-12-07T19:34:27.874+0000 7f8f24457700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022089856s,0.022420715s]) 2019-12-07T19:34:27.874+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 15 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.333340967s delta_ub 0.022339473s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8200 con 0x558c552c2400 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875893+0000 last_rx_front 2019-12-07T19:34:26.175631+0000 -> 2019-12-07T19:34:27.875903+0000 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:27.875562+0000) and older pending ping(s) 2019-12-07T19:34:27.874+0000 7f8f23c56700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021998590s,0.022339473s]) 2019-12-07T19:34:27.874+0000 7f8f24c58700 25 osd.7 50 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875883+0000 last_rx_front 2019-12-07T19:34:26.175589+0000 -> 2019-12-07T19:34:27.875916+0000 2019-12-07T19:34:27.874+0000 7f8f24c58700 25 osd.7 50 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:27.875562+0000) and older pending ping(s) 2019-12-07T19:34:27.874+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 15 ==== osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:27.875562+0000/51.311001494s send_stamp 51.347554562s delta_ub 0.036553068s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8200 con 0x558c551b7800 2019-12-07T19:34:27.874+0000 7f8f24c58700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021438294s,0.021792164s]) 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:26.175651+0000 -> 2019-12-07T19:34:27.875931+0000 last_rx_front 2019-12-07T19:34:27.875870+0000 2019-12-07T19:34:27.874+0000 7f8f23c56700 25 osd.7 50 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:27.875562+0000) and older pending ping(s) 2019-12-07T19:34:27.874+0000 7f8f23c56700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036184055s,0.036553068s]) 2019-12-07T19:34:27.877+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.927+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:27.977+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.027+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.077+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.127+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.177+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.227+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.277+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.327+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.342+0000 7f8f21451700 10 osd.7 50 tick 2019-12-07T19:34:28.342+0000 7f8f21451700 10 osd.7 50 do_waiters -- start 2019-12-07T19:34:28.342+0000 7f8f21451700 10 osd.7 50 do_waiters -- finish 2019-12-07T19:34:28.342+0000 7f8f21451700 20 osd.7 50 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:28.362+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 11 ==== osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:28.363176+0000/51.812402990s send_stamp 51.812402990s delta_ub 0.013976579s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c54c65400 2019-12-07T19:34:28.362+0000 7f8f24c58700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013572010s,0.013976579s]) 2019-12-07T19:34:28.362+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:28.363176+0000/51.812402990s send_stamp 51.798830980s delta_ub 18446744073.695979606s) v5 -- 0x558c576b4400 con 0x558c54c65400 2019-12-07T19:34:28.362+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 11 ==== osd_ping(ping e50 up_from 13 ping_stamp 2019-12-07T19:34:28.363176+0000/51.812402990s send_stamp 51.812402990s delta_ub 0.013976579s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57518800 con 0x558c54c65000 2019-12-07T19:34:28.362+0000 7f8f24c58700 20 osd.7 50 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013503361s,0.013976579s]) 2019-12-07T19:34:28.362+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e50 up_from 13 ping_stamp 2019-12-07T19:34:28.363176+0000/51.812402990s send_stamp 51.798899629s delta_ub 18446744073.696048255s) v5 -- 0x558c576b4400 con 0x558c54c65000 2019-12-07T19:34:28.377+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.428+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.478+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.528+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.578+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1294943 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 253 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:28.605+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.bs1 epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:28.605+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) pg 0x558c575ef000 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 51, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 254 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:28.605+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.3s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:28.605+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.605+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 255 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:28.605+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.605+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.bs1 epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:28.605+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.605+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.605+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.605+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) pg 0x558c5513e000 2019-12-07T19:34:28.605+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 51, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.605+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) pg 0x558c575ef000 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 51, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.605+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 256 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:28.606+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.3s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:28.606+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) pg 0x558c5513e000 2019-12-07T19:34:28.606+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 51, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.610+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 225 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:28.610+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.1s1 epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:28.610+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.610+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 226 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:28.610+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.6s2 epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:28.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.610+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) pg 0x558c5736a000 2019-12-07T19:34:28.610+0000 7f8f08189700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 51, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) pg 0x558c54d5c000 2019-12-07T19:34:28.610+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 51, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.611+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 227 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:28.611+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.fs5 epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:28.611+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) pg 0x558c57383000 2019-12-07T19:34:28.611+0000 7f8f05183700 20 osd.7 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 51, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.611+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 228 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:28.611+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.1s1 epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:28.611+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) pg 0x558c5736a000 2019-12-07T19:34:28.611+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 51, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.611+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 229 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:28.611+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.6s2 epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:28.611+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.611+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 230 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:28.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) pg 0x558c54d5c000 2019-12-07T19:34:28.611+0000 7f8f0197c700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 51, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.611+0000 7f8f23c56700 15 osd.7 50 enqueue_peering_evt 1.fs5 epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:28.611+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) pg 0x558c57383000 2019-12-07T19:34:28.611+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 51, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 378 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.612+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.8s5 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:28.612+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 379 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.612+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.0s3 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:28.612+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 380 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.612+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.as5 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:28.612+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 381 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.612+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.es4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:28.612+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 382 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.612+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.612+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.612+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) pg 0x558c54d98000 2019-12-07T19:34:28.612+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.es4 peering, item epoch is 51, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.7s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:28.612+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.612+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 383 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.612+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.as5 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:28.612+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 384 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.612+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.612+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.9s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:28.612+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.612+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) pg 0x558c57e8b000 2019-12-07T19:34:28.612+0000 7f8f07988700 20 osd.7 op_wq _add_slot_waiter 1.as5 peering, item epoch is 51, will wait on OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.612+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.612+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 385 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) pg 0x558c57e8b000 2019-12-07T19:34:28.613+0000 7f8f07988700 20 osd.7 op_wq _add_slot_waiter 1.as5 peering, item epoch is 51, will wait on OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:28.613+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.8s5 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:28.613+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 386 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.613+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.0s3 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:28.613+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.613+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) pg 0x558c54d5f000 2019-12-07T19:34:28.613+0000 7f8f05183700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 51, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 387 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.613+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.613+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.es4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:28.613+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) pg 0x558c5500a000 2019-12-07T19:34:28.613+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f08189700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 51, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) pg 0x558c57380000 2019-12-07T19:34:28.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 388 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c64000 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 51, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.613+0000 7f8f24457700 15 osd.7 50 enqueue_peering_evt 1.9s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) pg 0x558c5500d000 2019-12-07T19:34:28.613+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 51, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) pg 0x558c57380000 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 51, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) pg 0x558c5500d000 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 51, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.613+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.613+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) pg 0x558c54d98000 2019-12-07T19:34:28.613+0000 7f8f0197c700 20 osd.7 op_wq _add_slot_waiter 1.es4 peering, item epoch is 51, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.613+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.613+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 389 ==== osd_map(51..51 src has 1..51) v4 ==== 721+0+0 (crc 0 0 0) 0x558c54d2efc0 con 0x558c54c64000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 osd.7 50 OSD::ms_dispatch: osd_map(51..51 src has 1..51) v4 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 osd.7 50 do_waiters -- start 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 osd.7 50 do_waiters -- finish 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 osd.7 50 _dispatch 0x558c54d2efc0 osd_map(51..51 src has 1..51) v4 2019-12-07T19:34:28.613+0000 7f8f1199c700 3 osd.7 50 handle_osd_map epochs [51,51], i have 50, src has [1,51] 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 osd.7 50 handle_osd_map got inc map for epoch 51 2019-12-07T19:34:28.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ad4e826:::osdmap.50:0# 0x0~0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c39650 blob([0xaae000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57c396c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57c39730 blob([!~1000,0xaaf000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57c397a0 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ad4e826:::osdmap.50:0# 0x0~1448 = 5192 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 osd.7 50 add_map_bl 50 5192 bytes 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 osd.7 50 got_full_map 51, nothing requested 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 osd.7 50 handle_osd_map pg_num_history pg_num_history(e51 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {51={1=[c~1,11~1,17~1]}} 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) lower_bound to PSN__1_000000000000000b key 0x00000000000000000000000000000403'.PSN__1_000000000000000b' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) valid is at 0x00000000000000000000000000000403'.PSN__1_000000000000000d' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 snap_mapper._lookup_purged_snap pool 1 snap b found [d,e), no overlap 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) lower_bound to PSN__1_000000000000000d key 0x00000000000000000000000000000403'.PSN__1_000000000000000d' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) valid is at 0x00000000000000000000000000000403'.PSN__1_000000000000000d' 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps [c,d) - join with later [d,e) 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) lower_bound to PSN__1_0000000000000010 key 0x00000000000000000000000000000403'.PSN__1_0000000000000010' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000010' 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) lower_bound to PSN__1_0000000000000012 key 0x00000000000000000000000000000403'.PSN__1_0000000000000012' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 snap_mapper._lookup_purged_snap pool 1 snap 12 key 'PSN__1_0000000000000012' lower_bound not found 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps [11,12) - join with earlier [10,11) 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) lower_bound to PSN__1_0000000000000016 key 0x00000000000000000000000000000403'.PSN__1_0000000000000016' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 snap_mapper._lookup_purged_snap pool 1 snap 16 key 'PSN__1_0000000000000016' lower_bound not found 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) lower_bound to PSN__1_0000000000000018 key 0x00000000000000000000000000000403'.PSN__1_0000000000000018' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.OmapIteratorImpl(0x558c5517b100) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 snap_mapper._lookup_purged_snap pool 1 snap 18 key 'PSN__1_0000000000000018' lower_bound not found 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 1 keys, set 3 keys 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 osd.7 50 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e51 [1,51] lci=[13,51]) 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c580d2900 seq 42 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c8c7972f:::inc_osdmap.51:0# key 0x7f7fffffffffffffffc8c797'/!inc_osdmap.51!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c8c7972f:::inc_osdmap.51:0# 0x0~2a5 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1243 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c8c7972f:::inc_osdmap.51:0# 0x0~2a5 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~2a5 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd5b on front/back, now 0x0~1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2900 1 blobs 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xae3000~1000/1000,1000,0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xae3000~1000] 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b28c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809bab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b28c0 blob([0xae3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bab0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c550b28c0) get_ref 0x0~2a5 Blob(0x558c550b28c0 blob([0xae3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bab0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c550b28c0) get_ref init 0x1000, 1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2a5: 0x0~2a5 Blob(0x558c550b28c0 blob([0xae3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2a5) SharedBlob(0x558c5809bab0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5809bac8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2a5 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295788) dirty_range mark inline shard dirty 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c8c7972f:::inc_osdmap.51:0# 0x0~2a5 = 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3714e826:::osdmap.51:0# key 0x7f7fffffffffffffff3714e8'&!osdmap.51!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3714e826:::osdmap.51:0# 0x0~1480 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1244 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3714e826:::osdmap.51:0# 0x0~1480 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~480 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb80 on front/back, now 0x0~1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2900 2 blobs 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xae4000~2000/1000,2000,0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xae4000~2000] 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b838f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83960 blob([0xae4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b838f0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c58b83960) get_ref 0x0~1000 Blob(0x558c58b83960 blob([0xae4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b838f0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c58b83960) get_ref init 0x1000, 1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c58b83960 blob([0xae4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b838f0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c58b83908 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b83880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b83810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b83880 blob([!~1000,0xae5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83810 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c58b83880) get_ref 0x1000~480 Blob(0x558c58b83880 blob([!~1000,0xae5000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c58b83810 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c58b83880) get_ref init 0x2000, 1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~480: 0x1000~480 Blob(0x558c58b83880 blob([!~1000,0xae5000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,480]) SharedBlob(0x558c58b83810 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c58b83828 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1480 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295008) dirty_range mark inline shard dirty 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3714e826:::osdmap.51:0# 0x0~1480 = 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2900 0 blobs 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000403'.PSN__1_0000000000000010' <- PSN__1_0000000000000010 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_000000000000000d' <- PSN__1_000000000000000d 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000011' <- PSN__1_0000000000000011 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000017' <- PSN__1_0000000000000017 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2900 0 blobs 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d2900 cost 23598 (4 ios * 4000 + 7598 bytes) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d2900 onodes 0x558c53a91900,0x558c55276c80,0x558c58294f00,0x558c58295680 shared_blobs 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295008) update #-1:3714e826:::osdmap.51:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295008) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3714e826:::osdmap.51:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295788) update #-1:c8c7972f:::inc_osdmap.51:0# 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295788) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c8c7972f:::inc_osdmap.51:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d2900 allocated 0x[ae3000~3000] released 0x[] 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 freelist allocate 0xae3000~3000 2019-12-07T19:34:28.613+0000 7f8f1199c700 20 freelist _xor first_key 0xa80000 last_key 0xa80000 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2900 prepare 2019-12-07T19:34:28.613+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c580d2900 2019-12-07T19:34:28.613+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.613+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {51=} 2019-12-07T19:34:28.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) pg 0x558c5500a000 2019-12-07T19:34:28.614+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 51, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2900 aio_wait 2019-12-07T19:34:28.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d2900 2019-12-07T19:34:28.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2900 io_done 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4113 < min_flush_size 524288 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9051 < min_flush_size 524288 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58294f00 had 1 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295680 had 1 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3858235392 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998118) bluestore 86 GiB free (0.959881), bluefs_ratio 0.0399329 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9110 < min_flush_size 524288 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9110 < min_flush_size 524288 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x27efef~2396 to file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x27efef 0x27efef~2396 to file(ino 16 size 0x280000 mtime 2019-12-07T19:34:27.635325+0000 allocated 280000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~250000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _allocate len 0x1385 from 0 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _allocate len 0x1385 from 1 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd3b0000 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd3b0000~10000/10000,10000,acd3b0000 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x290000 to include allocated 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 57 (was clean) 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x24efef 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfef 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x385 and padding block with 0xc7b 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x290000 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x281385 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (57) on file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]), flushing log 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 57 len 0x4a crc 0x44a23820) 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x38000~1000 to file(ino 1 size 0x38000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x38000 0x38000~1000 to file(ino 1 size 0x38000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x39000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x38000 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x39000 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:28.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000026 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 57 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000766656s (0.000045088s flush + 0.000721568s kv commit) 2019-12-07T19:34:28.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d2900> 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c58141400> 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2900 kv_submitted 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d2900 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c580d2900 osr 0x558c54ea4b40 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 40 0x1b000~1000 crc af54a2b 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 40 0x2000~1000 crc 51d2b34 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6000 deferred_cleanup 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b6000 onodes 0x558c53a91900,0x558c55276c80,0x558c5738d900,0x558c57f35680 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54677128 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2ab0 space 0x558c54677128 0x1000~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b2088 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2990 space 0x558c550b2088 0x0~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d358 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2a20 space 0x558c5800d358 0x0~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6000 done 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829b500 deferred_cleanup 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b6000 [] 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b500 deferred_cleanup 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5829b500 onodes 0x558c53a91900,0x558c55276c80,0x558c57572f00,0x558c57e8f680 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3978 in 0x558c53a59340) _finish_write discard buffer(0x558c579fbc20 space 0x558c550b3978 0x0~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39438 in 0x558c53a59340) _finish_write discard buffer(0x558c58b850e0 space 0x558c57c39438 0x1000~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580d77b8 in 0x558c53a59340) _finish_write discard buffer(0x558c579fbb90 space 0x558c580d77b8 0x0~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829b500 done 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829b800 deferred_cleanup 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5829b500 [] 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829b800 deferred_cleanup 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5829b800 onodes 0x558c53a91900,0x558c55276c80,0x558c57a35680,0x558c58295900 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c39588 in 0x558c53a59340) _finish_write discard buffer(0x558c58b85290 space 0x558c57c39588 0x0~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c396d8 in 0x558c53a59340) _finish_write discard buffer(0x558c58b85320 space 0x558c57c396d8 0x0~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c397b8 in 0x558c53a59340) _finish_write discard buffer(0x558c58b853b0 space 0x558c57c397b8 0x1000~1000 writing nocache) 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829b800 done 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2900 deferred_queued 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5829b800 [] 2019-12-07T19:34:28.614+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7 50 _committed_osd_maps 51..51 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7 50 advance to epoch 51 (<= last 51 <= newest_map 51) 2019-12-07T19:34:28.615+0000 7f8f141a1700 7 osd.7 51 consume_map version 51 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.8s5 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.0s3 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.1s1 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.9s4 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.as5 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.2s0 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.bs1 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.3s4 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.cs0 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.4s0 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.5s4 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.ds2 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.es4 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.6s2 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.fs5 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 identify_splits_and_merges 1.7s4 e50 to e51 pg_nums {18=16} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 51 prune_pg_created 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:0.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:1.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:2.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.as5 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:3.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) pg 0x558c57e8b000 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:4.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:5.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:6.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7:7.consume_map 51 (was 50) 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 51 <= 51, requeueing 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {51=} 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f141a1700 10 osd.7 51 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.8s5 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.0s3 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.1s1 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.9s4 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.as5 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.2s0 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.bs1 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.3s4 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.cs0 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.4s0 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.5s4 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.ds2 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.es4 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.6s2 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.fs5 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f141a1700 15 osd.7 51 enqueue_peering_evt 1.7s4 epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f141a1700 7 osd.7 51 activate_map version 51 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c57676000 2019-12-07T19:34:28.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c54ddec00 seq 15 2019-12-07T19:34:28.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c54ddec00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c54ddec00 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c54ddec00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c57522000 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 prepare 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c54ddec00 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 io_done 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1] ps=[c~1,11~1,17~1]] on_active_advmap new removed_snaps [19~1], snap_trimq now [19~1] 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] on_active_advmap new purged_snaps [c~1,11~1,17~1], now [] 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e51) pg 0x558c57e8b000 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] share_pg_info 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 50 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c574e7800 con 0x558c54c64000 2019-12-07T19:34:28.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c582b7200 seq 16 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:28.615+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c574e4000 con 0x558c54c19800 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b7200 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b7200 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c574e5500 con 0x558c54c5bc00 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b7200 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7200 prepare 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b7200 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7200 io_done 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c57ed3c00 con 0x558c54c5b400 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c57e8b000 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c57ed2000 con 0x558c54c19c00 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c57e88000 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c55141000 2019-12-07T19:34:28.615+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] publish_stats_to_osd 49:154 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 50 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 50 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1] ps=[c~1,11~1,17~1]] on_active_advmap new removed_snaps [19~1], snap_trimq now [19~1] 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] on_active_advmap new purged_snaps [c~1,11~1,17~1], now [] 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] share_pg_info 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] state: Active: handling ActMap 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c54e07100 con 0x558c54c64000 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] Active: kicking snap trim 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c574e4700 con 0x558c54c19800 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] exit NotTrimming 2019-12-07T19:34:28.615+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] enter Trimming 2019-12-07T19:34:28.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c576dee00 con 0x558c54c5bc00 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) pg 0x558c54d5c000 2019-12-07T19:34:28.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c58123500 con 0x558c54c5b400 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57390e00 con 0x558c54c19c00 2019-12-07T19:34:28.615+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] publish_stats_to_osd 50:155 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:28.615+0000 7f8f06986700 7 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] state: handle ActMap primary 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f03980700 15 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] publish_stats_to_osd 49:137 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 50 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:28.615+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] publish_stats_to_osd 51:156 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 2 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2196 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] activate_map: Not dirtying info: last_persisted is 48 while current is 51 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2219 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2219 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:28.615+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x281385~8ab to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x281385 0x281385~8ab to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x251385 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x385 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c582ac300 seq 45 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc30 and padding block with 0x3d0 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] state: Active: handling ActMap 2019-12-07T19:34:28.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ac300 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ac300 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] Active: kicking snap trim 2019-12-07T19:34:28.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ac300 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:28.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac300 prepare 2019-12-07T19:34:28.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ac300 2019-12-07T19:34:28.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac300 io_done 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x281c30 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] exit NotTrimming 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c548ab000 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] enter Trimming 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000097158s (0.000000130s flush + 0.000097028s kv commit) 2019-12-07T19:34:28.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55115500 seq 92 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:28.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] SnapTrimmer state: NotTrimming: trimming 19 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim_wait trimq=[19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f03980700 15 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] publish_stats_to_osd 50:138 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55115500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55115500 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f03980700 7 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] state: handle ActMap primary 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55115500 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] publish_stats_to_osd 51:157 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x281c30~465 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x281c30 0x281c30~465 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115500 prepare 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55115500 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x251c30 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115500 io_done 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc30 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 50 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x95 and padding block with 0xf6b 2019-12-07T19:34:28.615+0000 7f8f0b990700 10 osd.7 51 queueing pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean trimq=[19~1]] for snaptrim 2019-12-07T19:34:28.615+0000 7f8f03980700 15 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] publish_stats_to_osd 51:139 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=51) prio 5 cost 1048576 e51) 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[c~1,11~1,17~1]] state: Active advmap 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e51) pg 0x558c54d5c000 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.6s0( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1] ps=[c~1,11~1,17~1]] on_active_advmap new removed_snaps [19~1], snap_trimq now [19~1] 2019-12-07T19:34:28.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] publish_stats_to_osd 51:158 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] on_active_advmap new purged_snaps [c~1,11~1,17~1], now [] 2019-12-07T19:34:28.615+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] share_pg_info 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c582b7500 seq 24 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c55115800 seq 93 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=51) prio 5 cost 1048576 e51) queued 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c54ddec00,0x558c582b7200> 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57ed5800 con 0x558c54c64000 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=51) prio 5 cost 1048576 e51) pg 0x558c57676000 2019-12-07T19:34:28.615+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55115800 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 kv_submitted 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] snap_trimmer posting 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55115800 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c54ddec00 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55115800 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c54ddec00 finishing 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b7500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 19 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c54ddec00 onodes 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115800 prepare 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55115800 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b7500 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c54ddec00 done 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b7500 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115800 io_done 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b7200 kv_submitted 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c54ddec00 [] 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) pg 0x558c5500d000 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7500 prepare 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7200 kv_submitted 2019-12-07T19:34:28.615+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b7500 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b7200 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7500 io_done 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7200 finishing 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) pg 0x558c54d98000 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b7200 onodes 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b7200 done 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b7200 [] 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c5737b100 con 0x558c54c19800 2019-12-07T19:34:28.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1ee80) upper_bound after SNA_1_0000000000000019_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.C' 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) pg 0x558c5513e000 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1ee80) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57378e00 con 0x558c54c5bc00 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000019_.0_0000000000000001.C) returns 0 SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239600 in raw 0x558c55239600 len 69 nref 1) ) 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 50 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c5737b800 con 0x558c54c5b400 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57392a00 con 0x558c54c19c00 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1ee80) upper_bound after SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. key 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 50 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57d1ee80) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..) returns 0 SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] publish_stats_to_osd 49:123 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_object_context: found obc in cache: 0x558c5738f8c0 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) pg 0x558c54d5f000 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_object_context: 0x558c5738f8c0 1:33f5bf01:::smithi13140796-2:19 rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:19(48'31 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[19,15,13,e]} 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 50 -> 51 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[19,15,13,e]} 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x282095 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 1:33f5bf01:::smithi13140796-2:19 old_snaps e,13,15,19 old snapset 19=[]:{19=[19,15,13,e]} 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000239134s (0.000000077s flush + 0.000239057s kv commit) 2019-12-07T19:34:28.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:28.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] simple_opc_create 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c582b7b00 seq 16 2019-12-07T19:34:28.615+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:28.615+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 1:33f5bf01:::smithi13140796-2:19 snaps e,13,15,19 -> e,13,15 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 2 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c55122900 seq 41 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2196 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b7b00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b7b00 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3298 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7b00 prepare 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b7b00 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7b00 io_done 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3321 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3321 < min_flush_size 524288 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55122900 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55122900 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x282095~cf9 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55122900 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 50 while current is 51 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x282095 0x282095~cf9 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] SnapTrimmer state: NotTrimming: trimming 19 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e51) pg 0x558c54d98000 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122900 prepare 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x252095 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55122900 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x95 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122900 io_done 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd8e and padding block with 0x272 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s)) prio 255 cost 10 e51) pg 0x558c5500d000 2019-12-07T19:34:28.615+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] publish_stats_to_osd 51:140 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582ac300> 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 60.076769972s ub 60.076769972s int 16s) 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac300 kv_submitted 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ac300 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b4600 seq 68 2019-12-07T19:34:28.615+0000 7f8f0b990700 10 osd.7 51 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[19~1]] for snaptrim 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac300 finishing 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c575c4000 seq 17 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ac300 onodes 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=51) prio 5 cost 1048576 e51) 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ac300 done 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ac300 [] 2019-12-07T19:34:28.615+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:28.615+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:28.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c4000 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.615+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:28.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[19~1]] publish_stats_to_osd 51:141 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c4000 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c55122c00 seq 42 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c4000 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c57d0b800 seq 58 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b4600 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b4600 onodes shared_blobs 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b4600 allocated 0x[] released 0x[] 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4000 prepare 2019-12-07T19:34:28.615+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c4000 2019-12-07T19:34:28.615+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4000 io_done 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4600 prepare 2019-12-07T19:34:28.615+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b4600 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.615+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4600 io_done 2019-12-07T19:34:28.615+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:28.615+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:28.615+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.615+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x282d8e 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55122c00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) pg 0x558c57383000 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55122c00 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55122c00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0b800 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0b800 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c54d98000 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0b800 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000149828s (0.000000084s flush + 0.000149744s kv commit) 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b800 prepare 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0b800 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b800 io_done 2019-12-07T19:34:28.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 390 ==== MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=235, reqid=client.4540.0:389, at_version=51'41, trim_to=0'0, roll_forward_to=45'39)) v2 ==== 2992+0+0 (crc 0 0 0) 0x558c54e40800 con 0x558c54c64000 2019-12-07T19:34:28.616+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c57334000 prio 127 cost 0 latency 0.000027 epoch 51 MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=235, reqid=client.4540.0:389, at_version=51'41, trim_to=0'0, roll_forward_to=45'39)) v2 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 50 -> 51 2019-12-07T19:34:28.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=235, reqid=client.4540.0:389, at_version=51'41, trim_to=0'0, roll_forward_to=45'39)) v2) prio 127 cost 0 e51) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c54d5c000 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 2 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e51) pg 0x558c5513e000 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 50 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2204 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f05984700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 2 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3298 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:28.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 1:33f5bf01:::smithi13140796-2:19 new snapset 19=[]:{19=[15,13,e]} on 1:33f5bf01:::smithi13140796-2:head(48'32 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4400 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4423 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4423 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 1:33f5bf01:::smithi13140796-2:19 writing updated snapset on 1:33f5bf01:::smithi13140796-2:head, snapset is 19=[]:{19=[15,13,e]} 2019-12-07T19:34:28.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 391 ==== MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=234, reqid=osd.1.0:234, at_version=51'27, trim_to=0'0, roll_forward_to=50'25)) v2 ==== 3570+0+0 (crc 0 0 0) 0x558c574a2000 con 0x558c54c64000 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c57d0bb00 seq 59 2019-12-07T19:34:28.616+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c57f3f4a0 prio 127 cost 0 latency 0.000021 epoch 51 MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=234, reqid=osd.1.0:234, at_version=51'27, trim_to=0'0, roll_forward_to=50'25)) v2 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x282d8e~1147 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:28.616+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=234, reqid=osd.1.0:234, at_version=51'27, trim_to=0'0, roll_forward_to=50'25)) v2) prio 127 cost 0 e51) 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x282d8e 0x282d8e~1147 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x252d8e 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd8e 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xed5 and padding block with 0x12b 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57d0bb00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57d0bb00 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57d0bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] new_repop rep_tid 38 (no op) 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0bb00 prepare 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57d0bb00 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0bb00 io_done 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] new_repop: repgather(0x558c576b6d80 0'0 rep_tid=38 committed?=0 r=0) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] simple_opc_submit 0x558c576b6d80 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) pg 0x558c575ef000 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55115500,0x558c55115800,0x558c582b7500> 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115500 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55115500 2019-12-07T19:34:28.616+0000 7f8f0297e700 7 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] issue_repop rep_tid 38 o 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115500 finishing 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55115500 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55115500 done 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55115800 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55115500 [] 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115800 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55115800 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55115800 finishing 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55115800 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55115800 done 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55115800 [] 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:19 v=51'34 tt=0'0 tid=38 reqid=osd.7.0:38 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 50 -> 51 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7500 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b7500 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7500 finishing 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b7500 onodes 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b7500 done 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 50 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b7500 [] 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) pg 0x558c5500a000 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:28.616+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key = 0 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=51) prio 5 cost 1048576 e51) queued 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 50 -> 51 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=51) prio 5 cost 1048576 e51) pg 0x558c57e88000 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 50 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 projected size 901120 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[19~1]] snap_trimmer posting 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 19 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:28.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c58b7fd40) upper_bound after SNA_1_0000000000000019_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.2' 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=51'34 tt=0'0 tid=38 reqid=osd.7.0:38 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c58b7fd40) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5810e000 seq 36 2019-12-07T19:34:28.616+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000019_.0_0000000000000001.2) returns 0 SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c552382c0 in raw 0x558c552382c0 len 69 nref 1) ) 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:19 v=51'34 tt=0'0 tid=38 reqid=osd.7.0:38 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:28.616+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=51'34 tt=0'0 tid=38 reqid=osd.7.0:38 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810e000 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim] SnapTrimmer state: adding snap 19 to purged_snaps 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810e000 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810e000 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c55123b00 seq 68 2019-12-07T19:34:28.616+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e000 prepare 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810e000 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e000 io_done 2019-12-07T19:34:28.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:19, current size is 901120 buffers are {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[19~1]] SnapTrimmer state: purged_snaps now [19~1], snap_trimq now [] 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e51) pg 0x558c575ef000 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55123b00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55123b00 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55123b00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c58110000 seq 25 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 4(0) info: 1.bs0( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123b00 prepare 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55123b00 2019-12-07T19:34:28.616+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123b00 io_done 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): new_size start 901120 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:28.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58110000 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e51) pg 0x558c5500a000 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5810e300 seq 37 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58110000 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58110000 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): to_append: {} 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.9s0( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110000 prepare 2019-12-07T19:34:28.616+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58110000 2019-12-07T19:34:28.616+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110000 io_done 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810e300 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810e300 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810e300 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 resetting hinfo to logical size 901120 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e300 prepare 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122c00 prepare 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810e300 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e300 io_done 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55122c00 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122c00 io_done 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c575ef000 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) pg 0x558c57380000 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c58112000 seq 69 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c5513e000 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): new_size start 1622016 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58112000 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58112000 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 50 -> 51 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58112000 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:28.616+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112000 prepare 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 50 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58112000 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): to_append: {} 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112000 io_done 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[19~1]] share_pg_info 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) pg 0x558c5736a000 2019-12-07T19:34:28.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:28.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:19={},1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map: 51 2019-12-07T19:34:28.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c58125800 con 0x558c54c19800 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 50 -> 51 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=51'34 tt=0'0 tid=38 reqid=osd.7.0:38 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 50 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57a52e00 con 0x558c54c5bc00 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] state: Started advmap 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:19,{} 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:34:28.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57a52700 con 0x558c54c5b400 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] plpg_on_pool_change 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] hit_set_clear 2019-12-07T19:34:28.616+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57a53500 con 0x558c54c19c00 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] agent_stop 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] handle_activate_map: 51 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[19~1]] exit Trimming 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c55122f00 seq 34 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map 2019-12-07T19:34:28.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 send_incremental_map 50 -> 51 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:28.616+0000 7f8f07988700 15 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] publish_stats_to_osd 51:142 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55122f00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c8c7972f:::inc_osdmap.51:0# 0x0~0 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55122f00 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] enter NotTrimming 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55122f00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~2a5 size 0x2a5 (677) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122f00 prepare 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55122f00 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122f00 io_done 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b28c0 blob([0xae3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2a5) SharedBlob(0x558c5809bab0 sbid 0x0)) need 0x0~2a5 cache has 0x[0~2a5] 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c8c7972f:::inc_osdmap.51:0# 0x0~2a5 = 677 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c58112300 seq 18 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 add_map_inc_bl 51 677 bytes 2019-12-07T19:34:28.616+0000 7f8f07988700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] snap_trimmer complete 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(51..51 src has 1..51) v4 -- 0x558c57968b40 con 0x558c54c64000 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 50 -> 51 (shared) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e51) pg 0x558c57380000 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 51/28 ECSubWrite(tid=38, reqid=osd.7.0:38, at_version=51'34, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c574a5000 con 0x558c54c64000 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58112300 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58112300 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58112300 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.8s0( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112300 prepare 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58112300 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112300 io_done 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 send_incremental_map 50 -> 51 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(51..51 src has 1..51) v4 -- 0x558c57968d80 con 0x558c54c19800 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e51) pg 0x558c5736a000 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 50 -> 51 (shared) 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c55123200 seq 35 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 51/28 ECSubWrite(tid=38, reqid=osd.7.0:38, at_version=51'34, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c551dd800 con 0x558c54c19800 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:28.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 send_incremental_map 50 -> 51 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55123200 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55123200 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(51..51 src has 1..51) v4 -- 0x558c57969680 con 0x558c54c5bc00 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 50 -> 51 (shared) 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55123200 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c58112600 seq 19 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 51/28 ECSubWrite(tid=38, reqid=osd.7.0:38, at_version=51'34, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c57dff000 con 0x558c54c5bc00 2019-12-07T19:34:28.616+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123200 prepare 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55123200 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123200 io_done 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 send_incremental_map 50 -> 51 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58112600 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58112600 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58112600 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c57380000 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(51..51 src has 1..51) v4 -- 0x558c579698c0 con 0x558c54c5b400 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 50 -> 51 (shared) 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112600 prepare 2019-12-07T19:34:28.616+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58112600 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x283ed5 2019-12-07T19:34:28.616+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112600 io_done 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 51/28 ECSubWrite(tid=38, reqid=osd.7.0:38, at_version=51'34, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c57dff800 con 0x558c54c5b400 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 send_incremental_map 50 -> 51 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000584797s (0.000000070s flush + 0.000584727s kv commit) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(51..51 src has 1..51) v4 -- 0x558c58192d80 con 0x558c54c19c00 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 12 submitting 12 deferred done 0 stable 0 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c5500d000 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c5736a000 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 51 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 50 -> 51 (shared) 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 51/28 ECSubWrite(tid=38, reqid=osd.7.0:38, at_version=51'34, trim_to=0'0, roll_forward_to=48'32)) v2 -- 0x558c58157800 con 0x558c54c19c00 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=235, reqid=client.4540.0:389, at_version=51'41, trim_to=0'0, roll_forward_to=45'39)) v2) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 2 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c5500a000 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2196 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=235, reqid=client.4540.0:389, at_version=51'41, trim_to=0'0, roll_forward_to=45'39)) v2) prio 127 cost 0 e51) pg 0x558c57380000 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 snap_mapper.update_snaps 1:33f5bf01:::smithi13140796-2:19 e,13,15 was 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:28.616+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3298 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 51 dequeue_op 0x558c57334000 prio 127 cost 0 latency 0.000634 MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=235, reqid=client.4540.0:389, at_version=51'41, trim_to=0'0, roll_forward_to=45'39)) v2 pg pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[19~1]] 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c586b4900 seq 20 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 2 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4400 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=235, reqid=client.4540.0:389, at_version=51'41, trim_to=0'0, roll_forward_to=45'39)) v2 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5494 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:33f5bf01:::smithi13140796-2:19 e,13,15,19 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.add_oid 1:10779340:::smithi13140796-32:1b e,13,15,1a,1b 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b4900 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b4900 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b4900 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6588 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 snap_mapper.set_snaps 1:33f5bf01:::smithi13140796-2:19 e,13,15 2019-12-07T19:34:28.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4900 prepare 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7682 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b4900 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4900 io_done 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 snap_mapper.update_snaps rm SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8776 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e51) pg 0x558c57383000 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:1b got.empty() 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 trim 51'33 (48'31) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9878 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.set_snaps 1:10779340:::smithi13140796-32:1b e,13,15,1a,1b 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 2 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 trim 51'34 (48'32) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10980 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 2 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12074 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13168 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_000000000000001A_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c586b4c00 seq 21 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 48'32 (0'0,48'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] append_log log((0'0,48'32], crt=48'32) [51'33 (48'31) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,51'34 (48'32) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'33 (0'0,51'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 lua=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] add_log_entry 51'33 (48'31) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b4c00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b4c00 onodes shared_blobs 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b4c00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13191 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 lua=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] add_log_entry 51'34 (48'32) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4c00 prepare 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b4c00 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13191 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4c00 io_done 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 lua=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] append_log approx pg log length = 34 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 45'39 (0'0,45'39] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[19~1]] append_log log((0'0,45'39], crt=45'39) [51'40 (29'25) clone 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:01.653811+0000 0 snaps [1b,1a,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,51'41 (29'25) modify 1:10779340:::smithi13140796-32:head by client.4540.0:389 2019-12-07T19:34:28.613112+0000 0 ObjectCleanRegions clean_offsets: [1523712~18446744073708027903], clean_omap: 0, new_object: 0] 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 lua=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x283ed5~3387 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c57383000 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x283ed5 0x283ed5~3387 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 51'33, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x253ed5 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xed5 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x25c and padding block with 0xda4 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'40 (0'0,51'40] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=45'39 crt=45'39 lcod 45'39 active mbc={} ps=[19~1]] add_log_entry 51'40 (29'25) clone 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:01.653811+0000 0 snaps [1b,1a,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 NullEvt) prio 255 cost 10 e51) pg 0x558c54d5f000 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=234, reqid=osd.1.0:234, at_version=51'27, trim_to=0'0, roll_forward_to=50'25)) v2) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=45'39 crt=45'39 lcod 45'39 active mbc={} ps=[19~1]] add_log_entry 51'41 (29'25) modify 1:10779340:::smithi13140796-32:head by client.4540.0:389 2019-12-07T19:34:28.613112+0000 0 ObjectCleanRegions clean_offsets: [1523712~18446744073708027903], clean_omap: 0, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=234, reqid=osd.1.0:234, at_version=51'27, trim_to=0'0, roll_forward_to=50'25)) v2) prio 127 cost 0 e51) pg 0x558c54d5f000 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=45'39 crt=45'39 lcod 45'39 active mbc={} ps=[19~1]] append_log approx pg log length = 41 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b7b00,0x558c55122900,0x558c575c4000,0x558c586b4600> 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7b00 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b7b00 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 51 dequeue_op 0x558c57f3f4a0 prio 127 cost 0 latency 0.000748 MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=234, reqid=osd.1.0:234, at_version=51'27, trim_to=0'0, roll_forward_to=50'25)) v2 pg pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={}] 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=45'39 crt=45'39 lcod 45'39 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f6c00 seq 46 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7b00 finishing 2019-12-07T19:34:28.616+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 51'40, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=234, reqid=osd.1.0:234, at_version=51'27, trim_to=0'0, roll_forward_to=50'25)) v2 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b7b00 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b7b00 done 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c4000 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys = 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 snap_mapper.update_snaps 1:ecc72bdb:::smithi13140796-10:1b 1a,1b was 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b7b00 [] 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122900 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55122900 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122900 finishing 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55122900 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55122900 done 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55122c00 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55122900 [] 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4000 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c4000 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4000 finishing 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c4000 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c4000 done 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:1b 19,1a,1b 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c4000 [] 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4600 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b4600 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:1b 1a,1b 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4600 finishing 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b4600 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b4600 done 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b4600 [] 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 snap_mapper.update_snaps rm SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c55123500 seq 36 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x28725c 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_0000000000000019_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10779340:::smithi13140796-32:1b# key 0x858000000000000001107793'@!smithi13140796-32!='0x000000000000001bffffffffffffffff'o' 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000051.00000000000000000033' <- 0000000051.00000000000000000033 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 50'25 (0'0,50'25] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={}] append_log log((0'0,50'25], crt=50'25) [51'26 (50'24) modify 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:28.613431+0000 0 snaps [1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,51'27 (50'25) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:28.613431+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 12 cleaned 0 in 0.000263811s (0.000000084s flush + 0.000263727s kv commit) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000051.00000000000000000034' <- 0000000051.00000000000000000034 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.8s5_head 5#1:10779340:::smithi13140796-32:head# -> 5#1:10779340:::smithi13140796-32:1b# 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1245 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.onode(0x558c5840c000).flush flush done 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:28.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 0x0 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f6c00 cost 8386 (1 ios * 4000 + 4386 bytes) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f6c00 onodes 0x558c58295180,0x558c58295400 shared_blobs 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 2 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'26 (0'0,51'26] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=50'25 crt=50'25 lcod 50'25 active mbc={}] add_log_entry 51'26 (50'24) modify 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:28.613431+0000 0 snaps [1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.8s5_head 5#1:10779340:::smithi13140796-32:head# -> 5#1:10779340:::smithi13140796-32:1b# 0x0~5d000 -> 0x0~5d000 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295288) update 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2196 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:19# is 1172 (410 bytes onode + 2 bytes spanning blobs + 760 bytes inline extents) 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2219 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2219 < min_flush_size 524288 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x0~2000: 0x0~2000 Blob(0x558c57dcb490 blob([0x7c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e631f0 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10594 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=50'25 crt=50'25 lcod 50'25 active mbc={}] add_log_entry 51'27 (50'25) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:28.613431+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57dcb490 blob([0x7c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e631f0 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1513 (557 bytes onode + 2 bytes spanning blobs + 954 bytes inline extents) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x28725c~8ab to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=50'25 crt=50'25 lcod 50'25 active mbc={}] append_log approx pg log length = 27 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57dcb490 blob([0x7c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=1)))) 2019-12-07T19:34:28.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x28725c 0x28725c~8ab to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5809b6c0 blob([0x7c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6c00 prepare 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=50'25 crt=50'25 lcod 50'25 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f6c00 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c5809b6c0) get_ref 0x0~2000 Blob(0x558c5809b6c0 blob([0x7c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6c00 io_done 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x25725c 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c5809b6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x25c 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb07 and padding block with 0x4f9 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x0~2000: 0x0~2000 Blob(0x558c5809b6c0 blob([0x7c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 51'26, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54df8690 blob([0x7c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b500 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10595 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c54df8690 blob([0x7c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b500 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 lua=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] op_applied version 51'34 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c54df8690 blob([0x7c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=1)))) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c580d7490 blob([0x7c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c580d7490) get_ref 0x0~2000 Blob(0x558c580d7490 blob([0x7c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c580d7490) get_ref init 0x2000, 1000 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c580d7490 blob([0x7c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57f3b810 blob([0x7c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67d50 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10596 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57f3b810 blob([0x7c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67d50 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] eval_repop repgather(0x558c576b6d80 51'34 rep_tid=38 committed?=0 r=0) (no op) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57f3b810 blob([0x7c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=1)))) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c57c42d20 blob([0x7c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c57c42d20) get_ref 0x0~2000 Blob(0x558c57c42d20 blob([0x7c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] calc_trim_to_aggressive limit = 48'32 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57d0b800,0x558c57d0bb00,0x558c5810e000,0x558c55123b00,0x558c58110000,0x558c5810e300,0x558c55122c00,0x558c58112000,0x558c55122f00,0x558c58112300,0x558c55123200,0x558c58112600> 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c57c42d20) get_ref init 0x2000, 1000 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b800 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0b800 2019-12-07T19:34:28.616+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b4f00 seq 69 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57c42d20 blob([0x7c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57f67dc0 blob([0x7c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67e30 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10597 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0b800 finishing 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0b800 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0b800 done 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57f67dc0 blob([0x7c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67e30 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# 1 keys 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57f67dc0 blob([0x7c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=1)))) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] enter Trimming/WaitRepops 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# 1 keys = 0 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0bb00 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c57e63420 blob([0x7c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0b800 [] 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# hinfo_key (42 bytes) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c57e63420) get_ref 0x0~2000 Blob(0x558c57e63420 blob([0x7c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c57e63420) get_ref init 0x2000, 1000 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0bb00 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57d0bb00 2019-12-07T19:34:28.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] snap_trimmer complete 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57d0bb00 finishing 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57e63420 blob([0x7c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57d0bb00 onodes 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57d0bb00 done 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57f67ea0 blob([0x7cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67f10 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57d0bb00 [] 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10598 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:34:28.616+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57f67ea0 blob([0x7cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67f10 sbid 0x0)) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e000 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810e000 2019-12-07T19:34:28.616+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e000 finishing 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57f67ea0 blob([0x7cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=1)))) 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810e000 onodes 2019-12-07T19:34:28.616+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810e000 done 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810e300 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810e000 [] 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c583ebdc0 blob([0x7cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=2)))) 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123b00 kv_submitted 2019-12-07T19:34:28.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55123b00 2019-12-07T19:34:28.616+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebdc0) get_ref 0x0~2000 Blob(0x558c583ebdc0 blob([0x7cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123b00 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55123b00 onodes 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c583ebdc0 blob([0x7cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55123b00 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112000 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55123b00 [] 2019-12-07T19:34:28.617+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57f67f80 blob([0x7cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6d20 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10599 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110000 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58110000 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57f67f80 blob([0x7cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6d20 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110000 finishing 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57f67f80 blob([0x7cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58110000 onodes 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58110000 done 2019-12-07T19:34:28.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5840f500 blob([0x7cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58110000 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f500) get_ref 0x0~2000 Blob(0x558c5840f500 blob([0x7cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f500) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e300 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810e300 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- SNA_1_0000000000000019_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e300 finishing 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5840f500 blob([0x7cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57bb6d90 blob([0x7cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6e00 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10600 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6d90 blob([0x7cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6e00 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:28.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000051.00000000000000000026' <- 0000000051.00000000000000000026 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810e300 onodes 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6d90 blob([0x7cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810e300 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:28.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810e300 [] 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5840f490 blob([0x7cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000051.00000000000000000027' <- 0000000051.00000000000000000027 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122c00 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55122c00 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f490) get_ref 0x0~2000 Blob(0x558c5840f490 blob([0x7cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122c00 finishing 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f490) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55122c00 onodes 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55122c00 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55122c00 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c5840f490 blob([0x7cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112000 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58112000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57bb6e70 blob([0x7d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6ee0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10601 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112000 finishing 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58112000 onodes 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6e70 blob([0x7d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6ee0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112000 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58112000 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6e70 blob([0x7d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122f00 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55122f00 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55122f00 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55122f00 onodes 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55122f00 done 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c583eba40 blob([0x7d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55123200 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55122f00 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c583eba40) get_ref 0x0~2000 Blob(0x558c583eba40 blob([0x7d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112300 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58112300 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c583eba40) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b4f00 cost 8533 (1 ios * 4000 + 4533 bytes) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112300 finishing 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c583eba40 blob([0x7d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58112300 onodes 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b4f00 onodes 0x558c575a2780,0x558c575a2a00 shared_blobs 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112300 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112600 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58112300 [] 2019-12-07T19:34:28.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57bb6f50 blob([0x7d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6fc0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10602 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123200 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55123200 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6f50 blob([0x7d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6fc0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123200 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55123200 onodes 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55123200 done 2019-12-07T19:34:28.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6f50 blob([0x7d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2888) update 4#1:ecc72bdb:::smithi13140796-10:1b# 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5840fa40 blob([0x7d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5840fa40) get_ref 0x0~2000 Blob(0x558c5840fa40 blob([0x7d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5840fa40) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55123500 prepare 2019-12-07T19:34:28.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5840fa40 blob([0x7d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55123200 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.615+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] state: Started advmap 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57bb7030 blob([0x7d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb70a0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10603 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112600 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58112600 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7030 blob([0x7d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb70a0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112600 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58112600 onodes 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7030 blob([0x7d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112600 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58112600 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c57881880 blob([0x7d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c57881880) get_ref 0x0~2000 Blob(0x558c57881880 blob([0x7d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c57881880) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x287b07 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:1b# is 57903 (57759 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57881880 blob([0x7d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57bb7110 blob([0x7d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7180 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10604 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] plpg_on_pool_change 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7110 blob([0x7d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7180 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7110 blob([0x7d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] hit_set_clear 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000224009s (0.000000069s flush + 0.000223940s kv commit) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] agent_stop 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e000 blob([0x7d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 51 advance_pg new pool opts old pool opts 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e000) get_ref 0x0~2000 Blob(0x558c5811e000 blob([0x7d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e000) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] handle_activate_map: 51 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c5811e000 blob([0x7d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5546 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57bb71f0 blob([0x7d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7260 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10605 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb71f0 blob([0x7d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7260 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] activate_map 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb71f0 blob([0x7d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] state: Active: handling ActMap 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 1137 (637 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e070 blob([0x7d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e070) get_ref 0x0~2000 Blob(0x558c5811e070 blob([0x7d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e070) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] Active: kicking snap trim 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295180 had 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c5811e070 blob([0x7d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b4f00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x18000~2000: 0x0~2000 Blob(0x558c57bb72d0 blob([0x7db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7340 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10606 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5569 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5569 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb72d0 blob([0x7db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7340 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb72d0 blob([0x7db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4f00 prepare 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e0e0 blob([0x7db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b4f00 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4f00 io_done 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e0e0) get_ref 0x0~2000 Blob(0x558c5811e0e0 blob([0x7db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x287b07~15c1 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] exit NotTrimming 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c5811e0e0 blob([0x7db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x287b07 0x287b07~15c1 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57bb73b0 blob([0x7dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7420 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10607 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] enter Trimming 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb73b0 blob([0x7dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7420 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~260000 x_off 0x257b07 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb07 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc8 and padding block with 0xf38 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb73b0 blob([0x7dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=50'25 crt=50'25 lcod 50'25 active mbc={}] op_applied version 51'27 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e150 blob([0x7dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e150) get_ref 0x0~2000 Blob(0x558c5811e150 blob([0x7dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e150) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0117b700 10 osd.7 51 dequeue_op 0x558c57f3f4a0 finish 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c5811e150 blob([0x7dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57bb7490 blob([0x7df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7500 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10608 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7490 blob([0x7df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7500 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7490 blob([0x7df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] publish_stats_to_osd 50:124 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e1c0 blob([0x7df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e1c0) get_ref 0x0~2000 Blob(0x558c5811e1c0 blob([0x7df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.617+0000 7f8f06986700 7 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] state: handle ActMap primary 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c5811e1c0 blob([0x7df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586b4900,0x558c586b4c00> 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4900 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c57bb7570 blob([0x7e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb75e0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b4900 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10609 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4900 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b4900 onodes 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b4900 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b4c00 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b4900 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4c00 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b4c00 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4c00 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b4c00 onodes 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b4c00 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b4c00 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.617+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2890c8 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.617+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000166374s (0.000000054s flush + 0.000166320s kv commit) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 28472 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61240 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61978 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2780 had 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 62001 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 62001 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2890c8~f231 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2890c8 0x2890c8~f231 to file(ino 16 size 0x290000 mtime 2019-12-07T19:34:28.615401+0000 allocated 290000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~260000]) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _allocate len 0x82f9 from 0 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _allocate len 0x82f9 from 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd3c0000 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd3c0000~10000/10000,10000,acd3c0000 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x2a0000 to include allocated 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 58 (was clean) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x2590c8 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc8 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2f9 and padding block with 0xd07 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x2a0000 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585f6c00> 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6c00 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f6c00 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6c00 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f6c00 onodes 0x558c58295180,0x558c58295400 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6c00 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f6c00 [] 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7570 blob([0x7e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb75e0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7570 blob([0x7e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e230 blob([0x7e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] publish_stats_to_osd 51:125 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e230) get_ref 0x0~2000 Blob(0x558c5811e230 blob([0x7e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e230) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c5811e230 blob([0x7e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] activate_map: Not dirtying info: last_persisted is 47 while current is 51 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x20000~2000: 0x0~2000 Blob(0x558c57bb7650 blob([0x7e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb76c0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10610 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7650 blob([0x7e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb76c0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7650 blob([0x7e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e2a0 blob([0x7e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e2a0) get_ref 0x0~2000 Blob(0x558c5811e2a0 blob([0x7e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 NullEvt 2019-12-07T19:34:28.617+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c5811e2a0 blob([0x7e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:28.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c582ac600 seq 39 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57bb7730 blob([0x7e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb77a0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10611 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7730 blob([0x7e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb77a0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7730 blob([0x7e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e310 blob([0x7e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e310) get_ref 0x0~2000 Blob(0x558c5811e310 blob([0x7e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e310) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:28.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:28.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ac600 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ac600 onodes shared_blobs 2019-12-07T19:34:28.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ac600 allocated 0x[] released 0x[] 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c5811e310 blob([0x7e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x24000~2000: 0x0~2000 Blob(0x558c57bb7810 blob([0x7e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7880 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10612 2019-12-07T19:34:28.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac600 prepare 2019-12-07T19:34:28.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ac600 2019-12-07T19:34:28.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac600 io_done 2019-12-07T19:34:28.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7810 blob([0x7e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7880 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2982f9 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7810 blob([0x7e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.617+0000 7f8f0b990700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e380 blob([0x7e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e380) get_ref 0x0~2000 Blob(0x558c5811e380 blob([0x7e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0b990700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] SnapTrimmer state: NotTrimming: trimming 19 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e380) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c5811e380 blob([0x7e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x26000~2000: 0x0~2000 Blob(0x558c57bb78f0 blob([0x7e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7960 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10613 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb78f0 blob([0x7e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7960 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb78f0 blob([0x7e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e3f0 blob([0x7e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0b990700 15 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] publish_stats_to_osd 51:126 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e3f0) get_ref 0x0~2000 Blob(0x558c5811e3f0 blob([0x7e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c5811e3f0 blob([0x7e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x28000~2000: 0x0~2000 Blob(0x558c57bb79d0 blob([0x7eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7a40 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (58) on file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]), flushing log 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10614 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb79d0 blob([0x7eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7a40 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0b990700 10 osd.7 51 queueing pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[19~1]] for snaptrim 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.617+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=51) prio 5 cost 1048576 e51) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb79d0 blob([0x7eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 58 len 0x4a crc 0x1da4c995) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e460 blob([0x7eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x39000~1000 to file(ino 1 size 0x39000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:28.617+0000 7f8f0b990700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e460) get_ref 0x0~2000 Blob(0x558c5811e460 blob([0x7eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.617+0000 7f8f0b990700 15 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[19~1]] publish_stats_to_osd 51:127 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x39000 0x39000~1000 to file(ino 1 size 0x39000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x3a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e460) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=51) prio 5 cost 1048576 e51) queued 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c5811e460 blob([0x7eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x39000 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c57bb7ab0 blob([0x7ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7b20 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10615 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=51) prio 5 cost 1048576 e51) pg 0x558c548ab000 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7ab0 blob([0x7ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7b20 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7ab0 blob([0x7ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e4d0 blob([0x7ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e4d0) get_ref 0x0~2000 Blob(0x558c5811e4d0 blob([0x7ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[19~1]] snap_trimmer posting 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c5811e4d0 blob([0x7ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c57bb7b90 blob([0x7ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7c00 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10616 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 19 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7b90 blob([0x7ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7c00 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7b90 blob([0x7ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x3a000 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e540 blob([0x7ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e540) get_ref 0x0~2000 Blob(0x558c5811e540 blob([0x7ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e540) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c5811e540 blob([0x7ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c57bb7c70 blob([0x7f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ce0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10617 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7c70 blob([0x7f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ce0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fdc0) upper_bound after SNA_1_0000000000000019_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.4' 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7c70 blob([0x7f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e5b0 blob([0x7f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e5b0) get_ref 0x0~2000 Blob(0x558c5811e5b0 blob([0x7f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000025 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fdc0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c5811e5b0 blob([0x7f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x30000~2000: 0x0~2000 Blob(0x558c57bb7d50 blob([0x7f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7dc0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10618 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000019_.0_0000000000000001.4) returns 0 SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7d50 blob([0x7f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7dc0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7d50 blob([0x7f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e620 blob([0x7f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e620) get_ref 0x0~2000 Blob(0x558c5811e620 blob([0x7f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e620) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c5811e620 blob([0x7f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim] SnapTrimmer state: adding snap 19 to purged_snaps 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x32000~2000: 0x0~2000 Blob(0x558c57bb7e30 blob([0x7f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ea0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10619 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7e30 blob([0x7f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ea0 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 58 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7e30 blob([0x7f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e690 blob([0x7f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e690) get_ref 0x0~2000 Blob(0x558c5811e690 blob([0x7f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e690) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[19~1]] SnapTrimmer state: purged_snaps now [19~1], snap_trimq now [] 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c5811e690 blob([0x7f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x34000~2000: 0x0~2000 Blob(0x558c57bb7f10 blob([0x7f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7f80 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10620 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000547173s (0.000000059s flush + 0.000547114s kv commit) 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c585f6f00 seq 40 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586b4f00> 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4f00 kv_submitted 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b4f00 2019-12-07T19:34:28.617+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b4f00 finishing 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b4f00 onodes 0x558c575a2780,0x558c575a2a00 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b4f00 done 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b4f00 [] 2019-12-07T19:34:28.617+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:28.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb7f10 blob([0x7f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7f80 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb7f10 blob([0x7f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:28.617+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 51/41 ECSubWriteReply(tid=234, last_complete=51'27, committed=1, applied=1)) v2 -- 0x558c5763c600 con 0x558c54c64000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e700 blob([0x7f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f6f00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e700) get_ref 0x0~2000 Blob(0x558c5811e700 blob([0x7f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e700) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f6f00 onodes shared_blobs 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f6f00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2982f9~465 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c5811e700 blob([0x7f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2982f9 0x2982f9~465 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x2682f9 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2f9 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x75e and padding block with 0x8a2 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6f00 prepare 2019-12-07T19:34:28.617+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f6f00 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x36000~2000: 0x0~2000 Blob(0x558c5840e000 blob([0x7f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e070 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10621 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6f00 io_done 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840e000 blob([0x7f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e070 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840e000 blob([0x7f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e770 blob([0x7f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[19~1]] share_pg_info 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e770) get_ref 0x0~2000 Blob(0x558c5811e770 blob([0x7f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e770) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c55054300 con 0x558c54c64000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c5811e770 blob([0x7f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x38000~2000: 0x0~2000 Blob(0x558c5840e0e0 blob([0x7fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e150 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10622 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840e0e0 blob([0x7fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e150 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840e0e0 blob([0x7fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e7e0 blob([0x7fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e7e0) get_ref 0x0~2000 Blob(0x558c5811e7e0 blob([0x7fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c5811e7e0 blob([0x7fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c5840e1c0 blob([0x7fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e230 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10623 2019-12-07T19:34:28.617+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840e1c0 blob([0x7fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e230 sbid 0x0)) 2019-12-07T19:34:28.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c576b8e00 con 0x558c54c5bc00 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840e1c0 blob([0x7fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=1)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e850 blob([0x7fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e850) get_ref 0x0~2000 Blob(0x558c5811e850 blob([0x7fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=2)))) 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e850) get_ref init 0x2000, 1000 2019-12-07T19:34:28.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29875e 2019-12-07T19:34:28.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.617+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c5811e850 blob([0x7fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0918b700 20 osd.7 51 agent_entry empty queue 2019-12-07T19:34:28.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c57a52000 con 0x558c54c5b400 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c5840e2a0 blob([0x7ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e310 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10624 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840e2a0 blob([0x7ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e310 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840e2a0 blob([0x7ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e8c0 blob([0x7ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e8c0) get_ref 0x0~2000 Blob(0x558c5811e8c0 blob([0x7ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 -- 0x558c5737aa00 con 0x558c54c19c00 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c5811e8c0 blob([0x7ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c5840e380 blob([0x801000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e3f0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10625 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840e380 blob([0x801000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e3f0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840e380 blob([0x801000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e930 blob([0x801000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e930) get_ref 0x0~2000 Blob(0x558c5811e930 blob([0x801000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e930) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[19~1]] exit Trimming 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c5811e930 blob([0x801000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x40000~2000: 0x0~2000 Blob(0x558c580ec000 blob([0x803000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec070 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10626 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c580ec000 blob([0x803000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec070 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c580ec000 blob([0x803000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] publish_stats_to_osd 51:128 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811e9a0 blob([0x803000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e9a0) get_ref 0x0~2000 Blob(0x558c5811e9a0 blob([0x803000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811e9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] enter NotTrimming 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c5811e9a0 blob([0x803000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x42000~2000: 0x0~2000 Blob(0x558c57a06770 blob([0x805000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6540 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10627 2019-12-07T19:34:28.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57a06770 blob([0x805000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6540 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57a06770 blob([0x805000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ea10 blob([0x805000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ea10) get_ref 0x0~2000 Blob(0x558c5811ea10 blob([0x805000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] snap_trimmer complete 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ea10) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c5811ea10 blob([0x805000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x44000~2000: 0x0~2000 Blob(0x558c57bb65b0 blob([0x807000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6620 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10628 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb65b0 blob([0x807000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6620 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb65b0 blob([0x807000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ea80 blob([0x807000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ea80) get_ref 0x0~2000 Blob(0x558c5811ea80 blob([0x807000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ea80) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c5811ea80 blob([0x807000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x46000~2000: 0x0~2000 Blob(0x558c57bb6690 blob([0x809000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6700 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10629 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6690 blob([0x809000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6700 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6690 blob([0x809000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811eaf0 blob([0x809000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811eaf0) get_ref 0x0~2000 Blob(0x558c5811eaf0 blob([0x809000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811eaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c5811eaf0 blob([0x809000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x48000~2000: 0x0~2000 Blob(0x558c57bb6770 blob([0x80b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb67e0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10630 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6770 blob([0x80b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb67e0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6770 blob([0x80b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811eb60 blob([0x80b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811eb60) get_ref 0x0~2000 Blob(0x558c5811eb60 blob([0x80b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811eb60) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c5811eb60 blob([0x80b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c57bb6850 blob([0x80d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb68c0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10631 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6850 blob([0x80d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb68c0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6850 blob([0x80d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ebd0 blob([0x80d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ebd0) get_ref 0x0~2000 Blob(0x558c5811ebd0 blob([0x80d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ebd0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c5811ebd0 blob([0x80d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c57bb6930 blob([0x80f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb69a0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10632 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6930 blob([0x80f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb69a0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6930 blob([0x80f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ec40 blob([0x80f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ec40) get_ref 0x0~2000 Blob(0x558c5811ec40 blob([0x80f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ec40) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c5811ec40 blob([0x80f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c57bb6a10 blob([0x811000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6a80 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10633 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6a10 blob([0x811000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6a80 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6a10 blob([0x811000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ecb0 blob([0x811000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ecb0) get_ref 0x0~2000 Blob(0x558c5811ecb0 blob([0x811000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ecb0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c5811ecb0 blob([0x811000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x50000~2000: 0x0~2000 Blob(0x558c57bb6af0 blob([0x813000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6b60 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10634 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6af0 blob([0x813000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6b60 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6af0 blob([0x813000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ed20 blob([0x813000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ed20) get_ref 0x0~2000 Blob(0x558c5811ed20 blob([0x813000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ed20) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c5811ed20 blob([0x813000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x52000~2000: 0x0~2000 Blob(0x558c57bb6bd0 blob([0x815000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6c40 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10635 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6bd0 blob([0x815000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6c40 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6bd0 blob([0x815000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ed90 blob([0x815000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ed90) get_ref 0x0~2000 Blob(0x558c5811ed90 blob([0x815000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ed90) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c5811ed90 blob([0x815000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x54000~2000: 0x0~2000 Blob(0x558c57bb6cb0 blob([0x817000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fab0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10636 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c57bb6cb0 blob([0x817000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fab0 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c57bb6cb0 blob([0x817000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ee00 blob([0x817000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ee00) get_ref 0x0~2000 Blob(0x558c5811ee00 blob([0x817000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ee00) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c5811ee00 blob([0x817000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x56000~2000: 0x0~2000 Blob(0x558c5840fb20 blob([0x819000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fb90 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10637 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840fb20 blob([0x819000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fb90 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840fb20 blob([0x819000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ee70 blob([0x819000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ee70) get_ref 0x0~2000 Blob(0x558c5811ee70 blob([0x819000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ee70) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c5811ee70 blob([0x819000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x58000~2000: 0x0~2000 Blob(0x558c5840fc00 blob([0x81b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fc70 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10638 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840fc00 blob([0x81b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fc70 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840fc00 blob([0x81b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811eee0 blob([0x81b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811eee0) get_ref 0x0~2000 Blob(0x558c5811eee0 blob([0x81b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811eee0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c5811eee0 blob([0x81b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c5840fce0 blob([0x81d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fd50 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10639 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840fce0 blob([0x81d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fd50 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840fce0 blob([0x81d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811ef50 blob([0x81d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ef50) get_ref 0x0~2000 Blob(0x558c5811ef50 blob([0x81d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811ef50) get_ref init 0x2000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c5811ef50 blob([0x81d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup src 0x5c000~1000: 0x0~1000 Blob(0x558c5840fdc0 blob([0x81f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5840fe30 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10640 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared Blob(0x558c5840fdc0 blob([0x81f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5840fe30 sbid 0x0)) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) make_blob_shared now Blob(0x558c5840fdc0 blob([0x81f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=1)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup new Blob(0x558c5811efc0 blob([0x81f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811efc0) get_ref 0x0~1000 Blob(0x558c5811efc0 blob([0x81f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.blob(0x558c5811efc0) get_ref init 0x1000, 1000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dup dst 0x5c000~1000: 0x0~1000 Blob(0x558c5811efc0 blob([0x81f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=2)))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) dirty_range mark inline shard dirty 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) dirty_range mark inline shard dirty 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.8s5_head 5#1:10779340:::smithi13140796-32:head# -> 5#1:10779340:::smithi13140796-32:1b# = 0 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# snapset 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# snapset = 0 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys = 0 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:10779340:::smithi13140796-32:head#29 key 0x858000000000000001107793'@!smithi13140796-32!='0xfffffffffffffffe0000000000000029'o' 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:28.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 151 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c58125800 con 0x558c54c19800 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#1:10779340:::smithi13140796-32:head# -> 5#1:10779340:::smithi13140796-32:head#29 2019-12-07T19:34:28.618+0000 7f8f24c58700 15 osd.7 51 enqueue_peering_evt 1.ds2 epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:28.618+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#1:10779340:::smithi13140796-32:head# -> 5#1:10779340:::smithi13140796-32:head#29 = 0 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1246 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.8s5_head 5#1:10779340:::smithi13140796-32:head# = 0 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.618+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 152 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58114600 con 0x558c54c19800 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e51) pg 0x558c55141000 2019-12-07T19:34:28.618+0000 7f8f24c58700 15 osd.7 51 enqueue_op 0x558c5511eb00 prio 196 cost 0 latency 0.000009 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_000000000000000E_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_0000000000000013_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:28.618+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57a56f00 seq 44 2019-12-07T19:34:28.618+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a56f00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_0000000000000015_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a56f00 onodes shared_blobs 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_000000000000001A_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_000000000000001B_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000687538s (0.000000148s flush + 0.000687390s kv commit) 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582ac600> 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac600 kv_submitted 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ac600 2019-12-07T19:34:28.618+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c5511eb00 prio 196 cost 0 latency 0.000103 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 2019-12-07T19:34:28.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac600 finishing 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ac600 onodes 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ac600 done 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6f00 kv_queued 2019-12-07T19:34:28.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ac600 [] 2019-12-07T19:34:28.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a56f00 allocated 0x[] released 0x[] 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56f00 prepare 2019-12-07T19:34:28.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a56f00 2019-12-07T19:34:28.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:28.618+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56f00 io_done 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000051.00000000000000000040' <- 0000000051.00000000000000000040 2019-12-07T19:34:28.618+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.618+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000051.00000000000000000041' <- 0000000051.00000000000000000041 2019-12-07T19:34:28.618+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c5511eb00 finish 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29875e~45d to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:28.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29875e 0x29875e~45d to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26875e 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x75e 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c55123500 cost 8661 (1 ios * 4000 + 4661 bytes) 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbbb and padding block with 0x445 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c55123500 onodes 0x558c5840c000,0x558c5840c500,0x558c5840c780 shared_blobs 0x558c5742b500,0x558c57bb6540,0x558c57bb6620,0x558c57bb6700,0x558c57bb67e0,0x558c57bb68c0,0x558c57bb69a0,0x558c57bb6a80,0x558c57bb6b60,0x558c57bb6c40,0x558c57bb6d20,0x558c57bb6e00,0x558c57bb6ee0,0x558c57bb6fc0,0x558c57bb70a0,0x558c57bb7180,0x558c57bb7260,0x558c57bb7340,0x558c57bb7420,0x558c57bb7500,0x558c57bb75e0,0x558c57bb76c0,0x558c57bb77a0,0x558c57bb7880,0x558c57bb7960,0x558c57bb7a40,0x558c57bb7b20,0x558c57bb7c00,0x558c57bb7ce0,0x558c57bb7dc0,0x558c57bb7ea0,0x558c57bb7f80,0x558c57e631f0,0x558c57f67d50,0x558c57f67e30,0x558c57f67f10,0x558c580ec070,0x558c5840e070,0x558c5840e150,0x558c5840e230,0x558c5840e310,0x558c5840e3f0,0x558c5840fab0,0x558c5840fb90,0x558c5840fc70,0x558c5840fd50,0x558c5840fe30 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update 5#1:10779340:::smithi13140796-32:head#29 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update inline shard 1269 bytes from 47 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore.extentmap(0x558c5840c108) reshard 0x[0,ffffffff) of 0 shards on 5#1:10779340:::smithi13140796-32:head#29 2019-12-07T19:34:28.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 392 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5763c600 con 0x558c54c64000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard new shard 0x0 2019-12-07T19:34:28.618+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000218 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard new shard 0x24000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard new shard 0x48000 2019-12-07T19:34:28.618+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard old [] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update 5#1:10779340:::smithi13140796-32:head#29 force 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c108) update shard 0x48000 is 299 bytes (was 0) from 11 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head#29 is 500 (498 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update 5#1:10779340:::smithi13140796-32:1b# 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update inline shard 1269 bytes from 47 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 10 bluestore.extentmap(0x558c5840c608) reshard 0x[0,ffffffff) of 0 shards on 5#1:10779340:::smithi13140796-32:1b# 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard new shard 0x0 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard new shard 0x24000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard new shard 0x48000 2019-12-07T19:34:28.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 393 ==== pg_lease(1.0s3 pg_lease(ru 60.076769972s ub 68.076978842s int 16s) e51/51) v1 ==== 58+0+0 (crc 0 0 0) 0x558c548cda00 con 0x558c54c64000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard old [] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update 5#1:10779340:::smithi13140796-32:1b# force 2019-12-07T19:34:28.618+0000 7f8f24457700 15 osd.7 51 enqueue_peering_evt 1.0s3 epoch_sent: 51 epoch_requested: 51 MLease epoch 51 from osd.1 pg_lease(ru 60.076769972s ub 68.076978842s int 16s) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:28.618+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MLease epoch 51 from osd.1 pg_lease(ru 60.076769972s ub 68.076978842s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update shard 0x48000 is 299 bytes (was 0) from 11 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:1b# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 507 (499 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2963 is 11 SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2983 is 11 SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2984 is 11 SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2985 is 11 SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2986 is 11 SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2987 is 11 SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2988 is 11 SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MLease epoch 51 from osd.1 pg_lease(ru 60.076769972s ub 68.076978842s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2989 is 11 SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298a is 11 SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MLease epoch 51 from osd.1 pg_lease(ru 60.076769972s ub 68.076978842s int 16s)) prio 255 cost 10 e51) pg 0x558c5500d000 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298b is 11 SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2967 is 11 SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2968 is 11 SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2969 is 11 SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000306 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296a is 11 SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296b is 11 SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296c is 11 SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f04982700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MLease epoch 51 from osd.1 pg_lease(ru 60.076769972s ub 68.076978842s int 16s) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296d is 11 SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296e is 11 SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296f is 11 SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2970 is 11 SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2971 is 11 SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f04982700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] proc_lease pg_lease(ru 60.076769972s ub 68.076978842s int 16s) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2972 is 11 SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2973 is 11 SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2974 is 11 SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f04982700 20 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] peer_clock_delta_ub 0.021792164s -> ru 60.054977808s 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2975 is 11 SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2976 is 11 SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2977 is 11 SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f04982700 20 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] proc_lease readable_until now 60.054977808s 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2978 is 11 SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2979 is 11 SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=2))) 2019-12-07T19:34:28.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x298bbb 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297a is 11 SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f04982700 20 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] peer_clock_delta_lb 0.021438294s -> ruub 68.055540548s 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297b is 11 SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297c is 11 SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2962 is 11 SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f04982700 20 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] proc_lease readable_until_ub now 68.055540548s 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2964 is 11 SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2965 is 11 SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2966 is 11 SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2982 is 11 SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000494376s (0.000000123s flush + 0.000494253s kv commit) 2019-12-07T19:34:28.619+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.0s0 pg_lease_ack(ruub 68.076978842s) e51/51) v1 -- 0x558c5734ad20 con 0x558c54c64000 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297d is 11 SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297e is 11 SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297f is 11 SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2980 is 11 SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2981 is 11 SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298c is 11 SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298d is 11 SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298e is 11 SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298f is 11 SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2990 is 11 SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=2))) 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c55123500 allocated 0x[] released 0x[] 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.619+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123500 prepare 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c55123500 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123500 io_done 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x298bbb~465 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x298bbb 0x298bbb~465 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x268bbb 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbbb 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x20 and padding block with 0xfe0 2019-12-07T19:34:28.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=45'39 crt=45'39 lcod 45'39 active mbc={} ps=[19~1]] op_applied version 51'41 2019-12-07T19:34:28.619+0000 7f8f0898a700 10 osd.7 51 dequeue_op 0x558c57334000 finish 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585f6f00> 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6f00 kv_submitted 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f6f00 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6f00 finishing 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f6f00 onodes 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6f00 done 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f6f00 [] 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x299020 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000140016s (0.000000116s flush + 0.000139900s kv commit) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8440 < min_flush_size 524288 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c000 had 1 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c500 had 1 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8463 < min_flush_size 524288 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8463 < min_flush_size 524288 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x299020~210f to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x299020 0x299020~210f to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x269020 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x20 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x12f and padding block with 0xed1 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a56f00> 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56f00 kv_submitted 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a56f00 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a56f00 finishing 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a56f00 onodes 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a56f00 done 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a56f00 [] 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29b12f 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000179667s (0.000000069s flush + 0.000179598s kv commit) 2019-12-07T19:34:28.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c55123500> 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123500 kv_submitted 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c55123500 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c55123500 finishing 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c55123500 onodes 0x558c5840c000,0x558c5840c500,0x558c5840c780 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c55123500 done 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c55123500 [] 2019-12-07T19:34:28.619+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.619+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 51/41 ECSubWriteReply(tid=235, last_complete=51'41, committed=1, applied=1)) v2 -- 0x558c5795da00 con 0x558c54c64000 2019-12-07T19:34:28.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 231 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f600 con 0x558c54c5bc00 2019-12-07T19:34:28.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000012 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.619+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.619+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 2019-12-07T19:34:28.619+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.619+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:34:28.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 97 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795da00 con 0x558c54c19c00 2019-12-07T19:34:28.619+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c57f3fe40 prio 196 cost 0 latency 0.000013 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.619+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c57f3fe40 prio 196 cost 0 latency 0.000039 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 2019-12-07T19:34:28.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.619+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c57f3fe40 finish 2019-12-07T19:34:28.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 394 ==== MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=236, reqid=client.4540.0:389, at_version=0'0, trim_to=0'0, roll_forward_to=51'41)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c58157800 con 0x558c54c64000 2019-12-07T19:34:28.620+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c57f3f340 prio 127 cost 0 latency 0.000021 epoch 51 MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=236, reqid=client.4540.0:389, at_version=0'0, trim_to=0'0, roll_forward_to=51'41)) v2 2019-12-07T19:34:28.620+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=236, reqid=client.4540.0:389, at_version=0'0, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e51) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=236, reqid=client.4540.0:389, at_version=0'0, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=236, reqid=client.4540.0:389, at_version=0'0, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e51) pg 0x558c57380000 2019-12-07T19:34:28.620+0000 7f8f04982700 10 osd.7 51 dequeue_op 0x558c57f3f340 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=236, reqid=client.4540.0:389, at_version=0'0, trim_to=0'0, roll_forward_to=51'41)) v2 pg pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[19~1]] 2019-12-07T19:34:28.620+0000 7f8f04982700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 51/41 ECSubWrite(tid=236, reqid=client.4540.0:389, at_version=0'0, trim_to=0'0, roll_forward_to=51'41)) v2 2019-12-07T19:34:28.620+0000 7f8f04982700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=45'39 active mbc={} ps=[19~1]] append_log log((0'0,51'41], crt=45'39) [] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] rollforward: entry=51'40 (29'25) clone 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:01.653811+0000 0 snaps [1b,1a,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.620+0000 7f8f04982700 20 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] rollforward: entry=51'41 (29'25) modify 1:10779340:::smithi13140796-32:head by client.4540.0:389 2019-12-07T19:34:28.613112+0000 0 ObjectCleanRegions clean_offsets: [1523712~18446744073708027903], clean_omap: 0, new_object: 0 2019-12-07T19:34:28.620+0000 7f8f04982700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] append_log approx pg log length = 41 2019-12-07T19:34:28.620+0000 7f8f04982700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:28.620+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.620+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58c64000 seq 37 2019-12-07T19:34:28.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.8s5_head 5#1:10779340:::smithi13140796-32:head#29 onode 0x558c5840c000 txc 0x558c58c64000 2019-12-07T19:34:28.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.8s5_head 5#1:10779340:::smithi13140796-32:head#29 0x0 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57dcb490) put_ref 0x0~2000 Blob(0x558c57dcb490 blob([0x7c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c54df8690) put_ref 0x0~2000 Blob(0x558c54df8690 blob([0x7c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57f3b810) put_ref 0x0~2000 Blob(0x558c57f3b810 blob([0x7c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57f67dc0) put_ref 0x0~2000 Blob(0x558c57f67dc0 blob([0x7c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57f67ea0) put_ref 0x0~2000 Blob(0x558c57f67ea0 blob([0x7cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57f67f80) put_ref 0x0~2000 Blob(0x558c57f67f80 blob([0x7cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6d90) put_ref 0x0~2000 Blob(0x558c57bb6d90 blob([0x7cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6e70) put_ref 0x0~2000 Blob(0x558c57bb6e70 blob([0x7d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6f50) put_ref 0x0~2000 Blob(0x558c57bb6f50 blob([0x7d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7030) put_ref 0x0~2000 Blob(0x558c57bb7030 blob([0x7d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7110) put_ref 0x0~2000 Blob(0x558c57bb7110 blob([0x7d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb71f0) put_ref 0x0~2000 Blob(0x558c57bb71f0 blob([0x7d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb72d0) put_ref 0x0~2000 Blob(0x558c57bb72d0 blob([0x7db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb73b0) put_ref 0x0~2000 Blob(0x558c57bb73b0 blob([0x7dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7490) put_ref 0x0~2000 Blob(0x558c57bb7490 blob([0x7df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7570) put_ref 0x0~2000 Blob(0x558c57bb7570 blob([0x7e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7650) put_ref 0x0~2000 Blob(0x558c57bb7650 blob([0x7e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7730) put_ref 0x0~2000 Blob(0x558c57bb7730 blob([0x7e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7810) put_ref 0x0~2000 Blob(0x558c57bb7810 blob([0x7e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb78f0) put_ref 0x0~2000 Blob(0x558c57bb78f0 blob([0x7e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb79d0) put_ref 0x0~2000 Blob(0x558c57bb79d0 blob([0x7eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7ab0) put_ref 0x0~2000 Blob(0x558c57bb7ab0 blob([0x7ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7b90) put_ref 0x0~2000 Blob(0x558c57bb7b90 blob([0x7ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7c70) put_ref 0x0~2000 Blob(0x558c57bb7c70 blob([0x7f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7d50) put_ref 0x0~2000 Blob(0x558c57bb7d50 blob([0x7f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7e30) put_ref 0x0~2000 Blob(0x558c57bb7e30 blob([0x7f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb7f10) put_ref 0x0~2000 Blob(0x558c57bb7f10 blob([0x7f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840e000) put_ref 0x0~2000 Blob(0x558c5840e000 blob([0x7f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840e0e0) put_ref 0x0~2000 Blob(0x558c5840e0e0 blob([0x7fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840e1c0) put_ref 0x0~2000 Blob(0x558c5840e1c0 blob([0x7fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840e2a0) put_ref 0x0~2000 Blob(0x558c5840e2a0 blob([0x7ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840e380) put_ref 0x0~2000 Blob(0x558c5840e380 blob([0x801000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c580ec000) put_ref 0x0~2000 Blob(0x558c580ec000 blob([0x803000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57a06770) put_ref 0x0~2000 Blob(0x558c57a06770 blob([0x805000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb65b0) put_ref 0x0~2000 Blob(0x558c57bb65b0 blob([0x807000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6690) put_ref 0x0~2000 Blob(0x558c57bb6690 blob([0x809000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6770) put_ref 0x0~2000 Blob(0x558c57bb6770 blob([0x80b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6850) put_ref 0x0~2000 Blob(0x558c57bb6850 blob([0x80d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6930) put_ref 0x0~2000 Blob(0x558c57bb6930 blob([0x80f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6a10) put_ref 0x0~2000 Blob(0x558c57bb6a10 blob([0x811000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6af0) put_ref 0x0~2000 Blob(0x558c57bb6af0 blob([0x813000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6bd0) put_ref 0x0~2000 Blob(0x558c57bb6bd0 blob([0x815000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6cb0) put_ref 0x0~2000 Blob(0x558c57bb6cb0 blob([0x817000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840fb20) put_ref 0x0~2000 Blob(0x558c5840fb20 blob([0x819000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840fc00) put_ref 0x0~2000 Blob(0x558c5840fc00 blob([0x81b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840fce0) put_ref 0x0~2000 Blob(0x558c5840fce0 blob([0x81d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.blob(0x558c5840fdc0) put_ref 0x0~1000 Blob(0x558c5840fdc0 blob([0x81f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c108) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c108) dirty_range mark shard 0x24000 dirty 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c108) dirty_range mark shard 0x48000 dirty 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57dcb490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7c3000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54df8690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7c5000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57f3b810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7c7000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57f67dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7c9000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57f67ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7cb000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57f67f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7cd000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57bb6d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7cf000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57bb6e70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7d1000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57bb6f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7d3000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57bb7030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7d5000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57bb7110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7d7000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57bb71f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7d9000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57bb72d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7db000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57bb73b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7dd000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57bb7490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7df000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c57bb7570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7e1000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57bb7650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7e3000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57bb7730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7e5000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c57bb7810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7e7000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c57bb78f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7e9000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c57bb79d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7eb000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c57bb7ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7ed000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c57bb7b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7ef000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c57bb7c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7f1000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c57bb7d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7f3000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c57bb7e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7f5000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c57bb7f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7f7000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~2000: 0x0~2000 Blob(0x558c5840e000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7f9000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x38000~2000: 0x0~2000 Blob(0x558c5840e0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7fb000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3a000~2000: 0x0~2000 Blob(0x558c5840e1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7fd000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3c000~2000: 0x0~2000 Blob(0x558c5840e2a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x7ff000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3e000~2000: 0x0~2000 Blob(0x558c5840e380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x801000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x40000~2000: 0x0~2000 Blob(0x558c580ec000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x803000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x42000~2000: 0x0~2000 Blob(0x558c57a06770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x805000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x44000~2000: 0x0~2000 Blob(0x558c57bb65b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x807000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x46000~2000: 0x0~2000 Blob(0x558c57bb6690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x809000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x48000~2000: 0x0~2000 Blob(0x558c57bb6770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x80b000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4a000~2000: 0x0~2000 Blob(0x558c57bb6850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x80d000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~2000: 0x0~2000 Blob(0x558c57bb6930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x80f000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4e000~2000: 0x0~2000 Blob(0x558c57bb6a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x811000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x50000~2000: 0x0~2000 Blob(0x558c57bb6af0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x813000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x52000~2000: 0x0~2000 Blob(0x558c57bb6bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x815000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x54000~2000: 0x0~2000 Blob(0x558c57bb6cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x817000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x56000~2000: 0x0~2000 Blob(0x558c5840fb20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x819000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x58000~2000: 0x0~2000 Blob(0x558c5840fc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x81b000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5a000~2000: 0x0~2000 Blob(0x558c5840fce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x81d000~2000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5c000~1000: 0x0~1000 Blob(0x558c5840fdc0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=2)))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x81f000~1000] 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x24000 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x48000 2019-12-07T19:34:28.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c5742b500,0x558c57bb6540,0x558c57bb6620,0x558c57bb6700,0x558c57bb67e0,0x558c57bb68c0,0x558c57bb69a0,0x558c57bb6a80,0x558c57bb6b60,0x558c57bb6c40,0x558c57bb6d20,0x558c57bb6e00,0x558c57bb6ee0,0x558c57bb6fc0,0x558c57bb70a0,0x558c57bb7180,0x558c57bb7260,0x558c57bb7340,0x558c57bb7420,0x558c57bb7500,0x558c57bb75e0,0x558c57bb76c0,0x558c57bb77a0,0x558c57bb7880,0x558c57bb7960,0x558c57bb7a40,0x558c57bb7b20,0x558c57bb7c00,0x558c57bb7ce0,0x558c57bb7dc0,0x558c57bb7ea0,0x558c57bb7f80,0x558c57e631f0,0x558c57f67d50,0x558c57f67e30,0x558c57f67f10,0x558c580ec070,0x558c5840e070,0x558c5840e150,0x558c5840e230,0x558c5840e310,0x558c5840e3f0,0x558c5840fab0,0x558c5840fb90,0x558c5840fc70,0x558c5840fd50,0x558c5840fe30 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c5840c780 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:28.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.8s5_head 5#1:10779340:::smithi13140796-32:head#29 = 0 2019-12-07T19:34:28.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:28.620+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c64000 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c64000 onodes shared_blobs 0x558c5742b500,0x558c57bb6540,0x558c57bb6620,0x558c57bb6700,0x558c57bb67e0,0x558c57bb68c0,0x558c57bb69a0,0x558c57bb6a80,0x558c57bb6b60,0x558c57bb6c40,0x558c57bb6d20,0x558c57bb6e00,0x558c57bb6ee0,0x558c57bb6fc0,0x558c57bb70a0,0x558c57bb7180,0x558c57bb7260,0x558c57bb7340,0x558c57bb7420,0x558c57bb7500,0x558c57bb75e0,0x558c57bb76c0,0x558c57bb77a0,0x558c57bb7880,0x558c57bb7960,0x558c57bb7a40,0x558c57bb7b20,0x558c57bb7c00,0x558c57bb7ce0,0x558c57bb7dc0,0x558c57bb7ea0,0x558c57bb7f80,0x558c57e631f0,0x558c57f67d50,0x558c57f67e30,0x558c57f67f10,0x558c580ec070,0x558c5840e070,0x558c5840e150,0x558c5840e230,0x558c5840e310,0x558c5840e3f0,0x558c5840fab0,0x558c5840fb90,0x558c5840fc70,0x558c5840fd50,0x558c5840fe30 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2963 is 11 SharedBlob(0x558c5742b500 loaded (sbid 0x2963 ref_map(0x7c5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2983 is 11 SharedBlob(0x558c57bb6540 loaded (sbid 0x2983 ref_map(0x805000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2984 is 11 SharedBlob(0x558c57bb6620 loaded (sbid 0x2984 ref_map(0x807000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2985 is 11 SharedBlob(0x558c57bb6700 loaded (sbid 0x2985 ref_map(0x809000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2986 is 11 SharedBlob(0x558c57bb67e0 loaded (sbid 0x2986 ref_map(0x80b000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2987 is 11 SharedBlob(0x558c57bb68c0 loaded (sbid 0x2987 ref_map(0x80d000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2988 is 11 SharedBlob(0x558c57bb69a0 loaded (sbid 0x2988 ref_map(0x80f000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2989 is 11 SharedBlob(0x558c57bb6a80 loaded (sbid 0x2989 ref_map(0x811000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298a is 11 SharedBlob(0x558c57bb6b60 loaded (sbid 0x298a ref_map(0x813000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298b is 11 SharedBlob(0x558c57bb6c40 loaded (sbid 0x298b ref_map(0x815000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2967 is 11 SharedBlob(0x558c57bb6d20 loaded (sbid 0x2967 ref_map(0x7cd000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2968 is 11 SharedBlob(0x558c57bb6e00 loaded (sbid 0x2968 ref_map(0x7cf000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2969 is 11 SharedBlob(0x558c57bb6ee0 loaded (sbid 0x2969 ref_map(0x7d1000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296a is 11 SharedBlob(0x558c57bb6fc0 loaded (sbid 0x296a ref_map(0x7d3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296b is 11 SharedBlob(0x558c57bb70a0 loaded (sbid 0x296b ref_map(0x7d5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296c is 11 SharedBlob(0x558c57bb7180 loaded (sbid 0x296c ref_map(0x7d7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296d is 11 SharedBlob(0x558c57bb7260 loaded (sbid 0x296d ref_map(0x7d9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296e is 11 SharedBlob(0x558c57bb7340 loaded (sbid 0x296e ref_map(0x7db000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x296f is 11 SharedBlob(0x558c57bb7420 loaded (sbid 0x296f ref_map(0x7dd000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2970 is 11 SharedBlob(0x558c57bb7500 loaded (sbid 0x2970 ref_map(0x7df000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2971 is 11 SharedBlob(0x558c57bb75e0 loaded (sbid 0x2971 ref_map(0x7e1000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2972 is 11 SharedBlob(0x558c57bb76c0 loaded (sbid 0x2972 ref_map(0x7e3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2973 is 11 SharedBlob(0x558c57bb77a0 loaded (sbid 0x2973 ref_map(0x7e5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2974 is 11 SharedBlob(0x558c57bb7880 loaded (sbid 0x2974 ref_map(0x7e7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2975 is 11 SharedBlob(0x558c57bb7960 loaded (sbid 0x2975 ref_map(0x7e9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2976 is 11 SharedBlob(0x558c57bb7a40 loaded (sbid 0x2976 ref_map(0x7eb000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2977 is 11 SharedBlob(0x558c57bb7b20 loaded (sbid 0x2977 ref_map(0x7ed000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2978 is 11 SharedBlob(0x558c57bb7c00 loaded (sbid 0x2978 ref_map(0x7ef000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2979 is 11 SharedBlob(0x558c57bb7ce0 loaded (sbid 0x2979 ref_map(0x7f1000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297a is 11 SharedBlob(0x558c57bb7dc0 loaded (sbid 0x297a ref_map(0x7f3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297b is 11 SharedBlob(0x558c57bb7ea0 loaded (sbid 0x297b ref_map(0x7f5000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297c is 11 SharedBlob(0x558c57bb7f80 loaded (sbid 0x297c ref_map(0x7f7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2962 is 11 SharedBlob(0x558c57e631f0 loaded (sbid 0x2962 ref_map(0x7c3000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2964 is 11 SharedBlob(0x558c57f67d50 loaded (sbid 0x2964 ref_map(0x7c7000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2965 is 11 SharedBlob(0x558c57f67e30 loaded (sbid 0x2965 ref_map(0x7c9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2966 is 11 SharedBlob(0x558c57f67f10 loaded (sbid 0x2966 ref_map(0x7cb000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2982 is 11 SharedBlob(0x558c580ec070 loaded (sbid 0x2982 ref_map(0x803000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297d is 11 SharedBlob(0x558c5840e070 loaded (sbid 0x297d ref_map(0x7f9000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297e is 11 SharedBlob(0x558c5840e150 loaded (sbid 0x297e ref_map(0x7fb000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x297f is 11 SharedBlob(0x558c5840e230 loaded (sbid 0x297f ref_map(0x7fd000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2980 is 11 SharedBlob(0x558c5840e310 loaded (sbid 0x2980 ref_map(0x7ff000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2981 is 11 SharedBlob(0x558c5840e3f0 loaded (sbid 0x2981 ref_map(0x801000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298c is 11 SharedBlob(0x558c5840fab0 loaded (sbid 0x298c ref_map(0x817000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298d is 11 SharedBlob(0x558c5840fb90 loaded (sbid 0x298d ref_map(0x819000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298e is 11 SharedBlob(0x558c5840fc70 loaded (sbid 0x298e ref_map(0x81b000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x298f is 11 SharedBlob(0x558c5840fd50 loaded (sbid 0x298f ref_map(0x81d000~2000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2990 is 11 SharedBlob(0x558c5840fe30 loaded (sbid 0x2990 ref_map(0x81f000~1000=1))) 2019-12-07T19:34:28.620+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c64000 allocated 0x[] released 0x[] 2019-12-07T19:34:28.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64000 prepare 2019-12-07T19:34:28.621+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c64000 2019-12-07T19:34:28.621+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64000 io_done 2019-12-07T19:34:28.621+0000 7f8f04982700 10 osd.7 51 dequeue_op 0x558c57f3f340 finish 2019-12-07T19:34:28.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2526 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c000 had 1 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2549 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2549 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29b12f~9f5 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29b12f 0x29b12f~9f5 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26b12f 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x12f 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb24 and padding block with 0x4dc 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29bb24 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000219988s (0.000000131s flush + 0.000219857s kv commit) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58c64000> 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64000 kv_submitted 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c64000 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64000 finishing 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c64000 onodes 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c64000 done 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c64000 [] 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 153 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1044+0+0 (crc 0 0 0) 0x558c58125800 con 0x558c54c19800 2019-12-07T19:34:28.621+0000 7f8f24c58700 15 osd.7 51 enqueue_peering_evt 1.5s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:28.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 154 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:28.621+0000 7f8f24c58700 15 osd.7 51 enqueue_peering_evt 1.ds2 epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:28.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 155 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c548a1c00 con 0x558c54c19800 2019-12-07T19:34:28.621+0000 7f8f24c58700 15 osd.7 51 enqueue_peering_evt 1.5s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:28.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 257 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c576b5400 con 0x558c54c5b400 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c54e79080 prio 196 cost 0 latency 0.000013 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.621+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e51) pg 0x558c57522000 2019-12-07T19:34:28.621+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:28.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 258 ==== MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=96, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57573b80 con 0x558c54c5b400 2019-12-07T19:34:28.621+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[c~1,11~1,17~1]] proc_primary_info updating purged_snaps to [] 2019-12-07T19:34:28.621+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c54e78420 prio 127 cost 0 latency 0.000016 epoch 51 MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=96, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:28.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:28.621+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=96, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5815f200 seq 17 2019-12-07T19:34:28.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815f200 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815f200 onodes shared_blobs 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815f200 allocated 0x[] released 0x[] 2019-12-07T19:34:28.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 259 ==== MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=97, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c58294500 con 0x558c54c5b400 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f200 prepare 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815f200 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f200 io_done 2019-12-07T19:34:28.621+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000020 epoch 51 MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=97, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:28.621+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=97, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e51) pg 0x558c55141000 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 260 ==== MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=98, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c58294780 con 0x558c54c5b400 2019-12-07T19:34:28.621+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c581cda20 prio 127 cost 0 latency 0.000009 epoch 51 MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=98, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:28.621+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.ds0( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:28.621+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=98, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29bb24~465 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29bb24 0x29bb24~465 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5815f500 seq 45 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26bb24 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb24 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf89 and padding block with 0x77 2019-12-07T19:34:28.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:28.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:28.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=96, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815f500 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815f500 onodes shared_blobs 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815f500 allocated 0x[] released 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f500 prepare 2019-12-07T19:34:28.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=96, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) pg 0x558c575ef000 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815f500 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f500 io_done 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.621+0000 7f8f07187700 10 osd.7 51 dequeue_op 0x558c54e78420 prio 127 cost 0 latency 0.000123 MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=96, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e51) pg 0x558c57522000 2019-12-07T19:34:28.621+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=96, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:28.621+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:28.621+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 51/41 ECSubWriteReply(tid=236, last_complete=51'41, committed=1, applied=1)) v2 -- 0x558c5795d000 con 0x558c54c64000 2019-12-07T19:34:28.621+0000 7f8f06185700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.621+0000 7f8f07187700 25 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:28.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:28.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~52000 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~52000 size 0x52000 (335872) 2019-12-07T19:34:28.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5815f800 seq 18 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:28.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29bf89 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5815f800 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5815f800 onodes shared_blobs 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5815f800 allocated 0x[] released 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f800 prepare 2019-12-07T19:34:28.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5815f800 2019-12-07T19:34:28.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f800 io_done 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132926s (0.000000111s flush + 0.000132815s kv commit) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2188 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2211 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2211 < min_flush_size 524288 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29bf89~8a3 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29bf89 0x29bf89~8a3 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26bf89 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf89 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x82c and padding block with 0x7d4 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5815f200> 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f200 kv_submitted 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815f200 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f200 finishing 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815f200 onodes 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815f200 done 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815f800 kv_submitted 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815f200 [] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) need 0x1000~1000 cache has 0x[] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) need {<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x23000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) need {<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2f000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) need {<0x0, 0x2000> : [0x31000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x31000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29c82c 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) need {<0x0, 0x2000> : [0x33000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x33000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) need {<0x0, 0x2000> : [0x35000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x35000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) need {<0x0, 0x2000> : [0x37000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x37000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000131891s (0.000000085s flush + 0.000131806s kv commit) 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5815f500,0x558c5815f800> 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) need {<0x0, 0x2000> : [0x39000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f500 kv_submitted 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x39000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815f500 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) need {<0x0, 0x2000> : [0x3b000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f500 finishing 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3b000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815f500 onodes 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815f500 done 2019-12-07T19:34:28.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815f500 [] 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) need {<0x0, 0x2000> : [0x3f000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3f000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f800 kv_submitted 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5815f800 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) need {<0x0, 0x2000> : [0x43000:0~2000]} 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5815f800 finishing 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x43000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5815f800 onodes 2019-12-07T19:34:28.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5815f800 done 2019-12-07T19:34:28.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5815f800 [] 2019-12-07T19:34:28.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.621+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) need {<0x0, 0x2000> : [0x45000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x45000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) need {<0x0, 0x2000> : [0x47000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x47000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) need {<0x0, 0x2000> : [0x49000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x49000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) need {<0x0, 0x2000> : [0x4b000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4b000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) need {<0x0, 0x2000> : [0x4d000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4d000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) need {<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x29000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2d000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) need {<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2b000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) need {<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x27000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need {<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x25000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) need {<0x0, 0x1000> : [0x22000:0~1000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~1000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) need {<0x0, 0x2000> : [0x3d000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3d000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) need {<0x0, 0x2000> : [0x41000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x41000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) need {<0x0, 0x2000> : [0x4f000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4f000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) need {<0x1000, 0x1000> : [0x51000:1000~1000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x51000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:28.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c54e79080 prio 196 cost 0 latency 0.000702 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=38, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:19 v=51'34 tt=0'0 tid=38 reqid=osd.7.0:38 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=48'32 crt=48'32 lcod 45'30 mlcod 45'30 active+clean+snaptrim trimq=[19~1]] repop_all_committed: repop tid 38 all committed 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] eval_repop repgather(0x558c576b6d80 51'34 rep_tid=38 committed?=1 r=0) (no op) 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] commit: repgather(0x558c576b6d80 51'34 rep_tid=38 committed?=1 r=0) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.622+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] publish_stats_to_osd 51:159 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] removing repgather(0x558c576b6d80 51'34 rep_tid=38 committed?=1 r=0) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] q front is repgather(0x558c576b6d80 51'34 rep_tid=38 committed?=1 r=0) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] exit Trimming/WaitRepops 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 51 queueing pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] for snaptrim 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=51) prio 5 cost 1048576 e51) 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=51) prio 5 cost 1048576 e51) queued 2019-12-07T19:34:28.622+0000 7f8f06986700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] publish_stats_to_osd 51:160 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] remove_repop repgather(0x558c576b6d80 51'34 rep_tid=38 committed?=1 r=0) 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=51'34 tt=0'0 tid=38 reqid=osd.7.0:38 roll_forward_to=48'32 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=39 reqid=osd.7.0:38 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=39 reqid=osd.7.0:38 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:28.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 51/28 ECSubWrite(tid=39, reqid=osd.7.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c57e7a800 con 0x558c54c64000 2019-12-07T19:34:28.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 51/28 ECSubWrite(tid=39, reqid=osd.7.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c57e7c000 con 0x558c54c19800 2019-12-07T19:34:28.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 51/28 ECSubWrite(tid=39, reqid=osd.7.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c551dc000 con 0x558c54c5bc00 2019-12-07T19:34:28.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 51/28 ECSubWrite(tid=39, reqid=osd.7.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c54d86800 con 0x558c54c5b400 2019-12-07T19:34:28.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 51/28 ECSubWrite(tid=39, reqid=osd.7.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c54d87000 con 0x558c54c19c00 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=48'32 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] append_log log((0'0,51'34], crt=48'32) [] 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] rollforward: entry=51'33 (48'31) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 snaps [e,13,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] rollforward: entry=51'34 (48'32) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:28.617077+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] append_log approx pg log length = 34 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:28.622+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.622+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:28.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c582ac900 seq 47 2019-12-07T19:34:28.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:28.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:28.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:28.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ac900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:28.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ac900 onodes shared_blobs 2019-12-07T19:34:28.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ac900 allocated 0x[] released 0x[] 2019-12-07T19:34:28.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac900 prepare 2019-12-07T19:34:28.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ac900 2019-12-07T19:34:28.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac900 io_done 2019-12-07T19:34:28.622+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c54e79080 finish 2019-12-07T19:34:28.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=51) prio 5 cost 1048576 e51) pg 0x558c57676000 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] snap_trimmer posting 2019-12-07T19:34:28.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 395 ==== MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=238, reqid=osd.1.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=51'27)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c574a5000 con 0x558c54c64000 2019-12-07T19:34:28.622+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000022 epoch 51 MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=238, reqid=osd.1.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=51'27)) v2 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 19 2019-12-07T19:34:28.622+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=238, reqid=osd.1.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=51'27)) v2) prio 127 cost 0 e51) 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c53aa2140) upper_bound after SNA_1_0000000000000019_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000019_.0_0000000000000001.C' 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c53aa2140) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000019_.0_0000000000000001.C) returns 0 SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:28.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim] SnapTrimmer state: adding snap 19 to purged_snaps 2019-12-07T19:34:28.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:28.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:28.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:28.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29c82c~471 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29c82c 0x29c82c~471 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26c82c 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x82c 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim ps=[19~1]] SnapTrimmer state: purged_snaps now [19~1], snap_trimq now [] 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc9d and padding block with 0x363 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f7200 seq 48 2019-12-07T19:34:28.622+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f7200 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f7200 onodes shared_blobs 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f7200 allocated 0x[] released 0x[] 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7200 prepare 2019-12-07T19:34:28.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f7200 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7200 io_done 2019-12-07T19:34:28.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=238, reqid=osd.1.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=51'27)) v2) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim ps=[19~1]] share_pg_info 2019-12-07T19:34:28.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=238, reqid=osd.1.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=51'27)) v2) prio 127 cost 0 e51) pg 0x558c54d5f000 2019-12-07T19:34:28.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c57378700 con 0x558c54c64000 2019-12-07T19:34:28.622+0000 7f8f0117b700 10 osd.7 51 dequeue_op 0x558c5740c2c0 prio 127 cost 0 latency 0.000167 MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=238, reqid=osd.1.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=51'27)) v2 pg pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={}] 2019-12-07T19:34:28.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s4 51/41 ECSubWrite(tid=238, reqid=osd.1.0:234, at_version=0'0, trim_to=0'0, roll_forward_to=51'27)) v2 2019-12-07T19:34:28.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c5737b800 con 0x558c54c19800 2019-12-07T19:34:28.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c57378e00 con 0x558c54c5bc00 2019-12-07T19:34:28.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c576dee00 con 0x558c54c5b400 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29cc9d 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.623+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e51/51) v1 -- 0x558c574e5c00 con 0x558c54c19c00 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000175461s (0.000000082s flush + 0.000175379s kv commit) 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim ps=[19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim ps=[19~1]] exit Trimming 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:28.623+0000 7f8f0297e700 15 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] publish_stats_to_osd 51:161 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] enter NotTrimming 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29cc9d~45d to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29cc9d 0x29cc9d~45d to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26cc9d 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc9d 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfa and padding block with 0xf06 2019-12-07T19:34:28.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:28.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] snap_trimmer complete 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582ac900> 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac900 kv_submitted 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ac900 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ac900 finishing 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ac900 onodes 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ac900 done 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f7200 kv_submitted 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ac900 [] 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29d0fa 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000155478s (0.000000084s flush + 0.000155394s kv commit) 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.622+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=50'25 active mbc={}] append_log log((0'0,51'27], crt=50'25) [] 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={}] rollforward: entry=51'26 (50'24) modify 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:28.613431+0000 0 snaps [1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={}] rollforward: entry=51'27 (50'25) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:28.613431+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 156 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c54c19800 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={}] append_log approx pg log length = 27 2019-12-07T19:34:28.623+0000 7f8f24c58700 15 osd.7 51 enqueue_op 0x558c5511ec60 prio 196 cost 0 latency 0.000012 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.623+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.623+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b5200 seq 70 2019-12-07T19:34:28.623+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b5200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b5200 onodes shared_blobs 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b5200 allocated 0x[] released 0x[] 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5200 prepare 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b5200 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5200 io_done 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 osd.7 51 dequeue_op 0x558c5740c2c0 finish 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585f7200> 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7200 kv_submitted 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f7200 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7200 finishing 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f7200 onodes 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f7200 done 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f7200 [] 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29d0fa~471 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29d0fa 0x29d0fa~471 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26d0fa 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfa 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x56b and padding block with 0xa95 2019-12-07T19:34:28.623+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c5511ec60 prio 196 cost 0 latency 0.000116 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] 2019-12-07T19:34:28.623+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.623+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c5511ec60 finish 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29d56b 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000154804s (0.000000062s flush + 0.000154742s kv commit) 2019-12-07T19:34:28.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586b5200> 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5200 kv_submitted 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b5200 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5200 finishing 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b5200 onodes 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b5200 done 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b5200 [] 2019-12-07T19:34:28.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 51/41 ECSubWriteReply(tid=238, last_complete=51'27, committed=1, applied=1)) v2 -- 0x558c57568800 con 0x558c54c64000 2019-12-07T19:34:28.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 232 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f400 con 0x558c54c5bc00 2019-12-07T19:34:28.623+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c581ccf20 prio 196 cost 0 latency 0.000020 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.623+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.623+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c581ccf20 prio 196 cost 0 latency 0.000042 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] 2019-12-07T19:34:28.623+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.623+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c581ccf20 finish 2019-12-07T19:34:28.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 396 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c54c64000 2019-12-07T19:34:28.623+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c5740d760 prio 196 cost 0 latency 0.000017 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.623+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 397 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e51/51) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c5737aa00 con 0x558c54c64000 2019-12-07T19:34:28.623+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c5740d760 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] 2019-12-07T19:34:28.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.623+0000 7f8f24457700 15 osd.7 51 enqueue_peering_evt 1.7s4 epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:28.623+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c5740d760 finish 2019-12-07T19:34:28.623+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) 2019-12-07T19:34:28.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) queued 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e51) pg 0x558c54d5f000 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={}] do_peering_event: epoch_sent: 51 epoch_requested: 51 MInfoRec from 1(0) info: 1.7s0( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={}] proc_primary_info updating purged_snaps to [19~1] 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:28.623+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b5500 seq 71 2019-12-07T19:34:28.623+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:28.623+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:28.624+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:28.624+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:28.624+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b5500 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:28.624+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b5500 onodes shared_blobs 2019-12-07T19:34:28.624+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b5500 allocated 0x[] released 0x[] 2019-12-07T19:34:28.624+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5500 prepare 2019-12-07T19:34:28.624+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b5500 2019-12-07T19:34:28.624+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5500 io_done 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.624+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29d56b~45d to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29d56b 0x29d56b~45d to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26d56b 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x56b 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9c8 and padding block with 0x638 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29d9c8 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131103s (0.000000077s flush + 0.000131026s kv commit) 2019-12-07T19:34:28.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586b5500> 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5500 kv_submitted 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b5500 2019-12-07T19:34:28.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5500 finishing 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b5500 onodes 2019-12-07T19:34:28.624+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b5500 done 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:28.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b5500 [] 2019-12-07T19:34:28.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.624+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:28.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 98 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57568800 con 0x558c54c19c00 2019-12-07T19:34:28.624+0000 7f8f24457700 15 osd.7 51 enqueue_op 0x558c5740c580 prio 196 cost 0 latency 0.000017 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.624+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c5740c580 prio 196 cost 0 latency 0.000045 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] 2019-12-07T19:34:28.624+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.624+0000 7f8f06986700 10 osd.7 51 dequeue_op 0x558c5740c580 finish 2019-12-07T19:34:28.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5840f438 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.624+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f76510 space 0x558c5840f438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c550b3668 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.624+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759ebd0 space 0x558c550b3668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55106e88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.624+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759ec60 space 0x558c55106e88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x31000:0~2000]} 2019-12-07T19:34:28.624+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55106f68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759ecf0 space 0x558c55106f68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x33000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107048 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759ed80 space 0x558c55107048 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x35000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107128 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759ee10 space 0x558c55107128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x37000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759eea0 space 0x558c55107208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x39000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c551072e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759ef30 space 0x558c551072e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3b000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c551073c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759efc0 space 0x558c551073c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3f000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f050 space 0x558c55107a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x43000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f0e0 space 0x558c55107b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x45000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107c18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f170 space 0x558c55107c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x47000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107cf8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f200 space 0x558c55107cf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x49000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107dd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f290 space 0x558c55107dd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4b000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107eb8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f320 space 0x558c55107eb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4d000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55107f98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f3b0 space 0x558c55107f98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54677828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f440 space 0x558c54677828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c550b2ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f4d0 space 0x558c550b2ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970be8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f560 space 0x558c57970be8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970cc8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f5f0 space 0x558c57970cc8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970da8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f680 space 0x558c57970da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b22b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f710 space 0x558c579b22b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c583ead38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f7a0 space 0x558c583ead38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c574fd978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f830 space 0x558c574fd978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c552790b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f8c0 space 0x558c552790b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576adba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f950 space 0x558c576adba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22000:0~1000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971908 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c5759f9e0 space 0x558c57971908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57f76120 space 0x558c57971978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579719e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c551a9200 space 0x558c579719e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c57ece7e0 space 0x558c57971a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971ac8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc750 space 0x558c57971ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971b38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc6c0 space 0x558c57971b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970e88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc630 space 0x558c57970e88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3d000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57567128 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc5a0 space 0x558c57567128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x41000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c54cf96d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc510 space 0x558c54cf96d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970b08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc480 space 0x558c57970b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4f000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c580a2088 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc3f0 space 0x558c580a2088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x51000:1000~1000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c580a2168 in 0x558c53a59c00) _discard 0x1000~1000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc360 space 0x558c580a2168 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57897c88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc2d0 space 0x558c57897c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad668 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc240 space 0x558c576ad668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c580d79e8 in 0x558c53a59c00) _discard 0x1000~1000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c580bc1b0 space 0x558c580d79e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971ba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 _add level 1 near 0 on buffer(0x558c58b84fc0 space 0x558c57971ba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:28.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~52000 = 335872 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read read request=335872 r=335872 len=335872 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:28.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:34:28.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read: Checking hash of 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.625+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 51/41 ECSubReadReply(tid=96, attrs_read=0)) v2 -- 0x558c57599680 con 0x558c54c5b400 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 51 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=97, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=97, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) pg 0x558c575ef000 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 51 dequeue_op 0x558c581cdb80 prio 127 cost 0 latency 0.003908 MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=97, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=97, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:28.625+0000 7f8f07187700 25 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:28.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~52000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~52000 size 0x52000 (335872) 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) need 0x1000~1000 cache has 0x[1000~1000] 2019-12-07T19:34:28.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~52000 = 335872 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read read request=335872 r=335872 len=335872 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:28.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:34:28.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read: Checking hash of 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.625+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 51/41 ECSubReadReply(tid=97, attrs_read=0)) v2 -- 0x558c57573b80 con 0x558c54c5b400 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 51 dequeue_op 0x558c581cdb80 finish 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=98, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=98, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e51) pg 0x558c575ef000 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 51 dequeue_op 0x558c581cda20 prio 127 cost 0 latency 0.004217 MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=98, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] 2019-12-07T19:34:28.625+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpRead(1.bs1 51/41 ECSubRead(tid=98, to_read={1:d07269f7:::smithi13140796-28:head=0,335872,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:28.625+0000 7f8f07187700 25 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:28.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~52000 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~52000 size 0x52000 (335872) 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:28.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) need 0x1000~1000 cache has 0x[1000~1000] 2019-12-07T19:34:28.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~52000 = 335872 2019-12-07T19:34:28.626+0000 7f8f07187700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read read request=335872 r=335872 len=335872 2019-12-07T19:34:28.626+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.626+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:28.626+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:28.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:34:28.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:34:28.626+0000 7f8f07187700 20 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] handle_sub_read: Checking hash of 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:28.626+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 51/41 ECSubReadReply(tid=98, attrs_read=0)) v2 -- 0x558c58294500 con 0x558c54c5b400 2019-12-07T19:34:28.626+0000 7f8f07187700 10 osd.7 51 dequeue_op 0x558c581cda20 finish 2019-12-07T19:34:28.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:28.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 261 ==== MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5763c800 con 0x558c54c5b400 2019-12-07T19:34:28.628+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c581cc6e0 prio 196 cost 0 latency 0.000021 epoch 51 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.628+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) 2019-12-07T19:34:28.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) queued 2019-12-07T19:34:28.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2) prio 196 cost 0 e51) pg 0x558c57676000 2019-12-07T19:34:28.628+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c581cc6e0 prio 196 cost 0 latency 0.000060 MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 pg pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] 2019-12-07T19:34:28.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 51/28 ECSubWriteReply(tid=39, last_complete=51'34, committed=1, applied=1)) v2 2019-12-07T19:34:28.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=39 reqid=osd.7.0:38 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:28.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:28.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:28.628+0000 7f8f0297e700 10 osd.7 51 dequeue_op 0x558c581cc6e0 finish 2019-12-07T19:34:28.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:28.628+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1306407 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.641+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 157 ==== MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=60, reqid=client.4540.0:387, at_version=51'14, trim_to=0'0, roll_forward_to=45'12)) v2 ==== 388450+0+0 (crc 0 0 0) 0x558c58157000 con 0x558c54c19800 2019-12-07T19:34:28.641+0000 7f8f24c58700 15 osd.7 51 enqueue_op 0x558c5511edc0 prio 127 cost 0 latency 0.000036 epoch 51 MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=60, reqid=client.4540.0:387, at_version=51'14, trim_to=0'0, roll_forward_to=45'12)) v2 2019-12-07T19:34:28.641+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=60, reqid=client.4540.0:387, at_version=51'14, trim_to=0'0, roll_forward_to=45'12)) v2) prio 127 cost 0 e51) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=60, reqid=client.4540.0:387, at_version=51'14, trim_to=0'0, roll_forward_to=45'12)) v2) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=60, reqid=client.4540.0:387, at_version=51'14, trim_to=0'0, roll_forward_to=45'12)) v2) prio 127 cost 0 e51) pg 0x558c55141000 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 osd.7 51 dequeue_op 0x558c5511edc0 prio 127 cost 0 latency 0.000083 MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=60, reqid=client.4540.0:387, at_version=51'14, trim_to=0'0, roll_forward_to=45'12)) v2 pg pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[19~1]] 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=60, reqid=client.4540.0:387, at_version=51'14, trim_to=0'0, roll_forward_to=45'12)) v2 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.add_oid 1:bca18876:::smithi13140796-30:1b e,13,15,1a,1b 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.get_snaps 1:bca18876:::smithi13140796-30:1b got.empty() 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.set_snaps 1:bca18876:::smithi13140796-30:1b e,13,15,1a,1b 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_0000000000000013_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 45'12 (0'0,45'12] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[19~1]] append_log log((0'0,45'12], crt=45'12) [51'13 (21'2) clone 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [1b,1a,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,51'14 (21'2) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:387 0.000000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 1, new_object: 0] 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'13 (0'0,51'13] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=45'12 crt=45'12 lcod 45'12 active mbc={} ps=[19~1]] add_log_entry 51'13 (21'2) clone 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [1b,1a,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=45'12 crt=45'12 lcod 45'12 active mbc={} ps=[19~1]] add_log_entry 51'14 (21'2) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:387 0.000000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=45'12 crt=45'12 lcod 45'12 active mbc={} ps=[19~1]] append_log approx pg log length = 14 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=45'12 crt=45'12 lcod 45'12 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.641+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 51'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57a57200 seq 46 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bca18876:::smithi13140796-30:1b# key 0x828000000000000001bca188'v!smithi13140796-30!='0x000000000000001bffffffffffffffff'o' 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:1b# 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1247 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.onode(0x558c57454a00).flush flush done 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 0x0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:1b# 0x0~1e000 -> 0x0~1e000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x0~2000: 0x0~2000 Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10641 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3340 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3340 blob([0xfe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c57583ea0 blob([0xfe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ea0) get_ref 0x0~2000 Blob(0x558c57583ea0 blob([0xfe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57583ea0 blob([0xfe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10642 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e36c0 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e36c0 blob([0x100000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c578df960 blob([0x100000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578df960) get_ref 0x0~2000 Blob(0x558c578df960 blob([0x100000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578df960) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c578df960 blob([0x100000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10643 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e35e0 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e35e0 blob([0x102000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c5809abd0 blob([0x102000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c5809abd0) get_ref 0x0~2000 Blob(0x558c5809abd0 blob([0x102000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c5809abd0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c5809abd0 blob([0x102000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10644 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3500 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3500 blob([0x104000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c57583ce0 blob([0x104000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ce0) get_ref 0x0~2000 Blob(0x558c57583ce0 blob([0x104000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57583ce0 blob([0x104000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10645 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e33b0 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e33b0 blob([0x106000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c578f1030 blob([0x106000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578f1030) get_ref 0x0~2000 Blob(0x558c578f1030 blob([0x106000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578f1030) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c578f1030 blob([0x106000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10646 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3730 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3730 blob([0x108000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c578dfab0 blob([0x108000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578dfab0) get_ref 0x0~2000 Blob(0x558c578dfab0 blob([0x108000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578dfab0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c578dfab0 blob([0x108000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10647 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3810 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3810 blob([0x10a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c57583a40 blob([0x10a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583a40) get_ref 0x0~2000 Blob(0x558c57583a40 blob([0x10a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583a40) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57583a40 blob([0x10a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10648 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e38f0 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e38f0 blob([0x10c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c578dfa40 blob([0x10c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578dfa40) get_ref 0x0~2000 Blob(0x558c578dfa40 blob([0x10c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578dfa40) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c578dfa40 blob([0x10c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10649 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e39d0 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e39d0 blob([0x10e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c54676f50 blob([0x10e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c54676f50) get_ref 0x0~2000 Blob(0x558c54676f50 blob([0x10e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c54676f50) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c54676f50 blob([0x10e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10650 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3ab0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3ab0 blob([0x110000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c57f513b0 blob([0x110000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57f513b0) get_ref 0x0~2000 Blob(0x558c57f513b0 blob([0x110000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57f513b0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57f513b0 blob([0x110000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10651 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3b90 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3b90 blob([0x112000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c57583960 blob([0x112000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583960) get_ref 0x0~2000 Blob(0x558c57583960 blob([0x112000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583960) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57583960 blob([0x112000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10652 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3c70 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3c70 blob([0x114000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c578f10a0 blob([0x114000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578f10a0) get_ref 0x0~2000 Blob(0x558c578f10a0 blob([0x114000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578f10a0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c578f10a0 blob([0x114000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10653 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3d50 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3d50 blob([0x116000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c57583880 blob([0x116000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583880) get_ref 0x0~2000 Blob(0x558c57583880 blob([0x116000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c57583880) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57583880 blob([0x116000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10654 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3e30 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3e30 blob([0x118000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c578f0070 blob([0x118000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0070) get_ref 0x0~2000 Blob(0x558c578f0070 blob([0x118000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c578f0070) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c578f0070 blob([0x118000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10655 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574e3f10 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 sbid 0x0)) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574e3f10 blob([0x11a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=1)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup new Blob(0x558c575837a0 blob([0x11a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c575837a0) get_ref 0x0~2000 Blob(0x558c575837a0 blob([0x11a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.blob(0x558c575837a0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c575837a0 blob([0x11a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=2)))) 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dirty_range mark inline shard dirty 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96888) dirty_range mark inline shard dirty 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:1b# = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# snapset 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# snapset = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bca18876:::smithi13140796-30:head#e key 0x828000000000000001bca188'v!smithi13140796-30!='0xfffffffffffffffe000000000000000e'o' 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:head#e 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:head#e = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1248 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 3 keys 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 3 keys = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:28.641+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~5e000 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 2#1:bca18876:::smithi13140796-30:head# 0x0~5e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:28.641+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~5e000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57a57200 47 blobs 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.641+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:28.642+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate 0x5e000/1000,5e000,0 2019-12-07T19:34:28.642+0000 7f8f0217d700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xae6000~5e000/1000,5e000,0 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xae6000~5e000] 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575836c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575836c0 blob([0xae6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51340 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575836c0) get_ref 0x0~2000 Blob(0x558c575836c0 blob([0xae6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51340 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575836c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c575836c0 blob([0xae6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f51358 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575835e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575835e0 blob([0xae8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1110 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575835e0) get_ref 0x0~2000 Blob(0x558c575835e0 blob([0xae8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1110 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575835e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c575835e0 blob([0xae8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c578f1128 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583260 blob([0xaea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57583260) get_ref 0x0~2000 Blob(0x558c57583260 blob([0xaea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57583260) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c57583260 blob([0xaea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c578f0168 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f00e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538a80 blob([0xaec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f00e0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57538a80) get_ref 0x0~2000 Blob(0x558c57538a80 blob([0xaec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f00e0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57538a80) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57538a80 blob([0xaec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c578f00f8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575389a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f512d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575389a0 blob([0xaee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f512d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575389a0) get_ref 0x0~2000 Blob(0x558c575389a0 blob([0xaee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f512d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575389a0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c575389a0 blob([0xaee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f512e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575388c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575388c0 blob([0xaf0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1180 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575388c0) get_ref 0x0~2000 Blob(0x558c575388c0 blob([0xaf0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1180 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575388c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c575388c0 blob([0xaf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c578f1198 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575387e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e63d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575387e0 blob([0xaf2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63d50 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575387e0) get_ref 0x0~2000 Blob(0x558c575387e0 blob([0xaf2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63d50 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c575387e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c575387e0 blob([0xaf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57e63d68 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57538690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574fd9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57538690 blob([0xaf4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd9d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57538690) get_ref 0x0~2000 Blob(0x558c57538690 blob([0xaf4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd9d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57538690) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57538690 blob([0xaf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574fd9e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c546772d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c546772d0 blob([0xaf6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1340 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c546772d0) get_ref 0x0~2000 Blob(0x558c546772d0 blob([0xaf6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1340 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c546772d0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c546772d0 blob([0xaf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c578f1358 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809b420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809b420 blob([0xaf8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8230 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c5809b420) get_ref 0x0~2000 Blob(0x558c5809b420 blob([0xaf8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8230 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c5809b420) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c5809b420 blob([0xaf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c54cf8248 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c42460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f51810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c42460 blob([0xafa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51810 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57c42460) get_ref 0x0~2000 Blob(0x558c57c42460 blob([0xafa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51810 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57c42460) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57c42460 blob([0xafa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57f51828 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794c690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550604d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794c690 blob([0xafc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550604d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c5794c690) get_ref 0x0~2000 Blob(0x558c5794c690 blob([0xafc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550604d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c5794c690) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c5794c690 blob([0xafc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c550604e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ec380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ec380 blob([0xafe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5960 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c580ec380) get_ref 0x0~2000 Blob(0x558c580ec380 blob([0xafe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5960 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c580ec380) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c580ec380 blob([0xafe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57db5978 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bea1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bea1c0 blob([0xb00000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4310 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57bea1c0) get_ref 0x0~2000 Blob(0x558c57bea1c0 blob([0xb00000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4310 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57bea1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57bea1c0 blob([0xb00000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57ff4328 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060230 blob([0xb02000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d9d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c55060230) get_ref 0x0~2000 Blob(0x558c55060230 blob([0xb02000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d9d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c55060230) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c55060230 blob([0xb02000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5794d9e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d3ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a06bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d3ab0 blob([0xb04000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06bd0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c574d3ab0) get_ref 0x0~2000 Blob(0x558c574d3ab0 blob([0xb04000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06bd0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c574d3ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c574d3ab0 blob([0xb04000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57a06be8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db55e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db55e0 blob([0xb06000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5570 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57db55e0) get_ref 0x0~2000 Blob(0x558c57db55e0 blob([0xb06000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5570 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57db55e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57db55e0 blob([0xb06000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57db5588 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bea540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5340 blob([0xb08000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea540 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57db5340) get_ref 0x0~2000 Blob(0x558c57db5340 blob([0xb08000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea540 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57db5340) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57db5340 blob([0xb08000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57bea558 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2310 blob([0xb0a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5500 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c574d2310) get_ref 0x0~2000 Blob(0x558c574d2310 blob([0xb0a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5500 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c574d2310) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c574d2310 blob([0xb0a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57db5518 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57881260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d2cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57881260 blob([0xb0c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2cb0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57881260) get_ref 0x0~2000 Blob(0x558c57881260 blob([0xb0c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2cb0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57881260) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57881260 blob([0xb0c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c574d2cc8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efc0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a07f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efc0e0 blob([0xb0e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07f80 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57efc0e0) get_ref 0x0~2000 Blob(0x558c57efc0e0 blob([0xb0e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07f80 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57efc0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57efc0e0 blob([0xb0e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57a07f98 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a07960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a07960 blob([0xb10000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b3b0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57a07960) get_ref 0x0~2000 Blob(0x558c57a07960 blob([0xb10000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b3b0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57a07960) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57a07960 blob([0xb10000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5809b3c8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff4150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57beac40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff4150 blob([0xb12000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beac40 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57ff4150) get_ref 0x0~2000 Blob(0x558c57ff4150 blob([0xb12000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beac40 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c57ff4150) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57ff4150 blob([0xb12000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c57beac58 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df8930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df8930 blob([0xb14000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b180 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c54df8930) get_ref 0x0~2000 Blob(0x558c54df8930 blob([0xb14000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b180 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c54df8930) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c54df8930 blob([0xb14000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c5809b198 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c67c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5500 blob([0xb16000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67c00 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c580a5500) get_ref 0x0~2000 Blob(0x558c580a5500 blob([0xb16000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67c00 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c580a5500) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c580a5500 blob([0xb16000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c67c18 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c67c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c67c70 blob([0xb18000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ce0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67c70) get_ref 0x0~2000 Blob(0x558c58c67c70 blob([0xb18000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ce0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67c70) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c58c67c70 blob([0xb18000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c67cf8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c67d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c67dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c67d50 blob([0xb1a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67dc0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67d50) get_ref 0x0~2000 Blob(0x558c58c67d50 blob([0xb1a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67dc0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67d50) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c58c67d50 blob([0xb1a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c67dd8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c67e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c67e30 blob([0xb1c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ea0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67e30) get_ref 0x0~2000 Blob(0x558c58c67e30 blob([0xb1c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ea0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67e30) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c58c67e30 blob([0xb1c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c67eb8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c67f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c67f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c67f10 blob([0xb1e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67f80 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67f10) get_ref 0x0~2000 Blob(0x558c58c67f10 blob([0xb1e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67f80 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67f10) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c58c67f10 blob([0xb1e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c67f98 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78000 blob([0xb20000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78070 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78000) get_ref 0x0~2000 Blob(0x558c58c78000 blob([0xb20000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78070 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78000) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c58c78000 blob([0xb20000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78088 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c780e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c780e0 blob([0xb22000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78150 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c780e0) get_ref 0x0~2000 Blob(0x558c58c780e0 blob([0xb22000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78150 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c780e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c58c780e0 blob([0xb22000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78168 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c781c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c781c0 blob([0xb24000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78230 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c781c0) get_ref 0x0~2000 Blob(0x558c58c781c0 blob([0xb24000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78230 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c781c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c58c781c0 blob([0xb24000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78248 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c782a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c782a0 blob([0xb26000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78310 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c782a0) get_ref 0x0~2000 Blob(0x558c58c782a0 blob([0xb26000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78310 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c782a0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c58c782a0 blob([0xb26000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78328 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c783f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78380 blob([0xb28000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c783f0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78380) get_ref 0x0~2000 Blob(0x558c58c78380 blob([0xb28000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c783f0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78380) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c58c78380 blob([0xb28000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78408 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c784d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78460 blob([0xb2a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c784d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78460) get_ref 0x0~2000 Blob(0x558c58c78460 blob([0xb2a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c784d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78460) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c58c78460 blob([0xb2a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c784e8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c785b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78540 blob([0xb2c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c785b0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78540) get_ref 0x0~2000 Blob(0x558c58c78540 blob([0xb2c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c785b0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78540) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c58c78540 blob([0xb2c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c785c8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78620 blob([0xb2e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78690 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78620) get_ref 0x0~2000 Blob(0x558c58c78620 blob([0xb2e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78690 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78620) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c58c78620 blob([0xb2e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c786a8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78700 blob([0xb30000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78770 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78700) get_ref 0x0~2000 Blob(0x558c58c78700 blob([0xb30000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78770 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78700) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c58c78700 blob([0xb30000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78788 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c787e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c787e0 blob([0xb32000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78850 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c787e0) get_ref 0x0~2000 Blob(0x558c58c787e0 blob([0xb32000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78850 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c787e0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c58c787e0 blob([0xb32000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78868 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c788c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c788c0 blob([0xb34000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78930 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c788c0) get_ref 0x0~2000 Blob(0x558c58c788c0 blob([0xb34000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78930 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c788c0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c58c788c0 blob([0xb34000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78948 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c789a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c789a0 blob([0xb36000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78a10 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c789a0) get_ref 0x0~2000 Blob(0x558c58c789a0 blob([0xb36000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78a10 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c789a0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c58c789a0 blob([0xb36000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78a28 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78a80 blob([0xb38000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78af0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78a80) get_ref 0x0~2000 Blob(0x558c58c78a80 blob([0xb38000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78af0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78a80) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c58c78a80 blob([0xb38000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78b08 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78b60 blob([0xb3a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78bd0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78b60) get_ref 0x0~2000 Blob(0x558c58c78b60 blob([0xb3a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78bd0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78b60) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c58c78b60 blob([0xb3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78be8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78c40 blob([0xb3c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78cb0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78c40) get_ref 0x0~2000 Blob(0x558c58c78c40 blob([0xb3c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78cb0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78c40) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c58c78c40 blob([0xb3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78cc8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78d20 blob([0xb3e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78d90 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78d20) get_ref 0x0~2000 Blob(0x558c58c78d20 blob([0xb3e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78d90 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78d20) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c58c78d20 blob([0xb3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78da8 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78e00 blob([0xb40000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78e70 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78e00) get_ref 0x0~2000 Blob(0x558c58c78e00 blob([0xb40000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78e70 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78e00) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c58c78e00 blob([0xb40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78e88 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c78ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c78f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c78ee0 blob([0xb42000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78f50 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78ee0) get_ref 0x0~2000 Blob(0x558c58c78ee0 blob([0xb42000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78f50 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~2000: 0x0~2000 Blob(0x558c58c78ee0 blob([0xb42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 sbid 0x0)) 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.BufferSpace(0x558c58c78f68 in 0x558c53a59dc0) _discard 0x0~2000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5e000 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dirty_range mark inline shard dirty 2019-12-07T19:34:28.642+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0~5e000 = 0 2019-12-07T19:34:28.642+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:28.642+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:28.642+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.642+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.642+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_000000000000000E_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_0000000000000013_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_0000000000000015_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_000000000000001B_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:28.642+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:28.642+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000051.00000000000000000013' <- 0000000051.00000000000000000013 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000051.00000000000000000014' <- 0000000051.00000000000000000014 2019-12-07T19:34:28.642+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.643+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:28.643+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a57200 cost 582119 (48 ios * 4000 + 390119 bytes) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a57200 onodes 0x558c54d96780,0x558c54e13900,0x558c57454a00 shared_blobs 0x558c54d3d1f0,0x558c574e32d0,0x558c574e3420,0x558c574e3570,0x558c574e3650,0x558c574e37a0,0x558c574e3880,0x558c574e3960,0x558c574e3a40,0x558c574e3b20,0x558c574e3c00,0x558c574e3ce0,0x558c574e3dc0,0x558c574e3ea0,0x558c574e3f80 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96888) update 2#1:bca18876:::smithi13140796-30:1b# 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96888) update inline shard 408 bytes from 15 extents 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:1b# is 826 (412 bytes onode + 2 bytes spanning blobs + 412 bytes inline extents) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update inline shard 896 bytes from 47 extents 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 1475 (573 bytes onode + 2 bytes spanning blobs + 900 bytes inline extents) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) update 2#1:bca18876:::smithi13140796-30:head#e 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) update inline shard 408 bytes from 15 extents 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head#e is 899 (485 bytes onode + 2 bytes spanning blobs + 412 bytes inline extents) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2991 is 11 SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2995 is 11 SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2994 is 11 SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2993 is 11 SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2992 is 11 SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2996 is 11 SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2997 is 11 SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2998 is 11 SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2999 is 11 SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299a is 11 SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299b is 11 SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299c is 11 SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299d is 11 SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299e is 11 SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299f is 11 SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=2))) 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a57200 allocated 0x[ae6000~5e000] released 0x[] 2019-12-07T19:34:28.643+0000 7f8f0217d700 10 freelist allocate 0xae6000~5e000 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 freelist _xor first_key 0xa80000 last_key 0xb00000 2019-12-07T19:34:28.643+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57200 prepare 2019-12-07T19:34:28.643+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57a57200 2019-12-07T19:34:28.643+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=45'12 crt=45'12 lcod 45'12 active mbc={} ps=[19~1]] op_applied version 51'14 2019-12-07T19:34:28.643+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.643+0000 7f8f0217d700 10 osd.7 51 dequeue_op 0x558c5511edc0 finish 2019-12-07T19:34:28.643+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.643+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57200 aio_wait 2019-12-07T19:34:28.643+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a57200 2019-12-07T19:34:28.643+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57200 io_done 2019-12-07T19:34:28.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:28.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6786 < min_flush_size 524288 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96780 had 1 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13900 had 1 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454a00 had 1 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6809 < min_flush_size 524288 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6809 < min_flush_size 524288 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29d9c8~1a99 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29d9c8 0x29d9c8~1a99 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26d9c8 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9c8 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x461 and padding block with 0xb9f 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29f461 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000370542s (0.000242838s flush + 0.000127704s kv commit) 2019-12-07T19:34:28.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a57200> 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57200 kv_submitted 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a57200 2019-12-07T19:34:28.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57200 finishing 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a57200 onodes 0x558c54d96780,0x558c54e13900,0x558c57454a00 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8248 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acf30 space 0x558c54cf8248 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550604e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578ad050 space 0x558c550604e8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d2cc8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd5f0 space 0x558c574d2cc8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574fd9e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578ace10 space 0x558c574fd9e8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f00f8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acbd0 space 0x558c578f00f8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0168 in 0x558c53a59dc0) _finish_write discard buffer(0x558c573d9710 space 0x558c578f0168 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1128 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acab0 space 0x558c578f1128 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1198 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578accf0 space 0x558c578f1198 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1358 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acea0 space 0x558c578f1358 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d9e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bc090 space 0x558c5794d9e8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a06be8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd3b0 space 0x558c57a06be8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a07f98 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd680 space 0x558c57a07f98 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bea558 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd4d0 space 0x558c57bea558 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57beac58 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd7a0 space 0x558c57beac58 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 51/41 ECSubWriteReply(tid=60, last_complete=51'14, committed=1, applied=1)) v2 -- 0x558c58512000 con 0x558c54c19800 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5518 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd560 space 0x558c57db5518 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5588 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd440 space 0x558c57db5588 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5978 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578ad0e0 space 0x558c57db5978 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e63d68 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acd80 space 0x558c57e63d68 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f512e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acc60 space 0x558c57f512e8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51358 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acb40 space 0x558c57f51358 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f51828 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578acfc0 space 0x558c57f51828 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ff4328 in 0x558c53a59dc0) _finish_write discard buffer(0x558c578ad170 space 0x558c57ff4328 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b198 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd830 space 0x558c5809b198 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b3c8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd710 space 0x558c5809b3c8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c67c18 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd8c0 space 0x558c58c67c18 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c67cf8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd950 space 0x558c58c67cf8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c67dd8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bd9e0 space 0x558c58c67dd8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c67eb8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bda70 space 0x558c58c67eb8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c67f98 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bdb00 space 0x558c58c67f98 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78088 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bdb90 space 0x558c58c78088 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78168 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bdc20 space 0x558c58c78168 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78248 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bdcb0 space 0x558c58c78248 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78328 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bdd40 space 0x558c58c78328 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78408 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bddd0 space 0x558c58c78408 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c784e8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bde60 space 0x558c58c784e8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c785c8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c580bdef0 space 0x558c58c785c8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c786a8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e000 space 0x558c58c786a8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78788 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e090 space 0x558c58c78788 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78868 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e120 space 0x558c58c78868 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78948 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e1b0 space 0x558c58c78948 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78a28 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e240 space 0x558c58c78a28 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78b08 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e2d0 space 0x558c58c78b08 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78be8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e360 space 0x558c58c78be8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78cc8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e3f0 space 0x558c58c78cc8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78da8 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e480 space 0x558c58c78da8 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78e88 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e510 space 0x558c58c78e88 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c78f68 in 0x558c53a59dc0) _finish_write discard buffer(0x558c58c8e5a0 space 0x558c58c78f68 0x0~2000 writing nocache) 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a57200 done 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:28.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a57200 [] 2019-12-07T19:34:28.644+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.644+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 158 ==== MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=61, reqid=client.4540.0:387, at_version=0'0, trim_to=0'0, roll_forward_to=51'14)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c58156800 con 0x558c54c19800 2019-12-07T19:34:28.644+0000 7f8f24c58700 15 osd.7 51 enqueue_op 0x558c5511ef20 prio 127 cost 0 latency 0.000022 epoch 51 MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=61, reqid=client.4540.0:387, at_version=0'0, trim_to=0'0, roll_forward_to=51'14)) v2 2019-12-07T19:34:28.644+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=61, reqid=client.4540.0:387, at_version=0'0, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e51) 2019-12-07T19:34:28.644+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=61, reqid=client.4540.0:387, at_version=0'0, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e51) queued 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=61, reqid=client.4540.0:387, at_version=0'0, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e51) pg 0x558c55141000 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 osd.7 51 dequeue_op 0x558c5511ef20 prio 127 cost 0 latency 0.000431 MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=61, reqid=client.4540.0:387, at_version=0'0, trim_to=0'0, roll_forward_to=51'14)) v2 pg pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[19~1]] 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 51/41 ECSubWrite(tid=61, reqid=client.4540.0:387, at_version=0'0, trim_to=0'0, roll_forward_to=51'14)) v2 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=45'12 active mbc={} ps=[19~1]] append_log log((0'0,51'14], crt=45'12) [] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] rollforward: entry=51'13 (21'2) clone 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [1b,1a,15,13,e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] rollforward: entry=51'14 (21'2) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:387 0.000000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] append_log approx pg log length = 14 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:28.645+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57a57500 seq 47 2019-12-07T19:34:28.645+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bca18876:::smithi13140796-30:head#e onode 0x558c57454a00 txc 0x558c57a57500 2019-12-07T19:34:28.645+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bca18876:::smithi13140796-30:head#e 0x0 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3340) put_ref 0x0~2000 Blob(0x558c574e3340 blob([0xfe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e36c0) put_ref 0x0~2000 Blob(0x558c574e36c0 blob([0x100000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e35e0) put_ref 0x0~2000 Blob(0x558c574e35e0 blob([0x102000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3500) put_ref 0x0~2000 Blob(0x558c574e3500 blob([0x104000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e33b0) put_ref 0x0~2000 Blob(0x558c574e33b0 blob([0x106000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3730) put_ref 0x0~2000 Blob(0x558c574e3730 blob([0x108000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3810) put_ref 0x0~2000 Blob(0x558c574e3810 blob([0x10a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e38f0) put_ref 0x0~2000 Blob(0x558c574e38f0 blob([0x10c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e39d0) put_ref 0x0~2000 Blob(0x558c574e39d0 blob([0x10e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3ab0) put_ref 0x0~2000 Blob(0x558c574e3ab0 blob([0x110000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3b90) put_ref 0x0~2000 Blob(0x558c574e3b90 blob([0x112000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3c70) put_ref 0x0~2000 Blob(0x558c574e3c70 blob([0x114000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3d50) put_ref 0x0~2000 Blob(0x558c574e3d50 blob([0x116000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3e30) put_ref 0x0~2000 Blob(0x558c574e3e30 blob([0x118000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3f10) put_ref 0x0~2000 Blob(0x558c574e3f10 blob([0x11a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57454b08) dirty_range mark inline shard dirty 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574e3340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xfe000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c574e36c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x100000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c574e35e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x102000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574e3500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x104000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c574e33b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x106000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c574e3730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x108000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c574e3810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x10a000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c574e38f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x10c000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c574e39d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x10e000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c574e3ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x110000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c574e3b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x112000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c574e3c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x114000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c574e3d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x116000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c574e3e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x118000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c574e3f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=2)))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x11a000~2000] 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3d1f0,0x558c574e32d0,0x558c574e3420,0x558c574e3570,0x558c574e3650,0x558c574e37a0,0x558c574e3880,0x558c574e3960,0x558c574e3a40,0x558c574e3b20,0x558c574e3c00,0x558c574e3ce0,0x558c574e3dc0,0x558c574e3ea0,0x558c574e3f80 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c54e13900 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bca18876:::smithi13140796-30:head#e = 0 2019-12-07T19:34:28.645+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a57500 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a57500 onodes shared_blobs 0x558c54d3d1f0,0x558c574e32d0,0x558c574e3420,0x558c574e3570,0x558c574e3650,0x558c574e37a0,0x558c574e3880,0x558c574e3960,0x558c574e3a40,0x558c574e3b20,0x558c574e3c00,0x558c574e3ce0,0x558c574e3dc0,0x558c574e3ea0,0x558c574e3f80 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2991 is 11 SharedBlob(0x558c54d3d1f0 loaded (sbid 0x2991 ref_map(0xfe000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2995 is 11 SharedBlob(0x558c574e32d0 loaded (sbid 0x2995 ref_map(0x106000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2994 is 11 SharedBlob(0x558c574e3420 loaded (sbid 0x2994 ref_map(0x104000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2993 is 11 SharedBlob(0x558c574e3570 loaded (sbid 0x2993 ref_map(0x102000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2992 is 11 SharedBlob(0x558c574e3650 loaded (sbid 0x2992 ref_map(0x100000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2996 is 11 SharedBlob(0x558c574e37a0 loaded (sbid 0x2996 ref_map(0x108000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2997 is 11 SharedBlob(0x558c574e3880 loaded (sbid 0x2997 ref_map(0x10a000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2998 is 11 SharedBlob(0x558c574e3960 loaded (sbid 0x2998 ref_map(0x10c000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2999 is 11 SharedBlob(0x558c574e3a40 loaded (sbid 0x2999 ref_map(0x10e000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299a is 11 SharedBlob(0x558c574e3b20 loaded (sbid 0x299a ref_map(0x110000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299b is 11 SharedBlob(0x558c574e3c00 loaded (sbid 0x299b ref_map(0x112000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299c is 11 SharedBlob(0x558c574e3ce0 loaded (sbid 0x299c ref_map(0x114000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299d is 11 SharedBlob(0x558c574e3dc0 loaded (sbid 0x299d ref_map(0x116000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299e is 11 SharedBlob(0x558c574e3ea0 loaded (sbid 0x299e ref_map(0x118000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x299f is 11 SharedBlob(0x558c574e3f80 loaded (sbid 0x299f ref_map(0x11a000~2000=1))) 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a57500 allocated 0x[] released 0x[] 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57500 prepare 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a57500 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57500 io_done 2019-12-07T19:34:28.645+0000 7f8f0217d700 10 osd.7 51 dequeue_op 0x558c5511ef20 finish 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1581 < min_flush_size 524288 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454a00 had 1 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29f461~644 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29f461 0x29f461~644 to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~270000 x_off 0x26f461 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x461 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xaa5 and padding block with 0x55b 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x29faa5 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:28.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000126312s (0.000000105s flush + 0.000126207s kv commit) 2019-12-07T19:34:28.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a57500> 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:28.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57500 kv_submitted 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a57500 2019-12-07T19:34:28.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57500 finishing 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a57500 onodes 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a57500 done 2019-12-07T19:34:28.645+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:28.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a57500 [] 2019-12-07T19:34:28.645+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:28.645+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 51/41 ECSubWriteReply(tid=61, last_complete=51'14, committed=1, applied=1)) v2 -- 0x558c58512200 con 0x558c54c19800 2019-12-07T19:34:28.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:28.645+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:28.678+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 10 osd.7 51 tick_without_osd_lock 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f4bc000/0x40000000/0x1680000000, data 0xc81547/0xb42000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 25 osd.7 51 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875883+0000 last_rx_front 2019-12-07T19:34:27.875916+0000 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 25 osd.7 51 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875931+0000 last_rx_front 2019-12-07T19:34:27.875870+0000 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 25 osd.7 51 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875893+0000 last_rx_front 2019-12-07T19:34:27.875903+0000 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 25 osd.7 51 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875766+0000 last_rx_front 2019-12-07T19:34:27.875813+0000 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 25 osd.7 51 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875796+0000 last_rx_front 2019-12-07T19:34:27.875837+0000 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 20 osd.7 51 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 20 osd.7 51 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 20 osd.7 51 scrub_load_below_threshold loadavg per cpu 0.04125 < max 5 = yes 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 20 osd.7 51 sched_scrub load_is_low=1 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 10 osd.7 51 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:08.239446+0000 > 2019-12-07T19:34:28.695314+0000 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 20 osd.7 51 sched_scrub done 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 10 osd.7 51 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 20 osd.7 51 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:28.694+0000 7f8f1fbbe700 10 osd.7 51 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:28.728+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.778+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.828+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.878+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.928+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:28.948+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:28.953+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 16 ==== osd_ping(ping e51 up_from 13 ping_stamp 2019-12-07T19:34:28.955000+0000/52.412513217s send_stamp 52.412513217s delta_ub 0.022420715s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c54c18000 2019-12-07T19:34:28.954+0000 7f8f23c56700 20 osd.7 51 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021855058s,0.022420715s]) 2019-12-07T19:34:28.954+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e51 up_from 13 ping_stamp 2019-12-07T19:34:28.955000+0000/52.412513217s send_stamp 52.390658159s delta_ub 18446744073.687696558s) v5 -- 0x558c54c58800 con 0x558c54c18000 2019-12-07T19:34:28.954+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 16 ==== osd_ping(ping e51 up_from 13 ping_stamp 2019-12-07T19:34:28.955000+0000/52.412513217s send_stamp 52.412513217s delta_ub 0.022420715s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c548cda00 con 0x558c552d1400 2019-12-07T19:34:28.954+0000 7f8f24457700 20 osd.7 51 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021836395s,0.022420715s]) 2019-12-07T19:34:28.954+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e51 up_from 13 ping_stamp 2019-12-07T19:34:28.955000+0000/52.412513217s send_stamp 52.390676822s delta_ub 18446744073.687715221s) v5 -- 0x558c58114200 con 0x558c552d1400 2019-12-07T19:34:28.979+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.029+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.079+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.129+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.179+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.229+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.269+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54d42400 0x558c552d6b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:29.269+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54d42400 0x558c552d6b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:29.269+0000 7f8f24457700 10 osd.7 51 ms_handle_authentication new session 0x558c57405860 con 0x558c54d42400 entity client.admin addr 2019-12-07T19:34:29.269+0000 7f8f24457700 10 osd.7 51 ms_handle_authentication session 0x558c57405860 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:29.269+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3080288806 conn(0x558c54d42400 0x558c552d6b00 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4514 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:29.269+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4514 172.21.15.131:0/3080288806 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c5734ad20 con 0x558c54d42400 2019-12-07T19:34:29.269+0000 7f8f164bb700 20 osd.7 51 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:29.269+0000 7f8f164bb700 10 osd.7 51 do_waiters -- start 2019-12-07T19:34:29.269+0000 7f8f164bb700 10 osd.7 51 do_waiters -- finish 2019-12-07T19:34:29.269+0000 7f8f164bb700 20 osd.7 51 _dispatch 0x558c5734ad20 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:29.270+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3080288806 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce1c00 con 0x558c54d42400 2019-12-07T19:34:29.279+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.293+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4514 172.21.15.131:0/3080288806 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c575cc5a0 con 0x558c54d42400 2019-12-07T19:34:29.293+0000 7f8f164bb700 20 osd.7 51 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:29.293+0000 7f8f164bb700 10 osd.7 51 do_waiters -- start 2019-12-07T19:34:29.293+0000 7f8f164bb700 10 osd.7 51 do_waiters -- finish 2019-12-07T19:34:29.293+0000 7f8f164bb700 20 osd.7 51 _dispatch 0x558c575cc5a0 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:29.293+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3080288806 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce1dc0 con 0x558c54d42400 2019-12-07T19:34:29.296+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3080288806 conn(0x558c54d42400 msgr2=0x558c552d6b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 72 2019-12-07T19:34:29.296+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3080288806 conn(0x558c54d42400 msgr2=0x558c552d6b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:29.296+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3080288806 conn(0x558c54d42400 0x558c552d6b00 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:29.296+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3080288806 conn(0x558c54d42400 0x558c552d6b00 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:29.296+0000 7f8f164bb700 2 osd.7 51 ms_handle_reset con 0x558c54d42400 session 0x558c57405860 2019-12-07T19:34:29.316+0000 7f8f21451700 10 osd.7 51 tick 2019-12-07T19:34:29.316+0000 7f8f21451700 10 osd.7 51 do_waiters -- start 2019-12-07T19:34:29.316+0000 7f8f21451700 10 osd.7 51 do_waiters -- finish 2019-12-07T19:34:29.316+0000 7f8f21451700 20 osd.7 51 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:29.330+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.380+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.430+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.480+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.531+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.581+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1327456 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 233 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c576dee00 con 0x558c54c5bc00 2019-12-07T19:34:29.616+0000 7f8f23c56700 15 osd.7 51 enqueue_peering_evt 1.1s1 epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:29.616+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e52) pg 0x558c5736a000 2019-12-07T19:34:29.616+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 52, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.617+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 235 ==== MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=137, reqid=osd.3.0:137, at_version=52'59, trim_to=0'0, roll_forward_to=47'57)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c551dc000 con 0x558c54c5bc00 2019-12-07T19:34:29.617+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000038 epoch 52 MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=137, reqid=osd.3.0:137, at_version=52'59, trim_to=0'0, roll_forward_to=47'57)) v2 2019-12-07T19:34:29.617+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=137, reqid=osd.3.0:137, at_version=52'59, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=137, reqid=osd.3.0:137, at_version=52'59, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=137, reqid=osd.3.0:137, at_version=52'59, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e52) pg 0x558c54d5c000 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 51 dequeue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000087 MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=137, reqid=osd.3.0:137, at_version=52'59, trim_to=0'0, roll_forward_to=47'57)) v2 pg pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 51 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 51 -> 52 (as per caller) 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=137, reqid=osd.3.0:137, at_version=52'59, trim_to=0'0, roll_forward_to=47'57)) v2 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6ad15c4c:::smithi13140796-31:13 e was 2019-12-07T19:34:29.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.617+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 234 ==== osd_map(52..52 src has 1..52) v4 ==== 661+0+0 (crc 0 0 0) 0x558c579698c0 con 0x558c54c5bc00 2019-12-07T19:34:29.617+0000 7f8f1199c700 20 osd.7 51 OSD::ms_dispatch: osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.617+0000 7f8f1199c700 10 osd.7 51 do_waiters -- start 2019-12-07T19:34:29.617+0000 7f8f1199c700 10 osd.7 51 do_waiters -- finish 2019-12-07T19:34:29.617+0000 7f8f1199c700 20 osd.7 51 _dispatch 0x558c579698c0 osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6ad15c4c:::smithi13140796-31:13 e,13 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6ad15c4c:::smithi13140796-31:13 e 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:29.617+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:29.617+0000 7f8f1199c700 3 osd.7 51 handle_osd_map epochs [52,52], i have 51, src has [1,52] 2019-12-07T19:34:29.617+0000 7f8f1199c700 10 osd.7 51 handle_osd_map got inc map for epoch 52 2019-12-07T19:34:29.617+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3714e826:::osdmap.51:0# 0x0~0 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 47'57 (0'0,47'57] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] append_log log((0'0,47'57], crt=47'57) [52'58 (45'47) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:29.618062+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'59 (45'48) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:29.618062+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1480 size 0x1480 (5248) 2019-12-07T19:34:29.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:29.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c58b83960 blob([0xae4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b838f0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'58 (0'0,52'58] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=47'57 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] add_log_entry 52'58 (45'47) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:29.618062+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.617+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c58b83880 blob([!~1000,0xae5000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,480]) SharedBlob(0x558c58b83810 sbid 0x0)) need 0x1000~480 cache has 0x[1000~480] 2019-12-07T19:34:29.617+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3714e826:::osdmap.51:0# 0x0~1480 = 5248 2019-12-07T19:34:29.617+0000 7f8f1199c700 10 osd.7 51 add_map_bl 51 5248 bytes 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'59 (0'0,52'59] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=47'57 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] add_log_entry 52'59 (45'48) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:29.618062+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'59 (0'0,52'59] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=47'57 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] append_log approx pg log length = 59 2019-12-07T19:34:29.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'59 (0'0,52'59] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=47'57 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.618+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'58, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582ae300 seq 94 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# 1 keys = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 osd.7 51 got_full_map 52, nothing requested 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_0000000000000013_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000052.00000000000000000058' <- 0000000052.00000000000000000058 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000052.00000000000000000059' <- 0000000052.00000000000000000059 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 osd.7 51 handle_osd_map pg_num_history pg_num_history(e52 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {52={}} 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 osd.7 51 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e52 [1,52] lci=[13,52]) 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ae300 cost 8349 (1 ios * 4000 + 4349 bytes) 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ae300 onodes 0x558c54ce2280,0x558c57e8f900 shared_blobs 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 573 (571 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c5829bb00 seq 43 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57e8fa08) update 2#1:6ad15c4c:::smithi13140796-31:13# 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c507972f:::inc_osdmap.52:0# key 0x7f7fffffffffffffffc50797'/!inc_osdmap.52!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:13# is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ae300 allocated 0x[] released 0x[] 2019-12-07T19:34:29.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c507972f:::inc_osdmap.52:0# 0x0~269 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1249 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c507972f:::inc_osdmap.52:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae300 prepare 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ae300 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae300 io_done 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829bb00 1 blobs 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb44000~1000/1000,1000,0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb44000~1000] 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829de30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58b836c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829de30 blob([0xb44000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b836c0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c5829de30) get_ref 0x0~269 Blob(0x558c5829de30 blob([0xb44000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b836c0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'59 (0'0,52'59] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=47'57 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] op_applied version 52'59 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c5829de30) get_ref init 0x1000, 1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c5829de30 blob([0xb44000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c58b836c0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c58b836d8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 51 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295c88) dirty_range mark inline shard dirty 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c507972f:::inc_osdmap.52:0# 0x0~269 = 0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3d94e826:::osdmap.52:0# key 0x7f7fffffffffffffff3d94e8'&!osdmap.52!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:29.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3d94e826:::osdmap.52:0# 0x0~1454 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1250 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3d94e826:::osdmap.52:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~454 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829bb00 2 blobs 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb45000~2000/1000,2000,0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb45000~2000] 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b835e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829c380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b835e0 blob([0xb45000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c380 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c58b835e0) get_ref 0x0~1000 Blob(0x558c58b835e0 blob([0xb45000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c380 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c58b835e0) get_ref init 0x1000, 1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c58b835e0 blob([0xb45000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5829c380 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5829c398 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829c310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829d9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829c310 blob([!~1000,0xb46000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d9d0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c5829c310) get_ref 0x1000~454 Blob(0x558c5829c310 blob([!~1000,0xb46000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c5829d9d0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c5829c310) get_ref init 0x2000, 1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558c5829c310 blob([!~1000,0xb46000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c5829d9d0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5829d9e8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1454 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e108) dirty_range mark inline shard dirty 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3d94e826:::osdmap.52:0# 0x0~1454 = 0 2019-12-07T19:34:29.618+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829bb00 0 blobs 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:29.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:29.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:29.618+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829bb00 0 blobs 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5829bb00 cost 23302 (4 ios * 4000 + 7302 bytes) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5829bb00 onodes 0x558c53a91900,0x558c55276c80,0x558c58295b80,0x558c5851e000 shared_blobs 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295c88) update #-1:c507972f:::inc_osdmap.52:0# 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58295c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c507972f:::inc_osdmap.52:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e108) update #-1:3d94e826:::osdmap.52:0# 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e108) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3d94e826:::osdmap.52:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1371 < min_flush_size 524288 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5829bb00 allocated 0x[b44000~3000] released 0x[] 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4245 < min_flush_size 524288 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 freelist allocate 0xb44000~3000 2019-12-07T19:34:29.618+0000 7f8f1199c700 20 freelist _xor first_key 0xb00000 last_key 0xb00000 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829bb00 prepare 2019-12-07T19:34:29.618+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5829bb00 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f900 had 1 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3858104320 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998084) bluestore 86 GiB free (0.959877), bluefs_ratio 0.0399317 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4268 < min_flush_size 524288 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4268 < min_flush_size 524288 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x29faa5~10ac to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x29faa5 0x29faa5~10ac to file(ino 16 size 0x2a0000 mtime 2019-12-07T19:34:28.618590+0000 allocated 2a0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~270000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _allocate len 0xb51 from 0 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _allocate len 0xb51 from 1 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd3d0000 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd3d0000~10000/10000,10000,acd3d0000 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x2b0000 to include allocated 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 59 (was clean) 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x26faa5 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xaa5 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb51 and padding block with 0x4af 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x2b0000 2019-12-07T19:34:29.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 236 ==== MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=138, reqid=osd.3.0:138, at_version=52'61, trim_to=0'0, roll_forward_to=47'57)) v2 ==== 3426+0+0 (crc 0 0 0) 0x558c54d86800 con 0x558c54c5bc00 2019-12-07T19:34:29.618+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c57f3f4a0 prio 127 cost 0 latency 0.000028 epoch 52 MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=138, reqid=osd.3.0:138, at_version=52'61, trim_to=0'0, roll_forward_to=47'57)) v2 2019-12-07T19:34:29.618+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=138, reqid=osd.3.0:138, at_version=52'61, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=138, reqid=osd.3.0:138, at_version=52'61, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=138, reqid=osd.3.0:138, at_version=52'61, trim_to=0'0, roll_forward_to=47'57)) v2) prio 127 cost 0 e52) pg 0x558c54d5c000 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 51 dequeue_op 0x558c57f3f4a0 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=138, reqid=osd.3.0:138, at_version=52'61, trim_to=0'0, roll_forward_to=47'57)) v2 pg pg[1.6s2( v 52'59 (0'0,52'59] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'59 (0'0,52'59] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=138, reqid=osd.3.0:138, at_version=52'61, trim_to=0'0, roll_forward_to=47'57)) v2 2019-12-07T19:34:29.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829bb00 aio_wait 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6336c6f2:::smithi13140796-16:17 e,15 was 2019-12-07T19:34:29.618+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5829bb00 2019-12-07T19:34:29.618+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829bb00 io_done 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6336c6f2:::smithi13140796-16:17 e,13,15 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6336c6f2:::smithi13140796-16:17 e,15 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2a0b51 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'59 (0'0,52'59] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] append_log log((0'0,52'59], crt=47'57) [52'60 (47'56) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:29.618968+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'61 (47'57) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:29.618968+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'60 (0'0,52'60] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'59 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] add_log_entry 52'60 (47'56) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:29.618968+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'59 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] add_log_entry 52'61 (47'57) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:29.618968+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'59 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] append_log approx pg log length = 61 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'59 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.618+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'60, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (59) on file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]), flushing log 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 59 len 0x4a crc 0x938f447d) 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x3a000~1000 to file(ino 1 size 0x3a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x3a000 0x3a000~1000 to file(ino 1 size 0x3a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x3b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3a000 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582ae600 seq 95 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x3b000 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:29.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_0000000000000013_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000052.00000000000000000060' <- 0000000052.00000000000000000060 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000052.00000000000000000061' <- 0000000052.00000000000000000061 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ae600 cost 7599 (1 ios * 4000 + 3599 bytes) 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ae600 onodes 0x558c54ce3900,0x558c57f8e500 shared_blobs 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1587 (566 bytes onode + 2 bytes spanning blobs + 1019 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e608) update 2#1:6336c6f2:::smithi13140796-16:17# 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:17# is 637 (412 bytes onode + 2 bytes spanning blobs + 223 bytes inline extents) 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ae600 allocated 0x[] released 0x[] 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae600 prepare 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ae600 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae600 io_done 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'59 crt=47'57 lcod 47'57 active mbc={} ps=[19~1]] op_applied version 52'61 2019-12-07T19:34:29.618+0000 7f8f0197c700 10 osd.7 51 dequeue_op 0x558c57f3f4a0 finish 2019-12-07T19:34:29.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000265 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 59 2019-12-07T19:34:29.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 262 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c576dee00 con 0x558c54c5b400 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000701472s (0.000000195s flush + 0.000701277s kv commit) 2019-12-07T19:34:29.619+0000 7f8f23c56700 15 osd.7 51 enqueue_peering_evt 1.3s4 epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:29.619+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582ae300> 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae300 kv_submitted 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ae300 2019-12-07T19:34:29.619+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 263 ==== osd_map(52..52 src has 1..52) v4 ==== 661+0+0 (unknown 0 0 0) 0x558c57969680 con 0x558c54c5b400 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae300 finishing 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ae300 onodes 0x558c54ce2280,0x558c57e8f900 2019-12-07T19:34:29.619+0000 7f8f1199c700 20 osd.7 51 OSD::ms_dispatch: osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ae300 done 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:29.619+0000 7f8f1199c700 10 osd.7 51 do_waiters -- start 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ae600 kv_queued 2019-12-07T19:34:29.619+0000 7f8f1199c700 10 osd.7 51 do_waiters -- finish 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295b80 had 1 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ae300 [] 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851e000 had 1 2019-12-07T19:34:29.619+0000 7f8f1199c700 20 osd.7 51 _dispatch 0x558c57969680 osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13066 < min_flush_size 524288 2019-12-07T19:34:29.619+0000 7f8f1199c700 3 osd.7 51 handle_osd_map epochs [52,52], i have 52, src has [1,52] 2019-12-07T19:34:29.619+0000 7f8f1199c700 10 osd.7 51 no new maps here, dropping 2019-12-07T19:34:29.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 264 ==== MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=99, reqid=osd.4.0:99, at_version=52'31, trim_to=0'0, roll_forward_to=45'29)) v2 ==== 3426+0+0 (unknown 0 0 0) 0x558c54d84000 con 0x558c54c5b400 2019-12-07T19:34:29.619+0000 7f8f23c56700 15 osd.7 51 enqueue_op 0x558c57f3fce0 prio 127 cost 0 latency 0.000026 epoch 52 MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=99, reqid=osd.4.0:99, at_version=52'31, trim_to=0'0, roll_forward_to=45'29)) v2 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e500 had 1 2019-12-07T19:34:29.619+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=99, reqid=osd.4.0:99, at_version=52'31, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13089 < min_flush_size 524288 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 13089 < min_flush_size 524288 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2a0b51~3321 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2a0b51 0x2a0b51~3321 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x270b51 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb51 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe72 and padding block with 0x18e 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e52) pg 0x558c5513e000 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 52, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=99, reqid=osd.4.0:99, at_version=52'31, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=99, reqid=osd.4.0:99, at_version=52'31, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e52) pg 0x558c575ef000 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 51 dequeue_op 0x558c57f3fce0 prio 127 cost 0 latency 0.000093 MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=99, reqid=osd.4.0:99, at_version=52'31, trim_to=0'0, roll_forward_to=45'29)) v2 pg pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 51 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 51 -> 52 (as per caller) 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=99, reqid=osd.4.0:99, at_version=52'31, trim_to=0'0, roll_forward_to=45'29)) v2 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 snap_mapper.update_snaps 1:dce63cc3:::smithi13140796-50:15 e,15 was 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 51/28 ECSubWriteReply(tid=137, last_complete=52'59, committed=1, applied=1)) v2 -- 0x558c580b9a00 con 0x558c54c5bc00 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 snap_mapper.get_snaps 1:dce63cc3:::smithi13140796-50:15 e,13,15 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 snap_mapper.set_snaps 1:dce63cc3:::smithi13140796-50:15 e,15 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:29.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 45'29 (0'0,45'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] append_log log((0'0,45'29], crt=45'29) [52'30 (45'22) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:29.619393+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'31 (45'23) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:29.619393+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 52'30 (0'0,52'30] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'29 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] add_log_entry 52'30 (45'22) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:29.619393+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'29 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] add_log_entry 52'31 (45'23) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:29.619393+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'29 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] append_log approx pg log length = 31 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'29 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.619+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'30, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2a3e72 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000293199s (0.000005067s flush + 0.000288132s kv commit) 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5829bb00,0x558c582ae600> 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829bb00 kv_submitted 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5829bb00 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5810e600 seq 38 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c5829bb00 osr 0x558c54ea4b40 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 40 0x1b000~1000 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7 51 _committed_osd_maps 52..52 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7 51 advance to epoch 52 (<= last 52 <= newest_map 52) 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 41 0x1b000~1000 crc 4eeb0cd8 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 40 0x2000~1000 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 41 0x2000~1000 crc 2a88dc94 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys = 0 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae600 kv_submitted 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ae600 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) 2019-12-07T19:34:29.619+0000 7f8f141a1700 7 osd.7 52 consume_map version 52 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae600 finishing 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ae600 onodes 0x558c54ce3900,0x558c57f8e500 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.8s5 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ae600 done 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.0s3 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ae600 [] 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.1s1 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.9s4 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.as5 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.2s0 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:34:29.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.bs1 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.3s4 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.cs0 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.4s0 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.5s4 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.ds2 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.es4 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.6s2 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.fs5 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 identify_splits_and_merges 1.7s4 e51 to e52 pg_nums {18=16} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 52 prune_pg_created 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_0000000000000013_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:29.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 51/28 ECSubWriteReply(tid=138, last_complete=52'61, committed=1, applied=1)) v2 -- 0x558c580b9c00 con 0x558c54c5bc00 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:0.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:1.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:29.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 pending_peering first epoch 52 <= 52, requeueing 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000052.00000000000000000030' <- 0000000052.00000000000000000030 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {52=} 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000052.00000000000000000031' <- 0000000052.00000000000000000031 2019-12-07T19:34:29.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:2.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:3.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 52 <= 52, requeueing 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {52=} 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810e600 cost 8389 (1 ios * 4000 + 4389 bytes) 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810e600 onodes 0x558c574a0a00,0x558c574a0c80 shared_blobs 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:4.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a0b08) update 1#1:dce63cc3:::smithi13140796-50:15# 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:5.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:6.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:15# is 1120 (412 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7:7.consume_map 52 (was 51) 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a0d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 1273 (565 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:29.619+0000 7f8f141a1700 10 osd.7 52 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810e600 allocated 0x[] released 0x[] 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.8s5 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e600 prepare 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810e600 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e600 io_done 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e52) pg 0x558c5736a000 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.0s3 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.1s1 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.9s4 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e52) pg 0x558c5513e000 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=45'29 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] op_applied version 52'31 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.as5 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f0317f700 10 osd.7 52 dequeue_op 0x558c57f3fce0 finish 2019-12-07T19:34:29.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.2s0 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.bs1 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.3s4 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.cs0 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:29.619+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.4s0 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.5s4 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c57380000 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.ds2 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.es4 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.6s2 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.fs5 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f141a1700 15 osd.7 52 enqueue_peering_evt 1.7s4 epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c57676000 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:29.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c57522000 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:29.619+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:29.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:29.619+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean ps=[19~1]] state: Active advmap 2019-12-07T19:34:29.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] on_active_advmap new removed_snaps [13~1], snap_trimq now [13~1] 2019-12-07T19:34:29.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c58112900 seq 20 2019-12-07T19:34:29.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:29.619+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:29.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58112900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:29.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58112900 onodes shared_blobs 2019-12-07T19:34:29.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58112900 allocated 0x[] released 0x[] 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112900 prepare 2019-12-07T19:34:29.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58112900 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112900 io_done 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c57383000 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c5500d000 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c5736a000 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c5500a000 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] state: Active: handling ActMap 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 51 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] Active: kicking snap trim 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 51 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 51 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] exit NotTrimming 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] enter Trimming 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f141a1700 7 osd.7 52 activate_map version 52 2019-12-07T19:34:29.619+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f0918b700 20 osd.7 52 agent_entry empty queue 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f06986700 15 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] publish_stats_to_osd 51:162 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f06986700 7 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] state: handle ActMap primary 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c54d5f000 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f06986700 15 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] publish_stats_to_osd 51:163 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 51 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.619+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c548ab000 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f0b990700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f0b990700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] SnapTrimmer state: NotTrimming: trimming 13 2019-12-07T19:34:29.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f0b990700 15 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:164 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 51 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:29.619+0000 7f8f0b990700 10 osd.7 52 queueing pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean trimq=[13~1] ps=[19~1]] for snaptrim 2019-12-07T19:34:29.619+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=52) prio 5 cost 1048576 e52) 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[19~1]] state: Active advmap 2019-12-07T19:34:29.619+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.619+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:29.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] on_active_advmap new removed_snaps [13~1], snap_trimq now [13~1] 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c57e8b000 2019-12-07T19:34:29.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=52) prio 5 cost 1048576 e52) queued 2019-12-07T19:34:29.619+0000 7f8f0b990700 15 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:165 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=52) prio 5 cost 1048576 e52) pg 0x558c57676000 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:29.619+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] snap_trimmer posting 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 51 -> 52 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 13 2019-12-07T19:34:29.619+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f06986700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.619+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.619+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.619+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.619+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] activate_map 2019-12-07T19:34:29.619+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] state: Active: handling ActMap 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] Active: kicking snap trim 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f03980700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.619+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:29.619+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] exit NotTrimming 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c54d98000 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] enter Trimming 2019-12-07T19:34:29.620+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c57e88000 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 51 -> 52 2019-12-07T19:34:29.620+0000 7f8f06986700 15 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] publish_stats_to_osd 51:129 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:29.620+0000 7f8f06986700 7 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] state: handle ActMap primary 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 51 -> 52 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.620+0000 7f8f06986700 15 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] publish_stats_to_osd 51:130 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 51 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:29.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[19~1]] state: Active advmap 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] on_active_advmap new removed_snaps [13~1], snap_trimq now [13~1] 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.619+0000 7f8f07187700 20 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.620+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] agent_stop 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.3s0( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c54d5c000 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] state: Active: handling ActMap 2019-12-07T19:34:29.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] Active: kicking snap trim 2019-12-07T19:34:29.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 51 -> 52 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 51 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:29.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c58522000 seq 60 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] exit NotTrimming 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] enter Trimming 2019-12-07T19:34:29.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.620+0000 7f8f0b990700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:29.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:29.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:29.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:29.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.620+0000 7f8f0b990700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] SnapTrimmer state: NotTrimming: trimming 13 2019-12-07T19:34:29.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58522000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58522000 onodes shared_blobs 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:29.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58522000 allocated 0x[] released 0x[] 2019-12-07T19:34:29.620+0000 7f8f03980700 15 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] publish_stats_to_osd 51:143 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522000 prepare 2019-12-07T19:34:29.620+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58522000 2019-12-07T19:34:29.620+0000 7f8f03980700 7 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] state: handle ActMap primary 2019-12-07T19:34:29.620+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522000 io_done 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f0b990700 15 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:131 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:29.620+0000 7f8f03980700 15 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] publish_stats_to_osd 51:144 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c575ef000 2019-12-07T19:34:29.620+0000 7f8f0b990700 10 osd.7 52 queueing pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[13~1] ps=[19~1]] for snaptrim 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=52) prio 5 cost 1048576 e52) 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f0b990700 15 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:132 2019-12-07T19:34:29.619+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 51 -> 52 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 51 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5236 < min_flush_size 524288 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c55141000 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 237 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c576dee00 con 0x558c54c5bc00 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6370 < min_flush_size 524288 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] handle_advance_map: 52 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 51 -> 52 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7504 < min_flush_size 524288 2019-12-07T19:34:29.620+0000 7f8f23c56700 15 osd.7 52 enqueue_peering_evt 1.fs5 epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 51 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:29.620+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] state: Started advmap 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7527 < min_flush_size 524288 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7527 < min_flush_size 524288 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 NullEvt) prio 255 cost 10 e52) pg 0x558c5513e000 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2a3e72~1d67 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] plpg_on_pool_change 2019-12-07T19:34:29.620+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2a3e72 0x2a3e72~1d67 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] hit_set_clear 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x273e72 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe72 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbd9 and padding block with 0x427 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] agent_stop 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=52) prio 5 cost 1048576 e52) queued 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 52 advance_pg new pool opts old pool opts 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=52) prio 5 cost 1048576 e52) pg 0x558c548ab000 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] handle_activate_map: 52 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[13~1] ps=[19~1]] snap_trimmer posting 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] activate_map 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] activate_map: Not dirtying info: last_persisted is 51 while current is 52 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 13 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fe40) upper_bound after SNA_1_0000000000000013_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.C' 2019-12-07T19:34:29.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 NullEvt 2019-12-07T19:34:29.620+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:29.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fe40) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:29.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000013_.0_0000000000000001.C) returns 0 SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239600 in raw 0x558c55239600 len 69 nref 1) ) 2019-12-07T19:34:29.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e52) pg 0x558c57383000 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c58b7fec0) upper_bound after SNA_1_0000000000000013_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.4' 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.620+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c58b7fec0) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:29.620+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fe40) upper_bound after SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. key 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:29.620+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000013_.0_0000000000000001.4) returns 0 SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239810 in raw 0x558c55239810 len 69 nref 1) ) 2019-12-07T19:34:29.620+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fe40) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:29.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:29.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c582a8900 seq 22 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..) returns 0 SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[19~1]] SnapTrimmer state: adding snap 13 to purged_snaps 2019-12-07T19:34:29.620+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:29.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:29.620+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:29.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:29.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:29.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a8900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a8900 onodes shared_blobs 2019-12-07T19:34:29.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a8900 allocated 0x[] released 0x[] 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_object_context: found obc in cache: 0x558c5738f8c0 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[13~1,19~1]] SnapTrimmer state: purged_snaps now [13~1,19~1], snap_trimq now [] 2019-12-07T19:34:29.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c582acc00 seq 41 2019-12-07T19:34:29.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8900 prepare 2019-12-07T19:34:29.620+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a8900 2019-12-07T19:34:29.620+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8900 io_done 2019-12-07T19:34:29.620+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_object_context: 0x558c5738f8c0 1:33f5bf01:::smithi13140796-2:19 rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:19(51'33 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[15,13,e]} 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:29.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:29.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582acc00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582acc00 onodes shared_blobs 2019-12-07T19:34:29.620+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582acc00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acc00 prepare 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(51'34 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[15,13,e]} 2019-12-07T19:34:29.620+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582acc00 2019-12-07T19:34:29.620+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acc00 io_done 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 1:33f5bf01:::smithi13140796-2:19 old_snaps e,13,15 old snapset 19=[]:{19=[15,13,e]} 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[13~1,19~1]] share_pg_info 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] simple_opc_create 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:29.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c55053c00 con 0x558c54c64000 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 1:33f5bf01:::smithi13140796-2:19 snaps e,13,15 -> e,15 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 1:33f5bf01:::smithi13140796-2:19 new snapset 19=[]:{19=[15,e]} on 1:33f5bf01:::smithi13140796-2:head(51'34 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:29.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c55053500 con 0x558c54c19800 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 1:33f5bf01:::smithi13140796-2:19 writing updated snapset on 1:33f5bf01:::smithi13140796-2:head, snapset is 19=[]:{19=[15,e]} 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] new_repop rep_tid 40 (no op) 2019-12-07T19:34:29.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c55052700 con 0x558c54c5bc00 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] new_repop: repgather(0x558c576b6ea0 0'0 rep_tid=40 committed?=0 r=0) 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] simple_opc_submit 0x558c576b6ea0 2019-12-07T19:34:29.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c55052e00 con 0x558c54c5b400 2019-12-07T19:34:29.620+0000 7f8f0297e700 7 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] issue_repop rep_tid 40 o 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:29.620+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c55054a00 con 0x558c54c19c00 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:19 v=52'36 tt=0'0 tid=40 reqid=osd.7.0:40 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim ps=[13~1,19~1]] exit Trimming 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:29.620+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key = 0 2019-12-07T19:34:29.620+0000 7f8f0b990700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:29.620+0000 7f8f06986700 15 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] publish_stats_to_osd 52:133 2019-12-07T19:34:29.620+0000 7f8f0b990700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] SnapTrimmer state: NotTrimming: trimming 13 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 projected size 901120 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] enter NotTrimming 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[13~1] ps=[19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:29.620+0000 7f8f0b990700 15 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:145 2019-12-07T19:34:29.620+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:29.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:29.620+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:29.620+0000 7f8f0b990700 10 osd.7 52 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[13~1] ps=[19~1]] for snaptrim 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=52) prio 5 cost 1048576 e52) 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=52'36 tt=0'0 tid=40 reqid=osd.7.0:40 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:29.620+0000 7f8f0b990700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0b990700 15 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:146 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=52) prio 5 cost 1048576 e52) queued 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:19 v=52'36 tt=0'0 tid=40 reqid=osd.7.0:40 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=52) prio 5 cost 1048576 e52) pg 0x558c57e88000 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=52'36 tt=0'0 tid=40 reqid=osd.7.0:40 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[13~1] ps=[19~1]] snap_trimmer posting 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 13 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:19, current size is 901120 buffers are {} 2019-12-07T19:34:29.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): new_size start 901120 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c58b7ff40) upper_bound after SNA_1_0000000000000013_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.2' 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): to_append: {} 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c58b7ff40) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 resetting hinfo to logical size 901120 2019-12-07T19:34:29.620+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000013_.0_0000000000000001.2) returns 0 SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c552382c0 in raw 0x558c552382c0 len 69 nref 1) ) 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): new_size start 1622016 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): to_append: {} 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[19~1]] SnapTrimmer state: adding snap 13 to purged_snaps 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:29.620+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:19={},1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=52'36 tt=0'0 tid=40 reqid=osd.7.0:40 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[13~1,19~1]] SnapTrimmer state: purged_snaps now [13~1,19~1], snap_trimq now [] 2019-12-07T19:34:29.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c58110300 seq 26 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:19,{} 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:34:29.620+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:29.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:29.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58110300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58110300 onodes shared_blobs 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58110300 allocated 0x[] released 0x[] 2019-12-07T19:34:29.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110300 prepare 2019-12-07T19:34:29.620+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58110300 2019-12-07T19:34:29.620+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110300 io_done 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 52 send_incremental_map 51 -> 52 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[13~1,19~1]] share_pg_info 2019-12-07T19:34:29.620+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c507972f:::inc_osdmap.52:0# 0x0~0 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5829de30 blob([0xb44000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c58b836c0 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c507972f:::inc_osdmap.52:0# 0x0~269 = 617 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 52 add_map_inc_bl 52 617 bytes 2019-12-07T19:34:29.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(52..52 src has 1..52) v4 -- 0x558c581938c0 con 0x558c54c64000 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 52 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 51 -> 52 (shared) 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 52/28 ECSubWrite(tid=40, reqid=osd.7.0:40, at_version=52'36, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c579e3000 con 0x558c54c64000 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 52 send_incremental_map 51 -> 52 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:29.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c57a53c00 con 0x558c54c19800 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(52..52 src has 1..52) v4 -- 0x558c58193b00 con 0x558c54c19800 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 52 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 51 -> 52 (shared) 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 52/28 ECSubWrite(tid=40, reqid=osd.7.0:40, at_version=52'36, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c57e7b800 con 0x558c54c19800 2019-12-07T19:34:29.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c54e04000 con 0x558c54c5bc00 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 52/28 ECSubWrite(tid=40, reqid=osd.7.0:40, at_version=52'36, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c57e7c000 con 0x558c54c5bc00 2019-12-07T19:34:29.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c54e06a00 con 0x558c54c5b400 2019-12-07T19:34:29.620+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 -- 0x558c54e04700 con 0x558c54c19c00 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 52/28 ECSubWrite(tid=40, reqid=osd.7.0:40, at_version=52'36, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c57e7b000 con 0x558c54c5b400 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 52 send_incremental_map 51 -> 52 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(52..52 src has 1..52) v4 -- 0x558c58193d40 con 0x558c54c19c00 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 52 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 51 -> 52 (shared) 2019-12-07T19:34:29.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 52/28 ECSubWrite(tid=40, reqid=osd.7.0:40, at_version=52'36, trim_to=0'0, roll_forward_to=51'34)) v2 -- 0x558c57a43800 con 0x558c54c19c00 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[13~1,19~1]] exit Trimming 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.update_snaps 1:33f5bf01:::smithi13140796-2:19 e,15 was 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:29.620+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.620+0000 7f8f07988700 15 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] publish_stats_to_osd 52:147 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] enter NotTrimming 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:33f5bf01:::smithi13140796-2:19 e,13,15 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.set_snaps 1:33f5bf01:::smithi13140796-2:19 e,15 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:29.620+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:29.620+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 trim 52'35 (51'33) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.620+0000 7f8f0297e700 20 trim 52'36 (51'34) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 51'34 (0'0,51'34] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] append_log log((0'0,51'34], crt=51'34) [52'35 (51'33) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'36 (51'34) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.620+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 160 ==== MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=62, reqid=osd.2.0:62, at_version=52'16, trim_to=0'0, roll_forward_to=51'14)) v2 ==== 3458+0+0 (crc 0 0 0) 0x558c575bf000 con 0x558c54c19800 2019-12-07T19:34:29.620+0000 7f8f24c58700 15 osd.7 52 enqueue_op 0x558c5511fb80 prio 127 cost 0 latency 0.000029 epoch 52 MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=62, reqid=osd.2.0:62, at_version=52'16, trim_to=0'0, roll_forward_to=51'14)) v2 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'35 (0'0,52'35] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 lua=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] add_log_entry 52'35 (51'33) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.620+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=62, reqid=osd.2.0:62, at_version=52'16, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 lua=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] add_log_entry 52'36 (51'34) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=62, reqid=osd.2.0:62, at_version=52'16, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 lua=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] append_log approx pg log length = 36 2019-12-07T19:34:29.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=62, reqid=osd.2.0:62, at_version=52'16, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e52) pg 0x558c55141000 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 lua=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 52 dequeue_op 0x558c5511fb80 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=62, reqid=osd.2.0:62, at_version=52'16, trim_to=0'0, roll_forward_to=51'14)) v2 pg pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.621+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'35, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 265 ==== MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=100, reqid=osd.4.0:100, at_version=52'33, trim_to=0'0, roll_forward_to=45'29)) v2 ==== 3426+0+0 (unknown 0 0 0) 0x558c57dfe800 con 0x558c54c5b400 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=62, reqid=osd.2.0:62, at_version=52'16, trim_to=0'0, roll_forward_to=51'14)) v2 2019-12-07T19:34:29.621+0000 7f8f23c56700 15 osd.7 52 enqueue_op 0x558c57335a20 prio 127 cost 0 latency 0.000027 epoch 52 MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=100, reqid=osd.4.0:100, at_version=52'33, trim_to=0'0, roll_forward_to=45'29)) v2 2019-12-07T19:34:29.621+0000 7f8f06185700 20 snap_mapper.update_snaps 1:bca18876:::smithi13140796-30:1b e,15,1a,1b was 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.621+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=100, reqid=osd.4.0:100, at_version=52'33, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.621+0000 7f8f06185700 20 snap_mapper.get_snaps 1:bca18876:::smithi13140796-30:1b e,13,15,1a,1b 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f7500 seq 49 2019-12-07T19:34:29.621+0000 7f8f06185700 20 snap_mapper.set_snaps 1:bca18876:::smithi13140796-30:1b e,15,1a,1b 2019-12-07T19:34:29.621+0000 7f8f06185700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:29.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys = 0 2019-12-07T19:34:29.621+0000 7f8f06185700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:29.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_0000000000000013_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 51'14 (0'0,51'14] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] append_log log((0'0,51'14], crt=51'14) [52'15 (51'13) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:29.621471+0000 0 snaps [e,15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'16 (51'14) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:29.621471+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000052.00000000000000000035' <- 0000000052.00000000000000000035 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000052.00000000000000000036' <- 0000000052.00000000000000000036 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'15 (0'0,52'15] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=51'14 crt=51'14 lcod 51'14 active mbc={} ps=[19~1]] add_log_entry 52'15 (51'13) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:29.621471+0000 0 snaps [e,15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f7500 cost 8364 (1 ios * 4000 + 4364 bytes) 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f7500 onodes 0x558c58295180,0x558c58295400 shared_blobs 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295288) update 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:19# is 1172 (410 bytes onode + 2 bytes spanning blobs + 760 bytes inline extents) 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'16 (0'0,52'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=51'14 crt=51'14 lcod 51'14 active mbc={} ps=[19~1]] add_log_entry 52'16 (51'14) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:29.621471+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1505 (549 bytes onode + 2 bytes spanning blobs + 954 bytes inline extents) 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f7500 allocated 0x[] released 0x[] 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'16 (0'0,52'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=51'14 crt=51'14 lcod 51'14 active mbc={} ps=[19~1]] append_log approx pg log length = 16 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7500 prepare 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f7500 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7500 io_done 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'16 (0'0,52'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=51'14 crt=51'14 lcod 51'14 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.621+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 159 ==== osd_map(52..52 src has 1..52) v4 ==== 661+0+0 (crc 0 0 0) 0x558c57968d80 con 0x558c54c19800 2019-12-07T19:34:29.621+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.621+0000 7f8f1199c700 20 osd.7 52 OSD::ms_dispatch: osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.621+0000 7f8f1199c700 10 osd.7 52 do_waiters -- start 2019-12-07T19:34:29.621+0000 7f8f1199c700 10 osd.7 52 do_waiters -- finish 2019-12-07T19:34:29.621+0000 7f8f1199c700 20 osd.7 52 _dispatch 0x558c57968d80 osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.621+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'15, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.621+0000 7f8f1199c700 3 osd.7 52 handle_osd_map epochs [52,52], i have 52, src has [1,52] 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 lua=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] op_applied version 52'36 2019-12-07T19:34:29.621+0000 7f8f1199c700 10 osd.7 52 no new maps here, dropping 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] eval_repop repgather(0x558c576b6ea0 52'36 rep_tid=40 committed?=0 r=0) (no op) 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] calc_trim_to_aggressive limit = 51'34 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58524900 seq 48 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] enter Trimming/WaitRepops 2019-12-07T19:34:29.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys 2019-12-07T19:34:29.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] snap_trimmer complete 2019-12-07T19:34:29.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=100, reqid=osd.4.0:100, at_version=52'33, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys = 0 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) 2019-12-07T19:34:29.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=100, reqid=osd.4.0:100, at_version=52'33, trim_to=0'0, roll_forward_to=45'29)) v2) prio 127 cost 0 e52) pg 0x558c575ef000 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.621+0000 7f8f07187700 10 osd.7 52 dequeue_op 0x558c57335a20 prio 127 cost 0 latency 0.000190 MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=100, reqid=osd.4.0:100, at_version=52'33, trim_to=0'0, roll_forward_to=45'29)) v2 pg pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.621+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=100, reqid=osd.4.0:100, at_version=52'33, trim_to=0'0, roll_forward_to=45'29)) v2 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_0000000000000013_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:29.621+0000 7f8f07187700 20 snap_mapper.update_snaps 1:db85c185:::smithi13140796-18:15 e,15 was 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000052.00000000000000000015' <- 0000000052.00000000000000000015 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000052.00000000000000000016' <- 0000000052.00000000000000000016 2019-12-07T19:34:29.621+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:29.621+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58524900 cost 8455 (1 ios * 4000 + 4455 bytes) 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58524900 onodes 0x558c54d96780,0x558c54e13900 shared_blobs 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore.extentmap(0x558c54d96888) update 2#1:bca18876:::smithi13140796-30:1b# 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:1b# is 826 (412 bytes onode + 2 bytes spanning blobs + 412 bytes inline extents) 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore.extentmap(0x558c54e13a08) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 1467 (565 bytes onode + 2 bytes spanning blobs + 900 bytes inline extents) 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58524900 allocated 0x[] released 0x[] 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524900 prepare 2019-12-07T19:34:29.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58524900 2019-12-07T19:34:29.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524900 io_done 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'16 (0'0,52'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=51'14 crt=51'14 lcod 51'14 active mbc={} ps=[19~1]] op_applied version 52'16 2019-12-07T19:34:29.621+0000 7f8f06185700 10 osd.7 52 dequeue_op 0x558c5511fb80 finish 2019-12-07T19:34:29.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 399 ==== MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=239, reqid=osd.1.0:239, at_version=52'43, trim_to=0'0, roll_forward_to=51'41)) v2 ==== 3458+0+0 (crc 0 0 0) 0x558c579e3000 con 0x558c54c64000 2019-12-07T19:34:29.621+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c5740c160 prio 127 cost 0 latency 0.000029 epoch 52 MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=239, reqid=osd.1.0:239, at_version=52'43, trim_to=0'0, roll_forward_to=51'41)) v2 2019-12-07T19:34:29.621+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=239, reqid=osd.1.0:239, at_version=52'43, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=239, reqid=osd.1.0:239, at_version=52'43, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=239, reqid=osd.1.0:239, at_version=52'43, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e52) pg 0x558c57380000 2019-12-07T19:34:29.621+0000 7f8f04982700 10 osd.7 52 dequeue_op 0x558c5740c160 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=239, reqid=osd.1.0:239, at_version=52'43, trim_to=0'0, roll_forward_to=51'41)) v2 pg pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.621+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=239, reqid=osd.1.0:239, at_version=52'43, trim_to=0'0, roll_forward_to=51'41)) v2 2019-12-07T19:34:29.621+0000 7f8f04982700 20 snap_mapper.update_snaps 1:10779340:::smithi13140796-32:1b e,15,1a,1b was 2019-12-07T19:34:29.621+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.621+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush cnt:2 2019-12-07T19:34:29.621+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 398 ==== osd_map(52..52 src has 1..52) v4 ==== 661+0+0 (crc 0 0 0) 0x558c58193d40 con 0x558c54c64000 2019-12-07T19:34:29.621+0000 7f8f1199c700 20 osd.7 52 OSD::ms_dispatch: osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.621+0000 7f8f1199c700 10 osd.7 52 do_waiters -- start 2019-12-07T19:34:29.621+0000 7f8f1199c700 10 osd.7 52 do_waiters -- finish 2019-12-07T19:34:29.621+0000 7f8f1199c700 20 osd.7 52 _dispatch 0x558c58193d40 osd_map(52..52 src has 1..52) v4 2019-12-07T19:34:29.621+0000 7f8f1199c700 3 osd.7 52 handle_osd_map epochs [52,52], i have 52, src has [1,52] 2019-12-07T19:34:29.621+0000 7f8f1199c700 10 osd.7 52 no new maps here, dropping 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2a5bd9 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.001630957s (0.000000128s flush + 0.001630829s kv commit) 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5810e600,0x558c58112900,0x558c58522000> 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e600 kv_submitted 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810e600 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3402 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e600 finishing 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810e600 onodes 0x558c574a0a00,0x558c574a0c80 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810e600 done 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810e600 [] 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8921 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112900 kv_submitted 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58112900 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112900 finishing 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58112900 onodes 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112900 done 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58112900 [] 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522000 kv_submitted 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58522000 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522000 finishing 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58522000 onodes 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522000 done 2019-12-07T19:34:29.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295180 had 1 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58522000 [] 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:29.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:29.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9255 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14140 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96780 had 1 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13900 had 1 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14163 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14163 < min_flush_size 524288 2019-12-07T19:34:29.621+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2a5bd9~3753 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2a5bd9 0x2a5bd9~3753 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x275bd9 2019-12-07T19:34:29.621+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbd9 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x32c and padding block with 0xcd4 2019-12-07T19:34:29.621+0000 7f8f07187700 20 snap_mapper.get_snaps 1:db85c185:::smithi13140796-18:15 e,13,15 2019-12-07T19:34:29.621+0000 7f8f07187700 20 snap_mapper.set_snaps 1:db85c185:::smithi13140796-18:15 e,15 2019-12-07T19:34:29.621+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:29.621+0000 7f8f07187700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:29.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.622+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'31 (0'0,52'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] append_log log((0'0,52'31], crt=45'29) [52'32 (45'24) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:29.621095+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'33 (45'25) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:29.621095+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.622+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'32 (0'0,52'32] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'31 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] add_log_entry 52'32 (45'24) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:29.621095+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.622+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'31 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] add_log_entry 52'33 (45'25) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:29.621095+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.622+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'31 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] append_log approx pg log length = 33 2019-12-07T19:34:29.622+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'31 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.622+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.622+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'32, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58522300 seq 39 2019-12-07T19:34:29.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys = 0 2019-12-07T19:34:29.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:34:29.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:1b e,13,15,1a,1b 2019-12-07T19:34:29.622+0000 7f8f04982700 20 snap_mapper.set_snaps 1:10779340:::smithi13140796-32:1b e,15,1a,1b 2019-12-07T19:34:29.622+0000 7f8f04982700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:29.622+0000 7f8f04982700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 51'41 (0'0,51'41] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] append_log log((0'0,51'41], crt=51'41) [52'42 (51'40) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:29.621617+0000 0 snaps [e,15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'43 (51'41) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:29.621617+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'42 (0'0,52'42] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=51'41 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] add_log_entry 52'42 (51'40) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:29.621617+0000 0 snaps [e,15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'43 (0'0,52'43] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=51'41 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] add_log_entry 52'43 (51'41) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:29.621617+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'43 (0'0,52'43] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=51'41 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] append_log approx pg log length = 43 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'43 (0'0,52'43] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=51'41 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.622+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.622+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'42, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58c64300 seq 38 2019-12-07T19:34:29.622+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_0000000000000013_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000052.00000000000000000042' <- 0000000052.00000000000000000042 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000052.00000000000000000043' <- 0000000052.00000000000000000043 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c64300 cost 8455 (1 ios * 4000 + 4455 bytes) 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c64300 onodes 0x558c5840c500,0x558c5840c780 shared_blobs 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c608) update 5#1:10779340:::smithi13140796-32:1b# 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:1b# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 499 (491 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c64300 allocated 0x[] released 0x[] 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64300 prepare 2019-12-07T19:34:29.622+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c64300 2019-12-07T19:34:29.622+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64300 io_done 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'43 (0'0,52'43] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=51'41 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] op_applied version 52'43 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 52 dequeue_op 0x558c5740c160 finish 2019-12-07T19:34:29.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.622+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_0000000000000013_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.622+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000052.00000000000000000032' <- 0000000052.00000000000000000032 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000052.00000000000000000033' <- 0000000052.00000000000000000033 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58522300 cost 7617 (1 ios * 4000 + 3617 bytes) 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58522300 onodes 0x558c57573680,0x558c57e8ec80 shared_blobs 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore.extentmap(0x558c57573788) update 1#1:db85c185:::smithi13140796-18:15# 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:15# is 664 (412 bytes onode + 2 bytes spanning blobs + 250 bytes inline extents) 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1595 (566 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58522300 allocated 0x[] released 0x[] 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522300 prepare 2019-12-07T19:34:29.622+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58522300 2019-12-07T19:34:29.622+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522300 io_done 2019-12-07T19:34:29.622+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'31 crt=45'29 lcod 45'29 active mbc={} ps=[19~1]] op_applied version 52'33 2019-12-07T19:34:29.622+0000 7f8f07187700 10 osd.7 52 dequeue_op 0x558c57335a20 finish 2019-12-07T19:34:29.622+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 52/41 ECSubWriteReply(tid=99, last_complete=52'31, committed=1, applied=1)) v2 -- 0x558c5841ea00 con 0x558c54c5b400 2019-12-07T19:34:29.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 400 ==== MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=240, reqid=osd.1.0:240, at_version=52'45, trim_to=0'0, roll_forward_to=51'41)) v2 ==== 3417+0+0 (crc 0 0 0) 0x558c57a43800 con 0x558c54c64000 2019-12-07T19:34:29.622+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c5740da20 prio 127 cost 0 latency 0.000031 epoch 52 MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=240, reqid=osd.1.0:240, at_version=52'45, trim_to=0'0, roll_forward_to=51'41)) v2 2019-12-07T19:34:29.622+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=240, reqid=osd.1.0:240, at_version=52'45, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.622+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 161 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57a53c00 con 0x558c54c19800 2019-12-07T19:34:29.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=240, reqid=osd.1.0:240, at_version=52'45, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.622+0000 7f8f24c58700 15 osd.7 52 enqueue_peering_evt 1.5s4 epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:29.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=240, reqid=osd.1.0:240, at_version=52'45, trim_to=0'0, roll_forward_to=51'41)) v2) prio 127 cost 0 e52) pg 0x558c57380000 2019-12-07T19:34:29.622+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.622+0000 7f8f04982700 10 osd.7 52 dequeue_op 0x558c5740da20 prio 127 cost 0 latency 0.000130 MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=240, reqid=osd.1.0:240, at_version=52'45, trim_to=0'0, roll_forward_to=51'41)) v2 pg pg[1.8s5( v 52'43 (0'0,52'43] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'43 (0'0,52'43] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=240, reqid=osd.1.0:240, at_version=52'45, trim_to=0'0, roll_forward_to=51'41)) v2 2019-12-07T19:34:29.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 162 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c576b4400 con 0x558c54c19800 2019-12-07T19:34:29.623+0000 7f8f04982700 20 snap_mapper.update_snaps 1:131421e8:::smithi13140796-7:15 e,15 was 2019-12-07T19:34:29.623+0000 7f8f24c58700 15 osd.7 52 enqueue_op 0x558c5511fe40 prio 196 cost 0 latency 0.000011 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.623+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush cnt:2 2019-12-07T19:34:29.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.623+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c5511fe40 prio 196 cost 0 latency 0.000090 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 2019-12-07T19:34:29.623+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.623+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c5511fe40 finish 2019-12-07T19:34:29.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 401 ==== MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=241, reqid=osd.1.0:241, at_version=52'34, trim_to=0'0, roll_forward_to=45'32)) v2 ==== 3385+0+0 (crc 0 0 0) 0x558c54d87000 con 0x558c54c64000 2019-12-07T19:34:29.623+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000026 epoch 52 MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=241, reqid=osd.1.0:241, at_version=52'34, trim_to=0'0, roll_forward_to=45'32)) v2 2019-12-07T19:34:29.623+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=241, reqid=osd.1.0:241, at_version=52'34, trim_to=0'0, roll_forward_to=45'32)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=241, reqid=osd.1.0:241, at_version=52'34, trim_to=0'0, roll_forward_to=45'32)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=241, reqid=osd.1.0:241, at_version=52'34, trim_to=0'0, roll_forward_to=45'32)) v2) prio 127 cost 0 e52) pg 0x558c5500a000 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 52 dequeue_op 0x558c5740d4a0 prio 127 cost 0 latency 0.000082 MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=241, reqid=osd.1.0:241, at_version=52'34, trim_to=0'0, roll_forward_to=45'32)) v2 pg pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=241, reqid=osd.1.0:241, at_version=52'34, trim_to=0'0, roll_forward_to=45'32)) v2 2019-12-07T19:34:29.623+0000 7f8f08189700 20 snap_mapper.update_snaps 1:9694a0e5:::smithi13140796-3:13 e was 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2a932c 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush cnt:2 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 5 cleaned 0 in 0.001432969s (0.000000096s flush + 0.001432873s kv commit) 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3841 < min_flush_size 524288 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c500 had 1 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8177 < min_flush_size 524288 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573680 had 1 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8200 < min_flush_size 524288 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8200 < min_flush_size 524288 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2a932c~2008 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2a932c 0x2a932c~2008 to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.623+0000 7f8f04982700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:15 e,13,15 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x27932c 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x32c 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x334 and padding block with 0xccc 2019-12-07T19:34:29.623+0000 7f8f04982700 20 snap_mapper.set_snaps 1:131421e8:::smithi13140796-7:15 e,15 2019-12-07T19:34:29.623+0000 7f8f04982700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:29.623+0000 7f8f04982700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582a8900,0x558c582acc00,0x558c58110300,0x558c585f7500,0x558c58524900> 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8900 kv_submitted 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a8900 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8900 finishing 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a8900 onodes 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a8900 done 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a8900 [] 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acc00 kv_submitted 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582acc00 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acc00 finishing 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582acc00 onodes 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582acc00 done 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582acc00 [] 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110300 kv_submitted 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58110300 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110300 finishing 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58110300 onodes 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58110300 done 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58110300 [] 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7500 kv_submitted 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f7500 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7500 finishing 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f7500 onodes 0x558c58295180,0x558c58295400 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'43 (0'0,52'43] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] append_log log((0'0,52'43], crt=51'41) [52'44 (45'38) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:29.622669+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'45 (45'39) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:29.622669+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f7500 done 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f7500 [] 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524900 kv_submitted 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58524900 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524900 finishing 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58524900 onodes 0x558c54d96780,0x558c54e13900 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58524900 done 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58524900 [] 2019-12-07T19:34:29.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'44 (0'0,52'44] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'43 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] add_log_entry 52'44 (45'38) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:29.622669+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'43 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] add_log_entry 52'45 (45'39) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:29.622669+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e52) pg 0x558c57522000 2019-12-07T19:34:29.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'43 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] append_log approx pg log length = 45 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'43 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.623+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.623+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:29.623+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'44, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.623+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.623+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:29.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:29.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c58524c00 seq 19 2019-12-07T19:34:29.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:29.623+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:29.623+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:29.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:29.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58524c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58524c00 onodes shared_blobs 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58c64600 seq 39 2019-12-07T19:34:29.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58524c00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524c00 prepare 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys 2019-12-07T19:34:29.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58524c00 2019-12-07T19:34:29.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524c00 io_done 2019-12-07T19:34:29.623+0000 7f8f08189700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:13 e,13 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys = 0 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) 2019-12-07T19:34:29.623+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 52/41 ECSubWriteReply(tid=62, last_complete=52'16, committed=1, applied=1)) v2 -- 0x558c58512400 con 0x558c54c19800 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 20 snap_mapper.set_snaps 1:9694a0e5:::smithi13140796-3:13 e 2019-12-07T19:34:29.623+0000 7f8f08189700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_0000000000000013_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 45'32 (0'0,45'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] append_log log((0'0,45'32], crt=45'32) [52'33 (45'29) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:29.623678+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'34 (45'30) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:29.623678+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.623+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000052.00000000000000000044' <- 0000000052.00000000000000000044 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000052.00000000000000000045' <- 0000000052.00000000000000000045 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._fastinfo' <- _fastinfo 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'33 (0'0,52'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'32 crt=45'32 lcod 45'32 active mbc={} ps=[19~1]] add_log_entry 52'33 (45'29) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:29.623678+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c64600 cost 7588 (1 ios * 4000 + 3588 bytes) 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c64600 onodes 0x558c57598000,0x558c575f9b80 shared_blobs 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore.extentmap(0x558c57598108) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'32 crt=45'32 lcod 45'32 active mbc={} ps=[19~1]] add_log_entry 52'34 (45'30) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:29.623678+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 1002 (564 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) update 5#1:131421e8:::smithi13140796-7:15# 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'32 crt=45'32 lcod 45'32 active mbc={} ps=[19~1]] append_log approx pg log length = 34 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:15# is 849 (411 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c64600 allocated 0x[] released 0x[] 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'32 crt=45'32 lcod 45'32 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.623+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64600 prepare 2019-12-07T19:34:29.623+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c64600 2019-12-07T19:34:29.623+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64600 io_done 2019-12-07T19:34:29.623+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'33, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'43 crt=51'41 lcod 51'41 active mbc={} ps=[19~1]] op_applied version 52'45 2019-12-07T19:34:29.623+0000 7f8f04982700 10 osd.7 52 dequeue_op 0x558c5740da20 finish 2019-12-07T19:34:29.623+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c58112c00 seq 70 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# 1 keys = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:29.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_0000000000000013_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000052.00000000000000000033' <- 0000000052.00000000000000000033 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000052.00000000000000000034' <- 0000000052.00000000000000000034 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58112c00 cost 8356 (1 ios * 4000 + 4356 bytes) 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58112c00 onodes 0x558c54ce2a00,0x558c54e45680 shared_blobs 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 571 (569 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore.extentmap(0x558c54e45788) update 4#1:9694a0e5:::smithi13140796-3:13# 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:13# is 906 (411 bytes onode + 2 bytes spanning blobs + 493 bytes inline extents) 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58112c00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112c00 prepare 2019-12-07T19:34:29.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58112c00 2019-12-07T19:34:29.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112c00 io_done 2019-12-07T19:34:29.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=45'32 crt=45'32 lcod 45'32 active mbc={} ps=[19~1]] op_applied version 52'34 2019-12-07T19:34:29.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f08189700 10 osd.7 52 dequeue_op 0x558c5740d4a0 finish 2019-12-07T19:34:29.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 99 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d000 con 0x558c54c19c00 2019-12-07T19:34:29.623+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c57efe2c0 prio 196 cost 0 latency 0.000011 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.623+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.623+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c57efe2c0 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 2019-12-07T19:34:29.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.623+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c57efe2c0 finish 2019-12-07T19:34:29.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 238 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841ea00 con 0x558c54c5bc00 2019-12-07T19:34:29.624+0000 7f8f23c56700 15 osd.7 52 enqueue_op 0x558c57335e40 prio 196 cost 0 latency 0.000011 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.624+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.624+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c57335e40 prio 196 cost 0 latency 0.000029 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 2019-12-07T19:34:29.624+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.624+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c57335e40 finish 2019-12-07T19:34:29.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 163 ==== MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=63, reqid=osd.2.0:63, at_version=52'18, trim_to=0'0, roll_forward_to=51'14)) v2 ==== 3362+0+0 (crc 0 0 0) 0x558c57e7b800 con 0x558c54c19800 2019-12-07T19:34:29.624+0000 7f8f24c58700 15 osd.7 52 enqueue_op 0x558c5511e2c0 prio 127 cost 0 latency 0.000033 epoch 52 MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=63, reqid=osd.2.0:63, at_version=52'18, trim_to=0'0, roll_forward_to=51'14)) v2 2019-12-07T19:34:29.624+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=63, reqid=osd.2.0:63, at_version=52'18, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=63, reqid=osd.2.0:63, at_version=52'18, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=63, reqid=osd.2.0:63, at_version=52'18, trim_to=0'0, roll_forward_to=51'14)) v2) prio 127 cost 0 e52) pg 0x558c55141000 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 52 dequeue_op 0x558c5511e2c0 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=63, reqid=osd.2.0:63, at_version=52'18, trim_to=0'0, roll_forward_to=51'14)) v2 pg pg[1.ds2( v 52'16 (0'0,52'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'16 (0'0,52'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=63, reqid=osd.2.0:63, at_version=52'18, trim_to=0'0, roll_forward_to=51'14)) v2 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 snap_mapper.update_snaps 1:bf81b261:::smithi13140796-37:13 e was 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush cnt:2 2019-12-07T19:34:29.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 402 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5763c600 con 0x558c54c64000 2019-12-07T19:34:29.624+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c57eff340 prio 196 cost 0 latency 0.000013 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.624+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c57eff340 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 2019-12-07T19:34:29.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2ab334 2019-12-07T19:34:29.624+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c57eff340 finish 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.001308949s (0.000000087s flush + 0.001308862s kv commit) 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58c64300,0x558c58522300> 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64300 kv_submitted 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c64300 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64300 finishing 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c64300 onodes 0x558c5840c500,0x558c5840c780 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c64300 done 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c64600 kv_queued 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c64300 [] 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522300 kv_submitted 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58522300 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522300 finishing 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58522300 onodes 0x558c57573680,0x558c57e8ec80 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522300 done 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58522300 [] 2019-12-07T19:34:29.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5029 < min_flush_size 524288 2019-12-07T19:34:29.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 52/41 ECSubWriteReply(tid=100, last_complete=52'33, committed=1, applied=1)) v2 -- 0x558c55271800 con 0x558c54c5b400 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57598000 had 1 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9b80 had 1 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9332 < min_flush_size 524288 2019-12-07T19:34:29.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45680 had 1 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9355 < min_flush_size 524288 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9355 < min_flush_size 524288 2019-12-07T19:34:29.624+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2ab334~248b to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2ab334 0x2ab334~248b to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x27b334 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x334 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.624+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 52/41 ECSubWriteReply(tid=239, last_complete=52'43, committed=1, applied=1)) v2 -- 0x558c5795dc00 con 0x558c54c64000 2019-12-07T19:34:29.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7bf and padding block with 0x841 2019-12-07T19:34:29.624+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 snap_mapper.get_snaps 1:bf81b261:::smithi13140796-37:13 e,13 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 snap_mapper.set_snaps 1:bf81b261:::smithi13140796-37:13 e 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'16 (0'0,52'16] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] append_log log((0'0,52'16], crt=51'14) [52'17 (45'11) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:29.623823+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'18 (45'12) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:29.623823+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'17 (0'0,52'17] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'16 crt=51'14 lcod 52'16 active mbc={} ps=[19~1]] add_log_entry 52'17 (45'11) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:29.623823+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'16 crt=51'14 lcod 52'16 active mbc={} ps=[19~1]] add_log_entry 52'18 (45'12) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:29.623823+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'16 crt=51'14 lcod 52'16 active mbc={} ps=[19~1]] append_log approx pg log length = 18 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'16 crt=51'14 lcod 52'16 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.624+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'17, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57a57800 seq 49 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# 1 keys = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# 2 keys = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_0000000000000013_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000052.00000000000000000017' <- 0000000052.00000000000000000017 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000052.00000000000000000018' <- 0000000052.00000000000000000018 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a57800 cost 7527 (1 ios * 4000 + 3527 bytes) 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a57800 onodes 0x558c54d96c80,0x558c57976000 shared_blobs 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96d88) update 2#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head# is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore.extentmap(0x558c57976108) update 2#1:bf81b261:::smithi13140796-37:13# 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:13# is 931 (412 bytes onode + 2 bytes spanning blobs + 517 bytes inline extents) 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a57800 allocated 0x[] released 0x[] 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57800 prepare 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a57800 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57800 io_done 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'16 crt=51'14 lcod 52'16 active mbc={} ps=[19~1]] op_applied version 52'18 2019-12-07T19:34:29.624+0000 7f8f0217d700 10 osd.7 52 dequeue_op 0x558c5511e2c0 finish 2019-12-07T19:34:29.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 266 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c55271800 con 0x558c54c5b400 2019-12-07T19:34:29.625+0000 7f8f23c56700 15 osd.7 52 enqueue_op 0x558c573354a0 prio 196 cost 0 latency 0.000013 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.625+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c573354a0 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=40, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:19 v=52'36 tt=0'0 tid=40 reqid=osd.7.0:40 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=51'34 crt=51'34 lcod 48'32 mlcod 48'32 active+clean+snaptrim trimq=[13~1] ps=[19~1]] repop_all_committed: repop tid 40 all committed 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] eval_repop repgather(0x558c576b6ea0 52'36 rep_tid=40 committed?=1 r=0) (no op) 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] commit: repgather(0x558c576b6ea0 52'36 rep_tid=40 committed?=1 r=0) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.625+0000 7f8f06986700 15 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:166 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] removing repgather(0x558c576b6ea0 52'36 rep_tid=40 committed?=1 r=0) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] q front is repgather(0x558c576b6ea0 52'36 rep_tid=40 committed?=1 r=0) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] exit Trimming/WaitRepops 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 52 queueing pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] for snaptrim 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=52) prio 5 cost 1048576 e52) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] prepare_stats_for_publish reporting purged_snaps [19~1] 2019-12-07T19:34:29.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f06986700 15 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] publish_stats_to_osd 52:167 2019-12-07T19:34:29.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=52) prio 5 cost 1048576 e52) queued 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] remove_repop repgather(0x558c576b6ea0 52'36 rep_tid=40 committed?=1 r=0) 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=52'36 tt=0'0 tid=40 reqid=osd.7.0:40 roll_forward_to=51'34 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=41 reqid=osd.7.0:40 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=41 reqid=osd.7.0:40 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:29.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 52/28 ECSubWrite(tid=41, reqid=osd.7.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c57e7c800 con 0x558c54c64000 2019-12-07T19:34:29.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 52/28 ECSubWrite(tid=41, reqid=osd.7.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c57e7d000 con 0x558c54c19800 2019-12-07T19:34:29.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 52/28 ECSubWrite(tid=41, reqid=osd.7.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c57e7d800 con 0x558c54c5bc00 2019-12-07T19:34:29.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 52/28 ECSubWrite(tid=41, reqid=osd.7.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5852a000 con 0x558c54c5b400 2019-12-07T19:34:29.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 52/28 ECSubWrite(tid=41, reqid=osd.7.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5852a800 con 0x558c54c19c00 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=51'34 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] append_log log((0'0,52'36], crt=51'34) [] 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] rollforward: entry=52'35 (51'33) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] rollforward: entry=52'36 (51'34) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:29.621658+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] append_log approx pg log length = 36 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.625+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.625+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:29.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c582acf00 seq 50 2019-12-07T19:34:29.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:29.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:29.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:29.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582acf00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:29.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582acf00 onodes shared_blobs 2019-12-07T19:34:29.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582acf00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acf00 prepare 2019-12-07T19:34:29.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582acf00 2019-12-07T19:34:29.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acf00 io_done 2019-12-07T19:34:29.625+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c573354a0 finish 2019-12-07T19:34:29.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=52) prio 5 cost 1048576 e52) pg 0x558c57676000 2019-12-07T19:34:29.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] snap_trimmer posting 2019-12-07T19:34:29.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 13 2019-12-07T19:34:29.625+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.625+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:29.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 239 ==== MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=139, reqid=osd.3.0:138, at_version=0'0, trim_to=0'0, roll_forward_to=52'61)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5852a000 con 0x558c54c5bc00 2019-12-07T19:34:29.625+0000 7f8f23c56700 15 osd.7 52 enqueue_op 0x558c578d3b80 prio 127 cost 0 latency 0.000022 epoch 52 MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=139, reqid=osd.3.0:138, at_version=0'0, trim_to=0'0, roll_forward_to=52'61)) v2 2019-12-07T19:34:29.625+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=139, reqid=osd.3.0:138, at_version=0'0, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=139, reqid=osd.3.0:138, at_version=0'0, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=139, reqid=osd.3.0:138, at_version=0'0, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e52) pg 0x558c54d5c000 2019-12-07T19:34:29.625+0000 7f8f05984700 10 osd.7 52 dequeue_op 0x558c578d3b80 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=139, reqid=osd.3.0:138, at_version=0'0, trim_to=0'0, roll_forward_to=52'61)) v2 pg pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.625+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 52/28 ECSubWrite(tid=139, reqid=osd.3.0:138, at_version=0'0, trim_to=0'0, roll_forward_to=52'61)) v2 2019-12-07T19:34:29.625+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=47'57 active mbc={} ps=[19~1]] append_log log((0'0,52'61], crt=47'57) [] 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] rollforward: entry=52'58 (45'47) modify 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:29.618062+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] rollforward: entry=52'59 (45'48) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:29.618062+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] rollforward: entry=52'60 (47'56) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:29.618968+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] rollforward: entry=52'61 (47'57) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:29.618968+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.625+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] append_log approx pg log length = 61 2019-12-07T19:34:29.625+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.625+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.625+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c575c4300 seq 96 2019-12-07T19:34:29.625+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:29.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c4300 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c4300 onodes shared_blobs 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c4300 allocated 0x[] released 0x[] 2019-12-07T19:34:29.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4300 prepare 2019-12-07T19:34:29.625+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c4300 2019-12-07T19:34:29.625+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4300 io_done 2019-12-07T19:34:29.625+0000 7f8f05984700 10 osd.7 52 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:29.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 267 ==== MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=101, reqid=osd.4.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=52'33)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c57e7d800 con 0x558c54c5b400 2019-12-07T19:34:29.625+0000 7f8f23c56700 15 osd.7 52 enqueue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000022 epoch 52 MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=101, reqid=osd.4.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=52'33)) v2 2019-12-07T19:34:29.625+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=101, reqid=osd.4.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=101, reqid=osd.4.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=101, reqid=osd.4.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e52) pg 0x558c575ef000 2019-12-07T19:34:29.625+0000 7f8f0317f700 10 osd.7 52 dequeue_op 0x558c57efeb00 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=101, reqid=osd.4.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=52'33)) v2 pg pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.bs1 52/41 ECSubWrite(tid=101, reqid=osd.4.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=52'33)) v2 2019-12-07T19:34:29.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=45'29 active mbc={} ps=[19~1]] append_log log((0'0,52'33], crt=45'29) [] 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] rollforward: entry=52'30 (45'22) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:29.619393+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] rollforward: entry=52'31 (45'23) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:29.619393+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 240 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58114400 con 0x558c54c5bc00 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] rollforward: entry=52'32 (45'24) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:29.621095+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.626+0000 7f8f23c56700 15 osd.7 52 enqueue_op 0x558c57efec60 prio 196 cost 0 latency 0.000047 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] rollforward: entry=52'33 (45'25) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:29.621095+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.626+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] append_log approx pg log length = 33 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.626+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5810e900 seq 40 2019-12-07T19:34:29.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810e900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810e900 onodes shared_blobs 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810e900 allocated 0x[] released 0x[] 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e900 prepare 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810e900 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e900 io_done 2019-12-07T19:34:29.626+0000 7f8f0317f700 10 osd.7 52 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:29.626+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.626+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 241 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e04000 con 0x558c54c5bc00 2019-12-07T19:34:29.626+0000 7f8f23c56700 15 osd.7 52 enqueue_peering_evt 1.6s2 epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.6s0( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:29.626+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.6s0( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.6s0( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.6s0( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e52) pg 0x558c54d5c000 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 3(0) info: 1.6s0( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582ae900 seq 97 2019-12-07T19:34:29.626+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:29.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 100 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795dc00 con 0x558c54c19c00 2019-12-07T19:34:29.626+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 268 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5795da00 con 0x558c54c5b400 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ae900 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ae900 onodes shared_blobs 2019-12-07T19:34:29.626+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c57efe6e0 prio 196 cost 0 latency 0.000015 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.626+0000 7f8f23c56700 15 osd.7 52 enqueue_op 0x558c57efe580 prio 196 cost 0 latency 0.000013 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ae900 allocated 0x[] released 0x[] 2019-12-07T19:34:29.626+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.626+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae900 prepare 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ae900 2019-12-07T19:34:29.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae900 io_done 2019-12-07T19:34:29.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 403 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58114200 con 0x558c54c64000 2019-12-07T19:34:29.626+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c54e78840 prio 196 cost 0 latency 0.000013 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.626+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.626+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 269 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c54e04000 con 0x558c54c5b400 2019-12-07T19:34:29.626+0000 7f8f23c56700 15 osd.7 52 enqueue_peering_evt 1.bs1 epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.bs0( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:29.626+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.bs0( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.bs0( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.bs0( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e52) pg 0x558c575ef000 2019-12-07T19:34:29.626+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 4(0) info: 1.bs0( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:29.626+0000 7f8f07187700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.626+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:29.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:29.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58522600 seq 41 2019-12-07T19:34:29.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:29.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:29.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:29.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:29.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58522600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:29.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58522600 onodes shared_blobs 2019-12-07T19:34:29.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58522600 allocated 0x[] released 0x[] 2019-12-07T19:34:29.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522600 prepare 2019-12-07T19:34:29.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58522600 2019-12-07T19:34:29.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522600 io_done 2019-12-07T19:34:29.626+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 164 ==== MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512200 con 0x558c54c19800 2019-12-07T19:34:29.626+0000 7f8f24c58700 15 osd.7 52 enqueue_op 0x558c5740d080 prio 196 cost 0 latency 0.000013 epoch 52 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.626+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2ad7bf 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.002707576s (0.000000111s flush + 0.002707465s kv commit) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58524c00,0x558c58c64600,0x558c58112c00> 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524c00 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58524c00 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524c00 finishing 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3411 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58524c00 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58524c00 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58524c00 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64600 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c64600 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64600 finishing 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c64600 onodes 0x558c57598000,0x558c575f9b80 2019-12-07T19:34:29.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c64600 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c64600 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112c00 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58112c00 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112c00 finishing 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58112c00 onodes 0x558c54ce2a00,0x558c54e45680 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4525 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112c00 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58112c00 [] 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5663 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 52/41 ECSubWriteReply(tid=240, last_complete=52'45, committed=1, applied=1)) v2 -- 0x558c55045400 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 2 2019-12-07T19:34:29.628+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 52/41 ECSubWriteReply(tid=241, last_complete=52'34, committed=1, applied=1)) v2 -- 0x558c575b8c00 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7ee80) upper_bound after SNA_1_0000000000000013_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.0_0000000000000001.C' 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6777 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 2 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7887 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7ee80) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8997 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9020 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9020 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000013_.0_0000000000000001.C) returns 0 SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2ad7bf~233c to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2ad7bf 0x2ad7bf~233c to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x27d7bf 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7bf 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xafb and padding block with 0x505 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[13~1] ps=[19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim ps=[19~1]] SnapTrimmer state: adding snap 13 to purged_snaps 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim ps=[13~1,19~1]] SnapTrimmer state: purged_snaps now [13~1,19~1], snap_trimq now [] 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f7800 seq 51 2019-12-07T19:34:29.628+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f7800 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f7800 onodes shared_blobs 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f7800 allocated 0x[] released 0x[] 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7800 prepare 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f7800 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7800 io_done 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim ps=[13~1,19~1]] share_pg_info 2019-12-07T19:34:29.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e52/52) v1 -- 0x558c574e5500 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2afafb 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e52/52) v1 -- 0x558c58124300 con 0x558c54c19800 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 6 cleaned 0 in 0.000166845s (0.000000086s flush + 0.000166759s kv commit) 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a57800,0x558c582acf00,0x558c575c4300,0x558c5810e900,0x558c582ae900,0x558c58522600> 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57800 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a57800 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57800 finishing 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a57800 onodes 0x558c54d96c80,0x558c57976000 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a57800 done 2019-12-07T19:34:29.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e52/52) v1 -- 0x558c58123500 con 0x558c54c5bc00 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a57800 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acf00 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582acf00 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582acf00 finishing 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582acf00 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582acf00 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f7800 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582acf00 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4300 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c4300 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 52/41 ECSubWriteReply(tid=63, last_complete=52'18, committed=1, applied=1)) v2 -- 0x558c58512600 con 0x558c54c19800 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4300 finishing 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c4300 onodes 2019-12-07T19:34:29.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e52/52) v1 -- 0x558c57ed3c00 con 0x558c54c5b400 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2afafb~46d to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c4300 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ae900 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c4300 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e900 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810e900 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2afafb 0x2afafb~46d to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810e900 finishing 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810e900 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810e900 done 2019-12-07T19:34:29.628+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e52/52) v1 -- 0x558c54e05c00 con 0x558c54c19c00 2019-12-07T19:34:29.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522600 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810e900 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae900 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ae900 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ae900 finishing 2019-12-07T19:34:29.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ae900 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ae900 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ae900 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522600 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58522600 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522600 finishing 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58522600 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522600 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58522600 [] 2019-12-07T19:34:29.628+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 52/28 ECSubWriteReply(tid=139, last_complete=52'61, committed=1, applied=1)) v2 -- 0x558c580b9e00 con 0x558c54c5bc00 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 52/41 ECSubWriteReply(tid=101, last_complete=52'33, committed=1, applied=1)) v2 -- 0x558c55271e00 con 0x558c54c5b400 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim ps=[13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:29.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~280000 x_off 0x27fafb 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xafb 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim ps=[13~1,19~1]] exit Trimming 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf68 and padding block with 0x98 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:29.628+0000 7f8f0297e700 15 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] publish_stats_to_osd 52:168 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] enter NotTrimming 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c57efec60 prio 196 cost 0 latency 0.001664 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c57efec60 finish 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c57efe6e0 prio 196 cost 0 latency 0.001493 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c57efe6e0 finish 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c54e78840 prio 196 cost 0 latency 0.001493 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c54e78840 finish 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) queued 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c5740d080 prio 196 cost 0 latency 0.001258 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.628+0000 7f8f06986700 10 osd.7 52 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2) prio 196 cost 0 e52) pg 0x558c57676000 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2aff68 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c57efe580 prio 196 cost 0 latency 0.001607 MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 pg pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 52/28 ECSubWriteReply(tid=41, last_complete=52'36, committed=1, applied=1)) v2 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000321791s (0.000000068s flush + 0.000321723s kv commit) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=41 reqid=osd.7.0:40 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:29.628+0000 7f8f0297e700 10 osd.7 52 dequeue_op 0x558c57efe580 finish 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585f7800> 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7800 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f7800 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7800 finishing 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f7800 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f7800 done 2019-12-07T19:34:29.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:29.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f7800 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 404 ==== MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=242, reqid=osd.1.0:240, at_version=0'0, trim_to=0'0, roll_forward_to=52'45)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c5852a800 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000020 epoch 52 MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=242, reqid=osd.1.0:240, at_version=0'0, trim_to=0'0, roll_forward_to=52'45)) v2 2019-12-07T19:34:29.628+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=242, reqid=osd.1.0:240, at_version=0'0, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=242, reqid=osd.1.0:240, at_version=0'0, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=242, reqid=osd.1.0:240, at_version=0'0, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e52) pg 0x558c57380000 2019-12-07T19:34:29.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 405 ==== MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=243, reqid=osd.1.0:241, at_version=0'0, trim_to=0'0, roll_forward_to=52'34)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57e7c800 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000013 epoch 52 MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=243, reqid=osd.1.0:241, at_version=0'0, trim_to=0'0, roll_forward_to=52'34)) v2 2019-12-07T19:34:29.628+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=243, reqid=osd.1.0:241, at_version=0'0, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 52 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000042 MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=242, reqid=osd.1.0:240, at_version=0'0, trim_to=0'0, roll_forward_to=52'45)) v2 pg pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=243, reqid=osd.1.0:241, at_version=0'0, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 52/41 ECSubWrite(tid=242, reqid=osd.1.0:240, at_version=0'0, trim_to=0'0, roll_forward_to=52'45)) v2 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=243, reqid=osd.1.0:241, at_version=0'0, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e52) pg 0x558c5500a000 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=51'41 active mbc={} ps=[19~1]] append_log log((0'0,52'45], crt=51'41) [] 2019-12-07T19:34:29.628+0000 7f8f04181700 10 osd.7 52 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000046 MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=243, reqid=osd.1.0:241, at_version=0'0, trim_to=0'0, roll_forward_to=52'34)) v2 pg pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] rollforward: entry=52'42 (51'40) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:29.621617+0000 0 snaps [e,15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.628+0000 7f8f04181700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.9s4 52/41 ECSubWrite(tid=243, reqid=osd.1.0:241, at_version=0'0, trim_to=0'0, roll_forward_to=52'34)) v2 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] rollforward: entry=52'43 (51'41) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:29.621617+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.628+0000 7f8f04181700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=45'32 active mbc={} ps=[19~1]] append_log log((0'0,52'34], crt=45'32) [] 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] rollforward: entry=52'44 (45'38) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:29.622669+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] rollforward: entry=52'45 (45'39) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:29.622669+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] append_log approx pg log length = 45 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[19~1]] rollforward: entry=52'33 (45'29) modify 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:29.623678+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.628+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.628+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[19~1]] rollforward: entry=52'34 (45'30) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:29.623678+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.628+0000 7f8f04181700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[19~1]] append_log approx pg log length = 34 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58c64900 seq 40 2019-12-07T19:34:29.628+0000 7f8f04181700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.628+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.628+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.628+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c64900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c64900 onodes shared_blobs 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c64900 allocated 0x[] released 0x[] 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64900 prepare 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c64900 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64900 io_done 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 52 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:29.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5511cc00 seq 71 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 152 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:29.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2aff68~47c to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2aff68 0x2aff68~47c to file(ino 16 size 0x2b0000 mtime 2019-12-07T19:34:29.619737+0000 allocated 2b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~280000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _allocate len 0x3e4 from 0 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _allocate len 0x3e4 from 1 2019-12-07T19:34:29.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd3e0000 2019-12-07T19:34:29.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511cc00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd3e0000~10000/10000,10000,acd3e0000 2019-12-07T19:34:29.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511cc00 onodes shared_blobs 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x2c0000 to include allocated 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 60 (was clean) 2019-12-07T19:34:29.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511cc00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x27ff68 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf68 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3e4 and padding block with 0xc1c 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x2c0000 2019-12-07T19:34:29.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cc00 prepare 2019-12-07T19:34:29.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511cc00 2019-12-07T19:34:29.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cc00 io_done 2019-12-07T19:34:29.628+0000 7f8f04181700 10 osd.7 52 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b03e4 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 406 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05c00 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f24457700 15 osd.7 52 enqueue_peering_evt 1.9s4 epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.9s0( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:29.628+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.9s0( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.9s0( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.9s0( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e52) pg 0x558c5500a000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (60) on file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]), flushing log 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.9s0( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 60 len 0x4a crc 0xb5a1802d) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x3b000~1000 to file(ino 1 size 0x3b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x3b000 0x3b000~1000 to file(ino 1 size 0x3b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:29.628+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x3c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3b000 2019-12-07T19:34:29.628+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:29.628+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:29.628+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c58112f00 seq 72 2019-12-07T19:34:29.628+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:29.628+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:29.628+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:29.628+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:29.628+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58112f00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:29.628+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58112f00 onodes shared_blobs 2019-12-07T19:34:29.628+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58112f00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x3c000 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:29.628+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112f00 prepare 2019-12-07T19:34:29.628+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58112f00 2019-12-07T19:34:29.628+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112f00 io_done 2019-12-07T19:34:29.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000030 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 407 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05c00 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f24457700 15 osd.7 52 enqueue_peering_evt 1.8s5 epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.8s0( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:29.628+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.8s0( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 408 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05c00 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f24457700 15 osd.7 52 enqueue_peering_evt 1.0s3 epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.8s0( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.8s0( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e52) pg 0x558c57380000 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.8s0( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 60 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5511d200 seq 41 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000724936s (0.000000112s flush + 0.000724824s kv commit) 2019-12-07T19:34:29.628+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58c64900> 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64900 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c64900 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64900 finishing 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c64900 onodes 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c64900 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511d200 prepare 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c64900 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e52) pg 0x558c5500d000 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2224 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2247 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2247 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.0s0( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b03e4~8c7 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b03e4 0x2b03e4~8c7 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f04982700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x2803e4 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3e4 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcab and padding block with 0x355 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511d200 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511d200 onodes shared_blobs 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511d200 allocated 0x[] released 0x[] 2019-12-07T19:34:29.628+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d200 prepare 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511d200 2019-12-07T19:34:29.628+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d200 io_done 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c58c64c00 seq 43 2019-12-07T19:34:29.628+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:29.628+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 52/41 ECSubWriteReply(tid=242, last_complete=52'45, committed=1, applied=1)) v2 -- 0x558c55045200 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c64c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c64c00 onodes shared_blobs 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c64c00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b0cab 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64c00 prepare 2019-12-07T19:34:29.628+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c64c00 2019-12-07T19:34:29.628+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64c00 io_done 2019-12-07T19:34:29.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000093524s (0.000000093s flush + 0.000093431s kv commit) 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5511cc00,0x558c58112f00> 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cc00 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511cc00 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cc00 finishing 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511cc00 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511cc00 done 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112f00 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511cc00 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112f00 kv_submitted 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58112f00 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2244 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58112f00 finishing 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58112f00 onodes 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58112f00 done 2019-12-07T19:34:29.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:29.628+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 52/41 ECSubWriteReply(tid=243, last_complete=52'34, committed=1, applied=1)) v2 -- 0x558c5841fa00 con 0x558c54c64000 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58112f00 [] 2019-12-07T19:34:29.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2267 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2267 < min_flush_size 524288 2019-12-07T19:34:29.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b0cab~8db to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b0cab 0x2b0cab~8db to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x280cab 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcab 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x586 and padding block with 0xa7a 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b1586 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000170594s (0.000000095s flush + 0.000170499s kv commit) 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5511d200,0x558c58c64c00> 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d200 kv_submitted 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511d200 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d200 finishing 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511d200 onodes 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511d200 done 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511d200 [] 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64c00 kv_submitted 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c64c00 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64c00 finishing 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c64c00 onodes 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c64c00 done 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c64c00 [] 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 409 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05c00 con 0x558c54c64000 2019-12-07T19:34:29.629+0000 7f8f24457700 15 osd.7 52 enqueue_peering_evt 1.as5 epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:29.629+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.629+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e52) pg 0x558c57e8b000 2019-12-07T19:34:29.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:29.629+0000 7f8f03980700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.629+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:29.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:29.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c582b7800 seq 17 2019-12-07T19:34:29.629+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:29.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:29.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:29.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:29.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:29.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b7800 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b7800 onodes shared_blobs 2019-12-07T19:34:29.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b7800 allocated 0x[] released 0x[] 2019-12-07T19:34:29.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7800 prepare 2019-12-07T19:34:29.629+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b7800 2019-12-07T19:34:29.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7800 io_done 2019-12-07T19:34:29.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b1586~485 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b1586 0x2b1586~485 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x281586 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x586 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa0b and padding block with 0x5f5 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b1a0b 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112259s (0.000000085s flush + 0.000112174s kv commit) 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b7800> 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7800 kv_submitted 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b7800 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b7800 finishing 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b7800 onodes 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b7800 done 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b7800 [] 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.629+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 165 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c58124300 con 0x558c54c19800 2019-12-07T19:34:29.629+0000 7f8f24c58700 15 osd.7 52 enqueue_peering_evt 1.ds2 epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.ds0( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:29.629+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.ds0( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.ds0( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.ds0( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e52) pg 0x558c55141000 2019-12-07T19:34:29.629+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 166 ==== MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=64, reqid=osd.2.0:63, at_version=0'0, trim_to=0'0, roll_forward_to=52'18)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57e7d000 con 0x558c54c19800 2019-12-07T19:34:29.629+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 2(0) info: 1.ds0( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:29.629+0000 7f8f24c58700 15 osd.7 52 enqueue_op 0x558c5740de40 prio 127 cost 0 latency 0.000021 epoch 52 MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=64, reqid=osd.2.0:63, at_version=0'0, trim_to=0'0, roll_forward_to=52'18)) v2 2019-12-07T19:34:29.629+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=64, reqid=osd.2.0:63, at_version=0'0, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.629+0000 7f8f06185700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.629+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=64, reqid=osd.2.0:63, at_version=0'0, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.629+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:29.629+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58524f00 seq 50 2019-12-07T19:34:29.629+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:29.629+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:29.629+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:29.629+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:29.629+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58524f00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:29.629+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58524f00 onodes shared_blobs 2019-12-07T19:34:29.629+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58524f00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.629+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524f00 prepare 2019-12-07T19:34:29.629+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58524f00 2019-12-07T19:34:29.629+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524f00 io_done 2019-12-07T19:34:29.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b1a0b~46d to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b1a0b 0x2b1a0b~46d to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x281a0b 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa0b 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe78 and padding block with 0x188 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=64, reqid=osd.2.0:63, at_version=0'0, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e52) pg 0x558c55141000 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 osd.7 52 dequeue_op 0x558c5740de40 prio 127 cost 0 latency 0.000209 MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=64, reqid=osd.2.0:63, at_version=0'0, trim_to=0'0, roll_forward_to=52'18)) v2 pg pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 52/41 ECSubWrite(tid=64, reqid=osd.2.0:63, at_version=0'0, trim_to=0'0, roll_forward_to=52'18)) v2 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=51'14 active mbc={} ps=[13~1,19~1]] append_log log((0'0,52'18], crt=51'14) [] 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] rollforward: entry=52'15 (51'13) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:29.621471+0000 0 snaps [e,15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] rollforward: entry=52'16 (51'14) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:29.621471+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b1e78 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000217076s (0.000000055s flush + 0.000217021s kv commit) 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] rollforward: entry=52'17 (45'11) modify 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:29.623823+0000 0 snaps [e] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58524f00> 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524f00 kv_submitted 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58524f00 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58524f00 finishing 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58524f00 onodes 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58524f00 done 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58524f00 [] 2019-12-07T19:34:29.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] rollforward: entry=52'18 (45'12) modify 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:29.623823+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 18 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.629+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c57a57b00 seq 51 2019-12-07T19:34:29.629+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57a57b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57a57b00 onodes shared_blobs 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57a57b00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57b00 prepare 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57a57b00 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57b00 io_done 2019-12-07T19:34:29.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.629+0000 7f8f0217d700 10 osd.7 52 dequeue_op 0x558c5740de40 finish 2019-12-07T19:34:29.629+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:29.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b1e78~471 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b1e78 0x2b1e78~471 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x281e78 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe78 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2e9 and padding block with 0xd17 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b22e9 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000129432s (0.000000073s flush + 0.000129359s kv commit) 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57a57b00> 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57b00 kv_submitted 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57a57b00 2019-12-07T19:34:29.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57a57b00 finishing 2019-12-07T19:34:29.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57a57b00 onodes 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57a57b00 done 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:29.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57a57b00 [] 2019-12-07T19:34:29.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.630+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 52/41 ECSubWriteReply(tid=64, last_complete=52'18, committed=1, applied=1)) v2 -- 0x558c58512800 con 0x558c54c19800 2019-12-07T19:34:29.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:29.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 410 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05c00 con 0x558c54c64000 2019-12-07T19:34:29.630+0000 7f8f24457700 15 osd.7 52 enqueue_peering_evt 1.es4 epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:29.630+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e52) pg 0x558c54d98000 2019-12-07T19:34:29.630+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:29.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 411 ==== MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=244, reqid=osd.1.0:244, at_version=52'29, trim_to=0'0, roll_forward_to=51'27)) v2 ==== 3554+0+0 (crc 0 0 0) 0x558c57e7a800 con 0x558c54c64000 2019-12-07T19:34:29.630+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.630+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000023 epoch 52 MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=244, reqid=osd.1.0:244, at_version=52'29, trim_to=0'0, roll_forward_to=51'27)) v2 2019-12-07T19:34:29.630+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=244, reqid=osd.1.0:244, at_version=52'29, trim_to=0'0, roll_forward_to=51'27)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.630+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:29.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:29.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c575c4600 seq 18 2019-12-07T19:34:29.630+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:29.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:29.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:29.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:29.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c4600 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:29.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c4600 onodes shared_blobs 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=244, reqid=osd.1.0:244, at_version=52'29, trim_to=0'0, roll_forward_to=51'27)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c4600 allocated 0x[] released 0x[] 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=244, reqid=osd.1.0:244, at_version=52'29, trim_to=0'0, roll_forward_to=51'27)) v2) prio 127 cost 0 e52) pg 0x558c54d5f000 2019-12-07T19:34:29.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4600 prepare 2019-12-07T19:34:29.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c4600 2019-12-07T19:34:29.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4600 io_done 2019-12-07T19:34:29.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 52 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000069 MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=244, reqid=osd.1.0:244, at_version=52'29, trim_to=0'0, roll_forward_to=51'27)) v2 pg pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=244, reqid=osd.1.0:244, at_version=52'29, trim_to=0'0, roll_forward_to=51'27)) v2 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 snap_mapper.update_snaps 1:ecc72bdb:::smithi13140796-10:15 e,15 was 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:15 e,13,15 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:15 e,15 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 snap_mapper.update_snaps rm SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 51'27 (0'0,51'27] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] append_log log((0'0,51'27], crt=51'27) [52'28 (45'20) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:29.630432+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,52'29 (51'27) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:29.630432+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'28 (0'0,52'28] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=51'27 crt=51'27 lcod 51'27 active mbc={} ps=[19~1]] add_log_entry 52'28 (45'20) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:29.630432+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=51'27 crt=51'27 lcod 51'27 active mbc={} ps=[19~1]] add_log_entry 52'29 (51'27) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:29.630432+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=51'27 crt=51'27 lcod 51'27 active mbc={} ps=[19~1]] append_log approx pg log length = 29 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=51'27 crt=51'27 lcod 51'27 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.630+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 52'28, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b5800 seq 72 2019-12-07T19:34:29.630+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330612 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_0000000000000013_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000052.00000000000000000028' <- 0000000052.00000000000000000028 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000052.00000000000000000029' <- 0000000052.00000000000000000029 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b5800 cost 8535 (1 ios * 4000 + 4535 bytes) 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b5800 onodes 0x558c575a2280,0x558c575a2a00 shared_blobs 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2388) update 4#1:ecc72bdb:::smithi13140796-10:15# 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:15# is 556 (412 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 1129 (629 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b5800 allocated 0x[] released 0x[] 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5800 prepare 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b5800 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5800 io_done 2019-12-07T19:34:29.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=51'27 crt=51'27 lcod 51'27 active mbc={} ps=[19~1]] op_applied version 52'29 2019-12-07T19:34:29.630+0000 7f8f0117b700 10 osd.7 52 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:34:29.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5750 < min_flush_size 524288 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2280 had 1 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5773 < min_flush_size 524288 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5773 < min_flush_size 524288 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b22e9~168d to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b22e9 0x2b22e9~168d to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x2822e9 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2e9 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x976 and padding block with 0x68a 2019-12-07T19:34:29.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b3976 2019-12-07T19:34:29.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.001205922s (0.000000066s flush + 0.001205856s kv commit) 2019-12-07T19:34:29.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c575c4600,0x558c586b5800> 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4600 kv_submitted 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c4600 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4600 finishing 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c4600 onodes 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c4600 done 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c4600 [] 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.631+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5800 kv_submitted 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b5800 2019-12-07T19:34:29.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5800 finishing 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b5800 onodes 0x558c575a2280,0x558c575a2a00 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b5800 done 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b5800 [] 2019-12-07T19:34:29.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.631+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 52/41 ECSubWriteReply(tid=244, last_complete=52'29, committed=1, applied=1)) v2 -- 0x558c58513200 con 0x558c54c64000 2019-12-07T19:34:29.631+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 412 ==== MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=245, reqid=osd.1.0:244, at_version=0'0, trim_to=0'0, roll_forward_to=52'29)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e0800 con 0x558c54c64000 2019-12-07T19:34:29.632+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c5511f760 prio 127 cost 0 latency 0.000026 epoch 52 MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=245, reqid=osd.1.0:244, at_version=0'0, trim_to=0'0, roll_forward_to=52'29)) v2 2019-12-07T19:34:29.632+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=245, reqid=osd.1.0:244, at_version=0'0, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e52) 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=245, reqid=osd.1.0:244, at_version=0'0, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e52) queued 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=245, reqid=osd.1.0:244, at_version=0'0, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e52) pg 0x558c54d5f000 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 osd.7 52 dequeue_op 0x558c5511f760 prio 127 cost 0 latency 0.000053 MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=245, reqid=osd.1.0:244, at_version=0'0, trim_to=0'0, roll_forward_to=52'29)) v2 pg pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] _handle_message: MOSDECSubOpWrite(1.7s4 52/41 ECSubWrite(tid=245, reqid=osd.1.0:244, at_version=0'0, trim_to=0'0, roll_forward_to=52'29)) v2 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=51'27 active mbc={} ps=[19~1]] append_log log((0'0,52'29], crt=51'27) [] 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[19~1]] rollforward: entry=52'28 (45'20) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:29.630432+0000 0 snaps [e,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[19~1]] rollforward: entry=52'29 (51'27) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:29.630432+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[19~1]] append_log approx pg log length = 29 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[19~1]] append_log transaction_applied = 1 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:29.632+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c586b5b00 seq 73 2019-12-07T19:34:29.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586b5b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586b5b00 onodes shared_blobs 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586b5b00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5b00 prepare 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586b5b00 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5b00 io_done 2019-12-07T19:34:29.632+0000 7f8f0117b700 10 osd.7 52 dequeue_op 0x558c5511f760 finish 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b3976~471 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b3976 0x2b3976~471 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x283976 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x976 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xde7 and padding block with 0x219 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b3de7 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122367s (0.000000079s flush + 0.000122288s kv commit) 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586b5b00> 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5b00 kv_submitted 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586b5b00 2019-12-07T19:34:29.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586b5b00 finishing 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586b5b00 onodes 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586b5b00 done 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:29.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586b5b00 [] 2019-12-07T19:34:29.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.632+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 52/41 ECSubWriteReply(tid=245, last_complete=52'29, committed=1, applied=1)) v2 -- 0x558c58513400 con 0x558c54c64000 2019-12-07T19:34:29.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 413 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e52/52) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c54e05c00 con 0x558c54c64000 2019-12-07T19:34:29.632+0000 7f8f24457700 15 osd.7 52 enqueue_peering_evt 1.7s4 epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.7s0( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:29.632+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.7s0( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e52) 2019-12-07T19:34:29.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:29.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.7s0( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e52) queued 2019-12-07T19:34:29.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:29.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.7s0( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e52) pg 0x558c54d5f000 2019-12-07T19:34:29.632+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[19~1]] do_peering_event: epoch_sent: 52 epoch_requested: 52 MInfoRec from 1(0) info: 1.7s0( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:29.632+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[19~1]] proc_primary_info updating purged_snaps to [13~1,19~1] 2019-12-07T19:34:29.632+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:29.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:29.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c582a8c00 seq 74 2019-12-07T19:34:29.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:29.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:29.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:29.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:29.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a8c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:29.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a8c00 onodes shared_blobs 2019-12-07T19:34:29.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a8c00 allocated 0x[] released 0x[] 2019-12-07T19:34:29.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8c00 prepare 2019-12-07T19:34:29.633+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a8c00 2019-12-07T19:34:29.633+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8c00 io_done 2019-12-07T19:34:29.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b3de7~46d to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b3de7 0x2b3de7~46d to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x283de7 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xde7 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x254 and padding block with 0xdac 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b4254 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:29.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138729s (0.000000091s flush + 0.000138638s kv commit) 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:29.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582a8c00> 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:29.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8c00 kv_submitted 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a8c00 2019-12-07T19:34:29.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8c00 finishing 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a8c00 onodes 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a8c00 done 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:29.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a8c00 [] 2019-12-07T19:34:29.633+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:29.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:29.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 10 osd.7 52 tick_without_osd_lock 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f4b9000/0x40000000/0x1680000000, data 0xc82c04/0xb45000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 25 osd.7 52 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875883+0000 last_rx_front 2019-12-07T19:34:27.875916+0000 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 25 osd.7 52 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875931+0000 last_rx_front 2019-12-07T19:34:27.875870+0000 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 25 osd.7 52 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875893+0000 last_rx_front 2019-12-07T19:34:27.875903+0000 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 25 osd.7 52 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875766+0000 last_rx_front 2019-12-07T19:34:27.875813+0000 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 25 osd.7 52 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875796+0000 last_rx_front 2019-12-07T19:34:27.875837+0000 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 20 osd.7 52 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 10 osd.7 52 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 20 osd.7 52 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:29.680+0000 7f8f1fbbe700 10 osd.7 52 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:29.680+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.731+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.781+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.831+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.881+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.932+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:29.981+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.032+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.045+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 12 ==== osd_ping(ping e52 up_from 13 ping_stamp 2019-12-07T19:34:30.045899+0000/53.512223029s send_stamp 53.512223029s delta_ub 0.031057290s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9e00 con 0x558c54c5ac00 2019-12-07T19:34:30.045+0000 7f8f23c56700 20 osd.7 52 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030660380s,0.031057290s]) 2019-12-07T19:34:30.045+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 12 ==== osd_ping(ping e52 up_from 13 ping_stamp 2019-12-07T19:34:30.045899+0000/53.512223029s send_stamp 53.512223029s delta_ub 0.031057290s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795d000 con 0x558c54c5a400 2019-12-07T19:34:30.045+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e52 up_from 13 ping_stamp 2019-12-07T19:34:30.045899+0000/53.512223029s send_stamp 53.481562649s delta_ub 18446744073.678891236s) v5 -- 0x558c55271e00 con 0x558c54c5ac00 2019-12-07T19:34:30.045+0000 7f8f24c58700 20 osd.7 52 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030640896s,0.031057290s]) 2019-12-07T19:34:30.045+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e52 up_from 13 ping_stamp 2019-12-07T19:34:30.045899+0000/53.512223029s send_stamp 53.481582133s delta_ub 18446744073.678910720s) v5 -- 0x558c58332000 con 0x558c54c5a400 2019-12-07T19:34:30.082+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.132+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.182+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.232+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.282+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.307+0000 7f8f21451700 10 osd.7 52 tick 2019-12-07T19:34:30.307+0000 7f8f21451700 10 osd.7 52 do_waiters -- start 2019-12-07T19:34:30.307+0000 7f8f21451700 10 osd.7 52 do_waiters -- finish 2019-12-07T19:34:30.307+0000 7f8f21451700 20 osd.7 52 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:30.332+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.383+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.433+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.483+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.523+0000 7f8f1219d700 20 osd.7 52 reports for 0 queries 2019-12-07T19:34:30.523+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c55049c00 con 0x558c5476d800 2019-12-07T19:34:30.523+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:30.523+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0xc44000/0xab3000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:30.523+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(3 pgs tid 0 v 0) v2 -- 0x558c55049500 con 0x558c5476d800 2019-12-07T19:34:30.533+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.583+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1330604 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 242 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c58123500 con 0x558c54c5bc00 2019-12-07T19:34:30.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 414 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c54e05c00 con 0x558c54c64000 2019-12-07T19:34:30.619+0000 7f8f23c56700 15 osd.7 52 enqueue_peering_evt 1.fs5 epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:30.619+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.619+0000 7f8f24457700 15 osd.7 52 enqueue_peering_evt 1.es4 epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:30.619+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e53) pg 0x558c54d98000 2019-12-07T19:34:30.620+0000 7f8f0197c700 20 osd.7 op_wq _add_slot_waiter 1.es4 peering, item epoch is 53, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 416 ==== MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=246, reqid=osd.1.0:246, at_version=53'47, trim_to=0'0, roll_forward_to=52'45)) v2 ==== 3426+0+0 (crc 0 0 0) 0x558c57e01800 con 0x558c54c64000 2019-12-07T19:34:30.620+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 415 ==== osd_map(53..53 src has 1..53) v4 ==== 661+0+0 (crc 0 0 0) 0x558c581938c0 con 0x558c54c64000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 osd.7 52 OSD::ms_dispatch: osd_map(53..53 src has 1..53) v4 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 osd.7 52 do_waiters -- start 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 osd.7 52 do_waiters -- finish 2019-12-07T19:34:30.620+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c5511fa20 prio 127 cost 0 latency 0.000028 epoch 53 MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=246, reqid=osd.1.0:246, at_version=53'47, trim_to=0'0, roll_forward_to=52'45)) v2 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 osd.7 52 _dispatch 0x558c581938c0 osd_map(53..53 src has 1..53) v4 2019-12-07T19:34:30.620+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=246, reqid=osd.1.0:246, at_version=53'47, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e53) pg 0x558c57383000 2019-12-07T19:34:30.620+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 53, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.620+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.620+0000 7f8f1199c700 3 osd.7 52 handle_osd_map epochs [53,53], i have 52, src has [1,53] 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 osd.7 52 handle_osd_map got inc map for epoch 53 2019-12-07T19:34:30.620+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3d94e826:::osdmap.52:0# 0x0~0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c58b835e0 blob([0xb45000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5829c380 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=246, reqid=osd.1.0:246, at_version=53'47, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5829c310 blob([!~1000,0xb46000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c5829d9d0 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=246, reqid=osd.1.0:246, at_version=53'47, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e53) pg 0x558c57380000 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3d94e826:::osdmap.52:0# 0x0~1454 = 5204 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 osd.7 52 add_map_bl 52 5204 bytes 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 52 dequeue_op 0x558c5511fa20 prio 127 cost 0 latency 0.000091 MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=246, reqid=osd.1.0:246, at_version=53'47, trim_to=0'0, roll_forward_to=52'45)) v2 pg pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 52 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 52 -> 53 (as per caller) 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=246, reqid=osd.1.0:246, at_version=53'47, trim_to=0'0, roll_forward_to=52'45)) v2 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 snap_mapper.update_snaps 1:10779340:::smithi13140796-32:1b 15,1a,1b was 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:1b e,15,1a,1b 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 snap_mapper.set_snaps 1:10779340:::smithi13140796-32:1b 15,1a,1b 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 osd.7 52 got_full_map 53, nothing requested 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 52'45 (0'0,52'45] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[13~1,19~1]] append_log log((0'0,52'45], crt=52'45) [53'46 (52'42) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:30.620496+0000 0 snaps [15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'47 (52'43) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:30.620496+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'46 (0'0,53'46] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'45 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] add_log_entry 53'46 (52'42) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:30.620496+0000 0 snaps [15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'47 (0'0,53'47] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'45 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] add_log_entry 53'47 (52'43) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:30.620496+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 osd.7 52 handle_osd_map pg_num_history pg_num_history(e53 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {53={}} 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'47 (0'0,53'47] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'45 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 47 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 osd.7 52 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e53 [1,53] lci=[13,53]) 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'47 (0'0,53'47] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'45 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c580d2c00 seq 44 2019-12-07T19:34:30.620+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'46, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:ce07972f:::inc_osdmap.53:0# key 0x7f7fffffffffffffffce0797'/!inc_osdmap.53!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ce07972f:::inc_osdmap.53:0# 0x0~269 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1251 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:ce07972f:::inc_osdmap.53:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2c00 1 blobs 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb47000~1000/1000,1000,0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb47000~1000] 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5511cf00 seq 42 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e31f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809ab60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e31f0 blob([0xb47000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809ab60 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c574e31f0) get_ref 0x0~269 Blob(0x558c574e31f0 blob([0xb47000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809ab60 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c574e31f0) get_ref init 0x1000, 1000 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c574e31f0 blob([0xb47000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5809ab60 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5809ab78 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e388) dirty_range mark inline shard dirty 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:ce07972f:::inc_osdmap.53:0# 0x0~269 = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3294e826:::osdmap.53:0# key 0x7f7fffffffffffffff3294e8'&!osdmap.53!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3294e826:::osdmap.53:0# 0x0~1464 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1252 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3294e826:::osdmap.53:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2c00 2 blobs 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb48000~2000/1000,2000,0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb48000~2000] 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829c230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829c2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829c230 blob([0xb48000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c2a0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c5829c230) get_ref 0x0~1000 Blob(0x558c5829c230 blob([0xb48000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c2a0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c5829c230) get_ref init 0x1000, 1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5829c230 blob([0xb48000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5829c2a0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5829c2b8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809aa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579712d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809aa10 blob([!~1000,0xb49000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579712d0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c5809aa10) get_ref 0x1000~464 Blob(0x558c5809aa10 blob([!~1000,0xb49000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c579712d0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c5809aa10) get_ref init 0x2000, 1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c5809aa10 blob([!~1000,0xb49000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c579712d0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c579712e8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e608) dirty_range mark inline shard dirty 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3294e826:::osdmap.53:0# 0x0~1464 = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2c00 0 blobs 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2c00 0 blobs 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d2c00 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d2c00 onodes 0x558c53a91900,0x558c55276c80,0x558c5851e280,0x558c5851e500 shared_blobs 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e388) update #-1:ce07972f:::inc_osdmap.53:0# 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_000000000000000E_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e388) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:ce07972f:::inc_osdmap.53:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e608) update #-1:3294e826:::osdmap.53:0# 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e608) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:30.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3294e826:::osdmap.53:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000053.00000000000000000046' <- 0000000053.00000000000000000046 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d2c00 allocated 0x[b47000~3000] released 0x[] 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 freelist allocate 0xb47000~3000 2019-12-07T19:34:30.620+0000 7f8f1199c700 20 freelist _xor first_key 0xb00000 last_key 0xb00000 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000053.00000000000000000047' <- 0000000053.00000000000000000047 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2c00 prepare 2019-12-07T19:34:30.620+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c580d2c00 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511cf00 cost 8397 (1 ios * 4000 + 4397 bytes) 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511cf00 onodes 0x558c5840c500,0x558c5840c780 shared_blobs 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update 5#1:10779340:::smithi13140796-32:1b# 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:1b# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 491 (483 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511cf00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cf00 prepare 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511cf00 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cf00 io_done 2019-12-07T19:34:30.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 243 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c58123500 con 0x558c54c5bc00 2019-12-07T19:34:30.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'47 (0'0,53'47] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=52'45 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] op_applied version 53'47 2019-12-07T19:34:30.620+0000 7f8f23c56700 15 osd.7 52 enqueue_peering_evt 1.1s1 epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:30.620+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.620+0000 7f8f0898a700 10 osd.7 52 dequeue_op 0x558c5511fa20 finish 2019-12-07T19:34:30.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3777 < min_flush_size 524288 2019-12-07T19:34:30.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e53) pg 0x558c5736a000 2019-12-07T19:34:30.620+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 53, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c500 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3857973248 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.99805) bluestore 86 GiB free (0.959877), bluefs_ratio 0.0399304 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3800 < min_flush_size 524288 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3800 < min_flush_size 524288 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b4254~ed8 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b4254 0x2b4254~ed8 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x284254 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x254 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x12c and padding block with 0xed4 2019-12-07T19:34:30.620+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2c00 aio_wait 2019-12-07T19:34:30.620+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d2c00 2019-12-07T19:34:30.620+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2c00 io_done 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b512c 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000273880s (0.000000136s flush + 0.000273744s kv commit) 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5511cf00> 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cf00 kv_submitted 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511cf00 2019-12-07T19:34:30.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511cf00 finishing 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511cf00 onodes 0x558c5840c500,0x558c5840c780 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511cf00 done 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:30.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511cf00 [] 2019-12-07T19:34:30.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:30.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851e280 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851e500 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b512c~226c to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b512c 0x2b512c~226c to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x28512c 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x12c 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x398 and padding block with 0xc68 2019-12-07T19:34:30.621+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 417 ==== MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=247, reqid=osd.1.0:247, at_version=53'49, trim_to=0'0, roll_forward_to=52'45)) v2 ==== 3385+0+0 (crc 0 0 0) 0x558c5748d000 con 0x558c54c64000 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b7398 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.621+0000 7f8f24457700 15 osd.7 52 enqueue_op 0x558c5511e000 prio 127 cost 0 latency 0.000033 epoch 53 MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=247, reqid=osd.1.0:247, at_version=53'49, trim_to=0'0, roll_forward_to=52'45)) v2 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.621+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=247, reqid=osd.1.0:247, at_version=53'49, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000158957s (0.000004772s flush + 0.000154185s kv commit) 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d2c00> 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2c00 kv_submitted 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d2c00 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c580d2c00 osr 0x558c54ea4b40 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 41 0x1b000~1000 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 42 0x1b000~1000 crc 5434a025 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) _discard drop 41 0x2000~1000 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=247, reqid=osd.1.0:247, at_version=53'49, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c57f53a40) prepare_write seq 42 0x2000~1000 crc 91b3c8a9 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7 52 _committed_osd_maps 53..53 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=247, reqid=osd.1.0:247, at_version=53'49, trim_to=0'0, roll_forward_to=52'45)) v2) prio 127 cost 0 e53) pg 0x558c57380000 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7 52 advance to epoch 53 (<= last 53 <= newest_map 53) 2019-12-07T19:34:30.621+0000 7f8f141a1700 7 osd.7 53 consume_map version 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 52 dequeue_op 0x558c5511e000 prio 127 cost 0 latency 0.000084 MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=247, reqid=osd.1.0:247, at_version=53'49, trim_to=0'0, roll_forward_to=52'45)) v2 pg pg[1.8s5( v 53'47 (0'0,53'47] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.8s5 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.0s3 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.1s1 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'47 (0'0,53'47] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=247, reqid=osd.1.0:247, at_version=53'49, trim_to=0'0, roll_forward_to=52'45)) v2 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.9s4 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.as5 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.2s0 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.bs1 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 snap_mapper.update_snaps 1:131421e8:::smithi13140796-7:15 15 was 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.3s4 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.cs0 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.4s0 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.5s4 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.ds2 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.es4 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.6s2 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.fs5 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 identify_splits_and_merges 1.7s4 e52 to e53 pg_nums {18=16} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 53 prune_pg_created 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:0.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:15 e,15 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:1.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 pending_peering first epoch 53 <= 53, requeueing 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 snap_mapper.set_snaps 1:131421e8:::smithi13140796-7:15 15 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {53=} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:2.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:3.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:4.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:5.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:6.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 pending_peering first epoch 53 <= 53, requeueing 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {53=} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e53) pg 0x558c5736a000 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7:7.consume_map 53 (was 52) 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 pending_peering first epoch 53 <= 53, requeueing 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'47 (0'0,53'47] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'47], crt=52'45) [53'48 (52'44) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:30.621458+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'49 (52'45) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:30.621458+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {53=} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:30.621+0000 7f8f141a1700 10 osd.7 53 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'48 (0'0,53'48] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'47 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] add_log_entry 53'48 (52'44) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:30.621458+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'47 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] add_log_entry 53'49 (52'45) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:30.621458+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.8s5 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.0s3 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'47 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 49 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.1s1 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'47 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.9s4 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'48, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e53) pg 0x558c57383000 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:30.621+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c578a6000 seq 43 2019-12-07T19:34:30.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:30.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c58113200 seq 21 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# 1 keys = 0 2019-12-07T19:34:30.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) 2019-12-07T19:34:30.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 52 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:30.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:30.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:30.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58113200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:30.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58113200 onodes shared_blobs 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:30.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58113200 allocated 0x[] released 0x[] 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113200 prepare 2019-12-07T19:34:30.621+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58113200 2019-12-07T19:34:30.621+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113200 io_done 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c5736a000 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_000000000000000E_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000053.00000000000000000048' <- 0000000053.00000000000000000048 2019-12-07T19:34:30.621+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:30.621+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000053.00000000000000000049' <- 0000000053.00000000000000000049 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._fastinfo' <- _fastinfo 2019-12-07T19:34:30.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b7398~495 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a6000 cost 7548 (1 ios * 4000 + 3548 bytes) 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a6000 onodes 0x558c57598000,0x558c575f9b80 shared_blobs 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b7398 0x2b7398~495 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57598108) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x287398 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x398 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x82d and padding block with 0x7d3 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 994 (556 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575f9c88) update 5#1:131421e8:::smithi13140796-7:15# 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:15# is 849 (411 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a6000 allocated 0x[] released 0x[] 2019-12-07T19:34:30.621+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6000 prepare 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a6000 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6000 io_done 2019-12-07T19:34:30.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:30.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c582a8f00 seq 23 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'47 crt=52'45 lcod 52'45 active mbc={} ps=[13~1,19~1]] op_applied version 53'49 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 53 dequeue_op 0x558c5511e000 finish 2019-12-07T19:34:30.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:30.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:30.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:30.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:30.621+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 52/41 ECSubWriteReply(tid=246, last_complete=53'47, committed=1, applied=1)) v2 -- 0x558c54c59200 con 0x558c54c64000 2019-12-07T19:34:30.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:30.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a8f00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:30.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a8f00 onodes shared_blobs 2019-12-07T19:34:30.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a8f00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8f00 prepare 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a8f00 2019-12-07T19:34:30.621+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8f00 io_done 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c57380000 2019-12-07T19:34:30.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.as5 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.2s0 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.bs1 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c5500a000 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.3s4 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.cs0 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.4s0 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.5s4 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.ds2 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.es4 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c575ef000 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 52 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e53) pg 0x558c54d98000 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b782d 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.6s2 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 lcod 53'47 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.fs5 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000211675s (0.000000137s flush + 0.000211538s kv commit) 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c57522000 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:30.621+0000 7f8f141a1700 15 osd.7 53 enqueue_peering_evt 1.7s4 epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f141a1700 7 osd.7 53 activate_map version 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2003 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c5500d000 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3866 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f0918b700 20 osd.7 53 agent_entry empty queue 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 52 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57598000 had 1 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9b80 had 1 2019-12-07T19:34:30.621+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:30.621+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5016 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5039 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5039 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c5513e000 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b782d~13af to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c57e8b000 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b782d 0x2b782d~13af to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x28782d 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x82d 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbdc and padding block with 0x424 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 52 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c55141000 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c57383000 2019-12-07T19:34:30.621+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:30.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c575c4900 seq 19 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 52 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c54d5f000 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:30.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c4900 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c4900 onodes shared_blobs 2019-12-07T19:34:30.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b8bdc 2019-12-07T19:34:30.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c4900 allocated 0x[] released 0x[] 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4900 prepare 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 52 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c4900 2019-12-07T19:34:30.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4900 io_done 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000163812s (0.000000101s flush + 0.000163711s kv commit) 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c57e88000 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b8bdc~495 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s0 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b8bdc 0x2b8bdc~495 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c54d98000 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 52 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x288bdc 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbdc 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x71 and padding block with 0xf8f 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[13~1,19~1]] state: Active advmap 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 52 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] on_active_advmap new removed_snaps [e~1], snap_trimq now [e~1] 2019-12-07T19:34:30.621+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c57676000 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] state: Active: handling ActMap 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] Active: kicking snap trim 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:30.621+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean ps=[13~1,19~1]] state: Active advmap 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] exit NotTrimming 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] on_active_advmap new removed_snaps [e~1], snap_trimq now [e~1] 2019-12-07T19:34:30.621+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b9071 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132868s (0.000000085s flush + 0.000132783s kv commit) 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.621+0000 7f8f07988700 15 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 52:148 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.621+0000 7f8f07988700 7 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] state: handle ActMap primary 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] state: Active: handling ActMap 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] Active: kicking snap trim 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:30.621+0000 7f8f07988700 15 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 52:149 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] exit NotTrimming 2019-12-07T19:34:30.621+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming 2019-12-07T19:34:30.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58113200,0x558c578a6000,0x558c582a8f00,0x558c575c4900> 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113200 kv_submitted 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58113200 2019-12-07T19:34:30.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113200 finishing 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58113200 onodes 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58113200 done 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58113200 [] 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: NotTrimming: trimming e 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6000 kv_submitted 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a6000 2019-12-07T19:34:30.621+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 52:169 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6000 finishing 2019-12-07T19:34:30.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a6000 onodes 0x558c57598000,0x558c575f9b80 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a6000 done 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a6000 [] 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.621+0000 7f8f06986700 7 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] state: handle ActMap primary 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8f00 kv_submitted 2019-12-07T19:34:30.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a8f00 2019-12-07T19:34:30.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a8f00 finishing 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a8f00 onodes 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a8f00 done 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:30.621+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 53/41 ECSubWriteReply(tid=247, last_complete=53'49, committed=1, applied=1)) v2 -- 0x558c54c58400 con 0x558c54c64000 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a8f00 [] 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:150 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4900 kv_submitted 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c4900 2019-12-07T19:34:30.621+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 52:170 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4900 finishing 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c4900 onodes 2019-12-07T19:34:30.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c4900 done 2019-12-07T19:34:30.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c4900 [] 2019-12-07T19:34:30.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f0b990700 10 osd.7 53 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[e~1] ps=[13~1,19~1]] for snaptrim 2019-12-07T19:34:30.621+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=53) prio 5 cost 1048576 e53) 2019-12-07T19:34:30.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.621+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=53) prio 5 cost 1048576 e53) queued 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:151 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 NullEvt) prio 255 cost 10 e53) pg 0x558c548ab000 2019-12-07T19:34:30.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=53) prio 5 cost 1048576 e53) pg 0x558c57e88000 2019-12-07T19:34:30.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:30.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: NotTrimming: trimming e 2019-12-07T19:34:30.621+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:30.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:30.621+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.621+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap e 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] handle_advance_map: 53 2019-12-07T19:34:30.621+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 52 -> 53 2019-12-07T19:34:30.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.621+0000 7f8f06986700 10 osd.7 pg_epoch: 52 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:30.621+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.622+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:30.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:171 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean ps=[13~1,19~1]] state: Active advmap 2019-12-07T19:34:30.622+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:30.622+0000 7f8f0b990700 10 osd.7 53 queueing pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean trimq=[e~1] ps=[13~1,19~1]] for snaptrim 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] on_active_advmap new removed_snaps [e~1], snap_trimq now [e~1] 2019-12-07T19:34:30.622+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=53) prio 5 cost 1048576 e53) 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] state: Started advmap 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:30.622+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:30.622+0000 7f8f0b990700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:172 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c58b7ffc0) upper_bound after SNA_1_000000000000000E_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.2' 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:30.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] hit_set_clear 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=53) prio 5 cost 1048576 e53) queued 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c58b7ffc0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] agent_stop 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=53) prio 5 cost 1048576 e53) pg 0x558c57676000 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 53 advance_pg new pool opts old pool opts 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] handle_activate_map: 53 2019-12-07T19:34:30.622+0000 7f8f03980700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000E_.0_0000000000000001.2) returns 0 SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..,buffer::list(len=70, buffer::ptr(0~70 0x558c55238f20 in raw 0x558c55238f20 len 70 nref 1) ) 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] activate_map 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap e 2019-12-07T19:34:30.622+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] state: Active: handling ActMap 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.622+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[13~1,19~1]] SnapTrimmer state: adding snap e to purged_snaps 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] Active: kicking snap trim 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:30.622+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fd00) upper_bound after SNA_1_000000000000000E_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.C' 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] exit NotTrimming 2019-12-07T19:34:30.622+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~1]] SnapTrimmer state: purged_snaps now [e~1,13~1,19~1], snap_trimq now [] 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming 2019-12-07T19:34:30.622+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fd00) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c5829a000 seq 27 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000E_.0_0000000000000001.C) returns 0 SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:30.622+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:30.622+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 52:134 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:30.622+0000 7f8f06986700 7 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] state: handle ActMap primary 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.622+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:30.622+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5829a000 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:30.622+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 52:135 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5829a000 onodes shared_blobs 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fd00) upper_bound after SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. key 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5829a000 allocated 0x[] released 0x[] 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] activate_map: Not dirtying info: last_persisted is 52 while current is 53 2019-12-07T19:34:30.622+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.622+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a000 prepare 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c58b7fd00) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:30.622+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5829a000 2019-12-07T19:34:30.622+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a000 io_done 2019-12-07T19:34:30.622+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 NullEvt 2019-12-07T19:34:30.622+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..) returns 0 SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239600 in raw 0x558c55239600 len 69 nref 1) ) 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:30.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.622+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~1]] share_pg_info 2019-12-07T19:34:30.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: found obc in cache: 0x558c576b2b40 2019-12-07T19:34:30.622+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c57a54300 con 0x558c54c64000 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: 0x558c576b2b40 1:3a5a3f5b:::smithi13140796-23:e rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:e(45'29 client.4540.0:234 dirty|data_digest s 1474560 uv 16 dd 4bbcd6e5 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e]} 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b9071~495 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b9071 0x2b9071~495 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.622+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c57a52e00 con 0x558c54c19800 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x289071 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x71 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x506 and padding block with 0xafa 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(45'30 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{e=[e]} 2019-12-07T19:34:30.622+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c57a55100 con 0x558c54c5bc00 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:3a5a3f5b:::smithi13140796-23:e old_snaps e old snapset e=[]:{e=[e]} 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] simple_opc_create 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c55055100 con 0x558c54c5b400 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:3a5a3f5b:::smithi13140796-23:e snaps e -> ... deleting 2019-12-07T19:34:30.622+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c55053500 con 0x558c54c19c00 2019-12-07T19:34:30.622+0000 7f8f03980700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:3a5a3f5b:::smithi13140796-23:e new snapset e=[]:{} on 1:3a5a3f5b:::smithi13140796-23:head(45'30 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:30.622+0000 7f8f03980700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~1]] exit Trimming 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:3a5a3f5b:::smithi13140796-23:e writing updated snapset on 1:3a5a3f5b:::smithi13140796-23:head, snapset is e=[]:{} 2019-12-07T19:34:30.622+0000 7f8f03980700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:30.622+0000 7f8f03980700 15 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:152 2019-12-07T19:34:30.622+0000 7f8f03980700 20 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] enter NotTrimming 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] new_repop rep_tid 42 (no op) 2019-12-07T19:34:30.622+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] new_repop: repgather(0x558c576b6360 0'0 rep_tid=42 committed?=0 r=0) 2019-12-07T19:34:30.622+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] simple_opc_submit 0x558c576b6360 2019-12-07T19:34:30.622+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:30.622+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:30.622+0000 7f8f0297e700 7 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] issue_repop rep_tid 42 o 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:e v=53'38 tt=0'0 tid=42 reqid=osd.7.0:42 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: found on disk, size 368640 2019-12-07T19:34:30.622+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# hinfo_key = 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:e projected size 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:30.622+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=53'38 tt=0'0 tid=42 reqid=osd.7.0:42 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2b9506 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:e v=53'38 tt=0'0 tid=42 reqid=osd.7.0:42 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000289246s (0.000000100s flush + 0.000289146s kv commit) 2019-12-07T19:34:30.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=53'38 tt=0'0 tid=42 reqid=osd.7.0:42 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5829a000> 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a000 kv_submitted 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5829a000 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a000 finishing 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5829a000 onodes 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829a000 done 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:30.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5829a000 [] 2019-12-07T19:34:30.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.622+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:e, current size is 1474560 buffers are {} 2019-12-07T19:34:30.622+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): new_size start 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_append: {} 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:e resetting hinfo to logical size 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): new_size start 507904 2019-12-07T19:34:30.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 418 ==== MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=248, reqid=osd.1.0:248, at_version=53'36, trim_to=0'0, roll_forward_to=52'34)) v2 ==== 2648+0+0 (crc 0 0 0) 0x558c5748e000 con 0x558c54c64000 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:30.622+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5511fce0 prio 127 cost 0 latency 0.000037 epoch 53 MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=248, reqid=osd.1.0:248, at_version=53'36, trim_to=0'0, roll_forward_to=52'34)) v2 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_append: {} 2019-12-07T19:34:30.622+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=248, reqid=osd.1.0:248, at_version=53'36, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:e={},1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:e v=53'38 tt=0'0 tid=42 reqid=osd.7.0:42 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:30.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=248, reqid=osd.1.0:248, at_version=53'36, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:e,{} 2019-12-07T19:34:30.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.622+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=248, reqid=osd.1.0:248, at_version=53'36, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e53) pg 0x558c5500a000 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 53 dequeue_op 0x558c5511fce0 prio 127 cost 0 latency 0.000107 MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=248, reqid=osd.1.0:248, at_version=53'36, trim_to=0'0, roll_forward_to=52'34)) v2 pg pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=248, reqid=osd.1.0:248, at_version=53'36, trim_to=0'0, roll_forward_to=52'34)) v2 2019-12-07T19:34:30.622+0000 7f8f04181700 20 snap_mapper.remove_oid 1:9694a0e5:::smithi13140796-3:13 2019-12-07T19:34:30.622+0000 7f8f04181700 20 snap_mapper._remove_oid 1:9694a0e5:::smithi13140796-3:13 2019-12-07T19:34:30.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 53/28 ECSubWrite(tid=42, reqid=osd.7.0:42, at_version=53'38, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c574a4000 con 0x558c54c64000 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 send_incremental_map 52 -> 53 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:30.622+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ce07972f:::inc_osdmap.53:0# 0x0~0 2019-12-07T19:34:30.622+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:13 e 2019-12-07T19:34:30.622+0000 7f8f04181700 20 snap_mapper.clear_snaps 1:9694a0e5:::smithi13140796-3:13 2019-12-07T19:34:30.622+0000 7f8f04181700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:30.622+0000 7f8f04181700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574e31f0 blob([0xb47000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5809ab60 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:ce07972f:::inc_osdmap.53:0# 0x0~269 = 617 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 add_map_inc_bl 53 617 bytes 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(53..53 src has 1..53) v4 -- 0x558c58526b40 con 0x558c54c19800 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 52 -> 53 (shared) 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 53/28 ECSubWrite(tid=42, reqid=osd.7.0:42, at_version=53'38, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c574a2800 con 0x558c54c19800 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 52'34 (0'0,52'34] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] append_log log((0'0,52'34], crt=52'34) [53'35 (52'33) delete 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:30.622942+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'36 (52'34) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:30.622942+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 send_incremental_map 52 -> 53 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(53..53 src has 1..53) v4 -- 0x558c58526d80 con 0x558c54c5bc00 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 52 -> 53 (shared) 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 53/28 ECSubWrite(tid=42, reqid=osd.7.0:42, at_version=53'38, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c574a5800 con 0x558c54c5bc00 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'35 (0'0,53'35] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=52'34 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] add_log_entry 53'35 (52'33) delete 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:30.622942+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 send_incremental_map 52 -> 53 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:30.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 245 ==== MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=140, reqid=osd.3.0:140, at_version=53'63, trim_to=0'0, roll_forward_to=52'61)) v2 ==== 2648+0+0 (crc 0 0 0) 0x558c57e7c000 con 0x558c54c5bc00 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(53..53 src has 1..53) v4 -- 0x558c58526fc0 con 0x558c54c5b400 2019-12-07T19:34:30.622+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c54e78580 prio 127 cost 0 latency 0.000028 epoch 53 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=140, reqid=osd.3.0:140, at_version=53'63, trim_to=0'0, roll_forward_to=52'61)) v2 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 52 -> 53 (shared) 2019-12-07T19:34:30.622+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=140, reqid=osd.3.0:140, at_version=53'63, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 53/28 ECSubWrite(tid=42, reqid=osd.7.0:42, at_version=53'38, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5831d800 con 0x558c54c5b400 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'36 (0'0,53'36] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=52'34 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] add_log_entry 53'36 (52'34) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:30.622942+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 send_incremental_map 52 -> 53 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(53..53 src has 1..53) v4 -- 0x558c58527200 con 0x558c54c19c00 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'36 (0'0,53'36] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=52'34 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 36 2019-12-07T19:34:30.622+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 244 ==== osd_map(53..53 src has 1..53) v4 ==== 661+0+0 (crc 0 0 0) 0x558c576c0fc0 con 0x558c54c5bc00 2019-12-07T19:34:30.622+0000 7f8f1199c700 20 osd.7 53 OSD::ms_dispatch: osd_map(53..53 src has 1..53) v4 2019-12-07T19:34:30.622+0000 7f8f1199c700 10 osd.7 53 do_waiters -- start 2019-12-07T19:34:30.622+0000 7f8f1199c700 10 osd.7 53 do_waiters -- finish 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 53 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 52 -> 53 (shared) 2019-12-07T19:34:30.622+0000 7f8f1199c700 20 osd.7 53 _dispatch 0x558c576c0fc0 osd_map(53..53 src has 1..53) v4 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'36 (0'0,53'36] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=52'34 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.622+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.622+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 53/28 ECSubWrite(tid=42, reqid=osd.7.0:42, at_version=53'38, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5831d000 con 0x558c54c19c00 2019-12-07T19:34:30.622+0000 7f8f1199c700 3 osd.7 53 handle_osd_map epochs [53,53], i have 53, src has [1,53] 2019-12-07T19:34:30.622+0000 7f8f1199c700 10 osd.7 53 no new maps here, dropping 2019-12-07T19:34:30.622+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'35, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.remove_oid 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper._remove_oid 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:e e 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.clear_snaps 1:3a5a3f5b:::smithi13140796-23:e 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper.clear_snaps rm OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=140, reqid=osd.3.0:140, at_version=53'63, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5511d500 seq 73 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=140, reqid=osd.3.0:140, at_version=53'63, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9694a0e5:::smithi13140796-3:13#23 key 0x8480000000000000019694a0e521736d'ithi13140796-3!='0x00000000000000130000000000000023'o' 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c54e78580 prio 127 cost 0 latency 0.000114 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=140, reqid=osd.3.0:140, at_version=53'63, trim_to=0'0, roll_forward_to=52'61)) v2 pg pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 trim 53'37 (45'29) delete 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 trim 53'38 (45'30) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=9 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=140, reqid=osd.3.0:140, at_version=53'63, trim_to=0'0, roll_forward_to=52'61)) v2 2019-12-07T19:34:30.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# -> 4#1:9694a0e5:::smithi13140796-3:13#23 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 snap_mapper.remove_oid 1:6c205177:::smithi13140796-8:10 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 snap_mapper._remove_oid 1:6c205177:::smithi13140796-8:10 2019-12-07T19:34:30.622+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 52'36 (0'0,52'36] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log log((0'0,52'36], crt=52'36) [53'37 (45'29) delete 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'38 (45'30) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13# -> 4#1:9694a0e5:::smithi13140796-3:13#23 = 0 2019-12-07T19:34:30.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'37 (0'0,53'37] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] add_log_entry 53'37 (45'29) delete 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:10 e 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 snap_mapper.clear_snaps 1:6c205177:::smithi13140796-8:10 2019-12-07T19:34:30.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] add_log_entry 53'38 (45'30) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log approx pg log length = 38 2019-12-07T19:34:30.622+0000 7f8f0197c700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:30.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- OBJ_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.622+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'37, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.9692507A.13.smithi13140796-3..' <- SNA_1_000000000000000E_.4_0000000000000001.9692507A.13.smithi13140796-3.. 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.622+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 52'61 (0'0,52'61] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] append_log log((0'0,52'61], crt=52'61) [53'62 (45'45) delete 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:30.623102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'63 (45'46) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:30.623102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000053.00000000000000000035' <- 0000000053.00000000000000000035 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000053.00000000000000000036' <- 0000000053.00000000000000000036 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511d500 cost 7542 (1 ios * 4000 + 3542 bytes) 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'62 (0'0,53'62] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'61 crt=52'61 lcod 52'61 active mbc={} ps=[13~1,19~1]] add_log_entry 53'62 (45'45) delete 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:30.623102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511d500 onodes 0x558c54ce2a00,0x558c54e45680 shared_blobs 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c54ce2b08) update 4#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:head# is 499 (497 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:30.622+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'63 (0'0,53'63] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'61 crt=52'61 lcod 52'61 active mbc={} ps=[13~1,19~1]] add_log_entry 53'63 (45'46) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:30.623102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore.extentmap(0x558c54e45788) update 4#1:9694a0e5:::smithi13140796-3:13#23 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585f7b00 seq 52 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9694a0e5:::smithi13140796-3:13#23 is 906 (411 bytes onode + 2 bytes spanning blobs + 493 bytes inline extents) 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'63 (0'0,53'63] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'61 crt=52'61 lcod 52'61 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 63 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511d500 allocated 0x[] released 0x[] 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'63 (0'0,53'63] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'61 crt=52'61 lcod 52'61 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.622+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d500 prepare 2019-12-07T19:34:30.622+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511d500 2019-12-07T19:34:30.622+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d500 io_done 2019-12-07T19:34:30.622+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:e#25 key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0x000000000000000e0000000000000025'o' 2019-12-07T19:34:30.622+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'62, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:30.622+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'36 (0'0,53'36] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=52'34 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] op_applied version 53'36 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# -> 0#1:3a5a3f5b:::smithi13140796-23:e#25 2019-12-07T19:34:30.623+0000 7f8f04181700 10 osd.7 53 dequeue_op 0x558c5511fce0 finish 2019-12-07T19:34:30.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e# -> 0#1:3a5a3f5b:::smithi13140796-23:e#25 = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:30.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23..' <- SNA_1_000000000000000E_.0_0000000000000001.C5A5CFAD.e.smithi13140796-23.. 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000053.00000000000000000037' <- 0000000053.00000000000000000037 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000053.00000000000000000038' <- 0000000053.00000000000000000038 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582aec00 seq 98 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f7b00 cost 7533 (1 ios * 4000 + 3533 bytes) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f7b00 onodes 0x558c57f35b80,0x558c58092000 shared_blobs 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) update 0#1:3a5a3f5b:::smithi13140796-23:e#25 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) update shard 0x0 is 487 bytes (was 487) from 18 extents 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:10#3e key 0x8280000000000000016c2051'w!smithi13140796-8!='0x0000000000000010000000000000003e'o' 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) update shard 0x23000 is 491 bytes (was 491) from 18 extents 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) update shard 0x47000 is 288 bytes (was 288) from 10 extents 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:e#25 is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58092108) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 795 (485 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# -> 2#1:6c205177:::smithi13140796-8:10#3e 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7b00 prepare 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f7b00 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7b00 io_done 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:10# -> 2#1:6c205177:::smithi13140796-8:10#3e = 0 2019-12-07T19:34:30.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3868 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] op_applied version 53'38 2019-12-07T19:34:30.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] eval_repop repgather(0x558c576b6360 53'38 rep_tid=42 committed?=0 r=0) (no op) 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] calc_trim_to_aggressive limit = 52'36 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6340A8EE.10.smithi13140796-8..' <- SNA_1_000000000000000E_.2_0000000000000001.6340A8EE.10.smithi13140796-8.. 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2a00 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45680 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851fb80 had 1 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: found obc in cache: 0x558c5738f8c0 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3891 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000062' <- 0000000053.00000000000000000062 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: 0x558c5738f8c0 1:33f5bf01:::smithi13140796-2:19 rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:19(52'35 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[15,e]} 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000063' <- 0000000053.00000000000000000063 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(52'36 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[15,e]} 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3891 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:33f5bf01:::smithi13140796-2:19 old_snaps e,15 old snapset 19=[]:{19=[15,e]} 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582aec00 cost 7542 (1 ios * 4000 + 3542 bytes) 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582aec00 onodes 0x558c574a1b80,0x558c575ba280 shared_blobs 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] simple_opc_create 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore.extentmap(0x558c574a1c88) update 2#1:6c205177:::smithi13140796-8:10#3e 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2b9506~f33 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:33f5bf01:::smithi13140796-2:19 snaps e,15 -> 15 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:10#3e is 687 (411 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2b9506 0x2b9506~f33 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x289506 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x506 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 1512 (483 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x439 and padding block with 0xbc7 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582aec00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aec00 prepare 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582aec00 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aec00 io_done 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:33f5bf01:::smithi13140796-2:19 new snapset 19=[]:{19=[15]} on 1:33f5bf01:::smithi13140796-2:head(52'36 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:33f5bf01:::smithi13140796-2:19 writing updated snapset on 1:33f5bf01:::smithi13140796-2:head, snapset is 19=[]:{19=[15]} 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'63 (0'0,53'63] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=52'61 crt=52'61 lcod 52'61 active mbc={} ps=[13~1,19~1]] op_applied version 53'63 2019-12-07T19:34:30.623+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:30.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] new_repop rep_tid 43 (no op) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] new_repop: repgather(0x558c576b6fc0 0'0 rep_tid=43 committed?=0 r=0) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] simple_opc_submit 0x558c576b6fc0 2019-12-07T19:34:30.623+0000 7f8f0297e700 7 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] issue_repop rep_tid 43 o 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:19 v=53'40 tt=0'0 tid=43 reqid=osd.7.0:43 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:30.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 168 ==== MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=65, reqid=osd.2.0:65, at_version=53'20, trim_to=0'0, roll_forward_to=52'18)) v2 ==== 3426+0+0 (crc 0 0 0) 0x558c574a2800 con 0x558c54c19800 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:30.623+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c581cd340 prio 127 cost 0 latency 0.000026 epoch 53 MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=65, reqid=osd.2.0:65, at_version=53'20, trim_to=0'0, roll_forward_to=52'18)) v2 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:30.623+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=65, reqid=osd.2.0:65, at_version=53'20, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key = 0 2019-12-07T19:34:30.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 169 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52e00 con 0x558c54c19800 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 projected size 901120 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:30.623+0000 7f8f24c58700 15 osd.7 53 enqueue_peering_evt 1.5s4 epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:30.623+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2ba439 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000244854s (0.000000108s flush + 0.000244746s kv commit) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e53) pg 0x558c57522000 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=53'40 tt=0'0 tid=43 reqid=osd.7.0:43 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5301 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:19 v=53'40 tt=0'0 tid=43 reqid=osd.7.0:43 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=53'40 tt=0'0 tid=43 reqid=osd.7.0:43 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:30.623+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c58525200 seq 20 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35b80 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092000 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a3180 had 1 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:19, current size is 901120 buffers are {} 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9963 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58525200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58525200 onodes shared_blobs 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58525200 allocated 0x[] released 0x[] 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): new_size start 901120 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525200 prepare 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58525200 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525200 io_done 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1b80 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e780 had 1 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_append: {} 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9986 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9986 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=65, reqid=osd.2.0:65, at_version=53'20, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 resetting hinfo to logical size 901120 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=65, reqid=osd.2.0:65, at_version=53'20, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e53) pg 0x558c55141000 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2ba439~2702 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2ba439 0x2ba439~2702 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x28a439 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x439 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 53 dequeue_op 0x558c581cd340 prio 127 cost 0 latency 0.000195 MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=65, reqid=osd.2.0:65, at_version=53'20, trim_to=0'0, roll_forward_to=52'18)) v2 pg pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb3b and padding block with 0x4c5 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=65, reqid=osd.2.0:65, at_version=53'20, trim_to=0'0, roll_forward_to=52'18)) v2 2019-12-07T19:34:30.623+0000 7f8f06185700 20 snap_mapper.update_snaps 1:bca18876:::smithi13140796-30:1b 15,1a,1b was 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): new_size start 1622016 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_append: {} 2019-12-07T19:34:30.623+0000 7f8f06185700 20 snap_mapper.get_snaps 1:bca18876:::smithi13140796-30:1b e,15,1a,1b 2019-12-07T19:34:30.623+0000 7f8f06185700 20 snap_mapper.set_snaps 1:bca18876:::smithi13140796-30:1b 15,1a,1b 2019-12-07T19:34:30.623+0000 7f8f06185700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:30.623+0000 7f8f06185700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:19={},1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 52'18 (0'0,52'18] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] append_log log((0'0,52'18], crt=52'18) [53'19 (52'15) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:30.623424+0000 0 snaps [15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'20 (52'16) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:30.623424+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=53'40 tt=0'0 tid=43 reqid=osd.7.0:43 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'19 (0'0,53'19] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'18 crt=52'18 lcod 52'18 active mbc={} ps=[13~1,19~1]] add_log_entry 53'19 (52'15) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:30.623424+0000 0 snaps [15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:19,{} 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'20 (0'0,53'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'18 crt=52'18 lcod 52'18 active mbc={} ps=[13~1,19~1]] add_log_entry 53'20 (52'16) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:30.623424+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'20 (0'0,53'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'18 crt=52'18 lcod 52'18 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 20 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'20 (0'0,53'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'18 crt=52'18 lcod 52'18 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.623+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.623+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'19, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 53/28 ECSubWrite(tid=43, reqid=osd.7.0:43, at_version=53'40, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5831c800 con 0x558c54c64000 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58525500 seq 52 2019-12-07T19:34:30.623+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 53/28 ECSubWrite(tid=43, reqid=osd.7.0:43, at_version=53'40, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5831c000 con 0x558c54c19800 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys 2019-12-07T19:34:30.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 419 ==== MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=249, reqid=osd.1.0:249, at_version=53'38, trim_to=0'0, roll_forward_to=52'34)) v2 ==== 2648+0+0 (crc 0 0 0) 0x558c5831d000 con 0x558c54c64000 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys = 0 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) 2019-12-07T19:34:30.623+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5511f4a0 prio 127 cost 0 latency 0.000030 epoch 53 MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=249, reqid=osd.1.0:249, at_version=53'38, trim_to=0'0, roll_forward_to=52'34)) v2 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.623+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=249, reqid=osd.1.0:249, at_version=53'38, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:30.623+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 53/28 ECSubWrite(tid=43, reqid=osd.7.0:43, at_version=53'40, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5831b800 con 0x558c54c5bc00 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 53/28 ECSubWrite(tid=43, reqid=osd.7.0:43, at_version=53'40, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c5831b000 con 0x558c54c5b400 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_000000000000000E_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 53/28 ECSubWrite(tid=43, reqid=osd.7.0:43, at_version=53'40, trim_to=0'0, roll_forward_to=52'36)) v2 -- 0x558c54e3e000 con 0x558c54c19c00 2019-12-07T19:34:30.623+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000053.00000000000000000019' <- 0000000053.00000000000000000019 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000053.00000000000000000020' <- 0000000053.00000000000000000020 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 snap_mapper.update_snaps 1:33f5bf01:::smithi13140796-2:19 15 was 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58525500 cost 8415 (1 ios * 4000 + 4415 bytes) 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58525500 onodes 0x558c54d96780,0x558c54e13900 shared_blobs 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore.extentmap(0x558c54d96888) update 2#1:bca18876:::smithi13140796-30:1b# 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:1b# is 826 (412 bytes onode + 2 bytes spanning blobs + 412 bytes inline extents) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:33f5bf01:::smithi13140796-2:19 e,15 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore.extentmap(0x558c54e13a08) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 snap_mapper.set_snaps 1:33f5bf01:::smithi13140796-2:19 15 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 1459 (557 bytes onode + 2 bytes spanning blobs + 900 bytes inline extents) 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58525500 allocated 0x[] released 0x[] 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525500 prepare 2019-12-07T19:34:30.623+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58525500 2019-12-07T19:34:30.623+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525500 io_done 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:30.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 trim 53'39 (52'35) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'20 (0'0,53'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=52'18 crt=52'18 lcod 52'18 active mbc={} ps=[13~1,19~1]] op_applied version 53'20 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 trim 53'40 (52'36) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.623+0000 7f8f06185700 10 osd.7 53 dequeue_op 0x558c581cd340 finish 2019-12-07T19:34:30.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=249, reqid=osd.1.0:249, at_version=53'38, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=249, reqid=osd.1.0:249, at_version=53'38, trim_to=0'0, roll_forward_to=52'34)) v2) prio 127 cost 0 e53) pg 0x558c5500a000 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'38 (0'0,53'38] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log log((0'0,53'38], crt=52'36) [53'39 (52'35) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'40 (52'36) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'39 (0'0,53'39] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=53'38 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] add_log_entry 53'39 (52'35) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.623+0000 7f8f04181700 10 osd.7 53 dequeue_op 0x558c5511f4a0 prio 127 cost 0 latency 0.000143 MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=249, reqid=osd.1.0:249, at_version=53'38, trim_to=0'0, roll_forward_to=52'34)) v2 pg pg[1.9s4( v 53'36 (0'0,53'36] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=53'38 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] add_log_entry 53'40 (52'36) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.623+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'36 (0'0,53'36] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=249, reqid=osd.1.0:249, at_version=53'38, trim_to=0'0, roll_forward_to=52'34)) v2 2019-12-07T19:34:30.623+0000 7f8f04181700 20 snap_mapper.remove_oid 1:9b8adabe:::smithi13140796-6:11 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=53'38 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log approx pg log length = 40 2019-12-07T19:34:30.623+0000 7f8f04181700 20 snap_mapper._remove_oid 1:9b8adabe:::smithi13140796-6:11 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=53'38 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.623+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.623+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:30.623+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'39, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.623+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 167 ==== osd_map(53..53 src has 1..53) v4 ==== 661+0+0 (crc 0 0 0) 0x558c5781c6c0 con 0x558c54c19800 2019-12-07T19:34:30.623+0000 7f8f1199c700 20 osd.7 53 OSD::ms_dispatch: osd_map(53..53 src has 1..53) v4 2019-12-07T19:34:30.623+0000 7f8f1199c700 10 osd.7 53 do_waiters -- start 2019-12-07T19:34:30.623+0000 7f8f1199c700 10 osd.7 53 do_waiters -- finish 2019-12-07T19:34:30.623+0000 7f8f1199c700 20 osd.7 53 _dispatch 0x558c5781c6c0 osd_map(53..53 src has 1..53) v4 2019-12-07T19:34:30.623+0000 7f8f1199c700 3 osd.7 53 handle_osd_map epochs [53,53], i have 53, src has [1,53] 2019-12-07T19:34:30.623+0000 7f8f1199c700 10 osd.7 53 no new maps here, dropping 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:30.623+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c578ab500 seq 53 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 1 keys = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:30.623+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.623+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.623+0000 7f8f0b990700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:30.623+0000 7f8f0b990700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: NotTrimming: trimming e 2019-12-07T19:34:30.623+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim_wait trimq=[e~1] ps=[13~1,19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:30.623+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.623+0000 7f8f0b990700 15 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:136 2019-12-07T19:34:30.623+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5511d500> 2019-12-07T19:34:30.623+0000 7f8f0b990700 10 osd.7 53 queueing pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean trimq=[e~1] ps=[13~1,19~1]] for snaptrim 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d500 kv_submitted 2019-12-07T19:34:30.623+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=53) prio 5 cost 1048576 e53) 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511d500 2019-12-07T19:34:30.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d500 finishing 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511d500 onodes 0x558c54ce2a00,0x558c54e45680 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511d500 done 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:30.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511d500 [] 2019-12-07T19:34:30.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.623+0000 7f8f0b990700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2bcb3b 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.623+0000 7f8f0b990700 15 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:137 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000614927s (0.000000098s flush + 0.000614829s kv commit) 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:30.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:30.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6318 < min_flush_size 524288 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96780 had 1 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13900 had 1 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6341 < min_flush_size 524288 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6341 < min_flush_size 524288 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2bcb3b~18c5 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2bcb3b 0x2bcb3b~18c5 to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~290000 x_off 0x28cb3b 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb3b 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x400 and padding block with 0xc00 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585f7b00,0x558c582aec00> 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7b00 kv_submitted 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f7b00 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f7b00 finishing 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f7b00 onodes 0x558c57f35b80,0x558c58092000 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f7b00 done 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578ab500 prepare 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f7b00 [] 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aec00 kv_submitted 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582aec00 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aec00 finishing 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582aec00 onodes 0x558c574a1b80,0x558c575ba280 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582aec00 done 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582aec00 [] 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.624+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.624+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 53/28 ECSubWriteReply(tid=140, last_complete=53'63, committed=1, applied=1)) v2 -- 0x558c58512200 con 0x558c54c5bc00 2019-12-07T19:34:30.624+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2be400 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000191198s (0.000000080s flush + 0.000191118s kv commit) 2019-12-07T19:34:30.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58525200,0x558c58525500> 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525200 kv_submitted 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58525200 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525200 finishing 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58525200 onodes 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58525200 done 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58525200 [] 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525500 kv_submitted 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58525500 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525500 finishing 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58525500 onodes 0x558c54d96780,0x558c54e13900 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58525500 done 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58525500 [] 2019-12-07T19:34:30.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.624+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 53/41 ECSubWriteReply(tid=65, last_complete=53'20, committed=1, applied=1)) v2 -- 0x558c58512a00 con 0x558c54c19800 2019-12-07T19:34:30.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=53) prio 5 cost 1048576 e53) queued 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=53) prio 5 cost 1048576 e53) pg 0x558c548ab000 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap e 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.624+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:30.624+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c58b7e4c0) upper_bound after SNA_1_000000000000000E_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.4' 2019-12-07T19:34:30.624+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c58b7e4c0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:30.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 101 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58513400 con 0x558c54c19c00 2019-12-07T19:34:30.624+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000E_.0_0000000000000001.4) returns 0 SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:30.624+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5511e580 prio 196 cost 0 latency 0.000015 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.624+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:30.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.624+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:30.624+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c58b7e4c0) upper_bound after SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. key 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' 2019-12-07T19:34:30.624+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c58b7e4c0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:30.624+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..) returns 0 SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239810 in raw 0x558c55239810 len 69 nref 1) ) 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: found obc in cache: 0x558c5524c480 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: 0x558c5524c480 1:276d5164:::smithi13140796-22:e rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:e(45'7 client.4540.0:65 dirty|data_digest s 360448 uv 2 dd 322c6cec alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e]} 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: found obc in cache: 0x558c57945440 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_object_context: 0x558c57945440 1:276d5164:::smithi13140796-22:head rwstate(none n=0 w=0) oi: 1:276d5164:::smithi13140796-22:head(45'8 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b7560 snapset: e=[]:{e=[e]} 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:276d5164:::smithi13140796-22:e old_snaps e old snapset e=[]:{e=[e]} 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] simple_opc_create 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:276d5164:::smithi13140796-22:e snaps e -> ... deleting 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:276d5164:::smithi13140796-22:e new snapset e=[]:{} on 1:276d5164:::smithi13140796-22:head(45'8 client.4540.0:305 dirty|data_digest s 311296 uv 4 dd 98d51239 alloc_hint [0 0 0]) 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 1:276d5164:::smithi13140796-22:e writing updated snapset on 1:276d5164:::smithi13140796-22:head, snapset is e=[]:{} 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] new_repop rep_tid 44 (no op) 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] new_repop: repgather(0x558c576b6ea0 0'0 rep_tid=44 committed?=0 r=0) 2019-12-07T19:34:30.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] simple_opc_submit 0x558c576b6ea0 2019-12-07T19:34:30.624+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:11 e 2019-12-07T19:34:30.624+0000 7f8f04181700 20 snap_mapper.clear_snaps 1:9b8adabe:::smithi13140796-6:11 2019-12-07T19:34:30.624+0000 7f8f06986700 7 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] issue_repop rep_tid 44 o 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.624+0000 7f8f04181700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:30.624+0000 7f8f04181700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] submit_transaction: op Op(1:276d5164:::smithi13140796-22:e v=53'10 tt=0'0 tid=44 reqid=osd.7.0:44 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.624+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'36 (0'0,53'36] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'36], crt=52'34) [53'37 (45'31) delete 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:30.623777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'38 (45'32) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:30.623777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:30.624+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'37 (0'0,53'37] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'36 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] add_log_entry 53'37 (45'31) delete 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:30.623777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.624+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'36 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] add_log_entry 53'38 (45'32) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:30.623777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: found on disk, size 90112 2019-12-07T19:34:30.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key 2019-12-07T19:34:30.624+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'36 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 38 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# hinfo_key = 0 2019-12-07T19:34:30.624+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'36 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.624+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:30.624+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'37, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:276d5164:::smithi13140796-22:e projected size 0 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:30.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5511d800 seq 74 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:30.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:34:30.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:11#25 key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0x00000000000000110000000000000025'o' 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:276d5164:::smithi13140796-22:head projected size 311296 2019-12-07T19:34:30.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# -> 4#1:9b8adabe:::smithi13140796-6:11#25 2019-12-07T19:34:30.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11# -> 4#1:9b8adabe:::smithi13140796-6:11#25 = 0 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] start_rmw: Op(1:276d5164:::smithi13140796-22:e v=53'10 tt=0'0 tid=44 reqid=osd.7.0:44 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:30.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:30.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:30.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_state_to_reads: Op(1:276d5164:::smithi13140796-22:e v=53'10 tt=0'0 tid=44 reqid=osd.7.0:44 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:30.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.624+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:e v=53'10 tt=0'0 tid=44 reqid=osd.7.0:44 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:30.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.9D15B5D7.11.smithi13140796-6..' <- SNA_1_000000000000000E_.4_0000000000000001.9D15B5D7.11.smithi13140796-6.. 2019-12-07T19:34:30.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000053.00000000000000000037' <- 0000000053.00000000000000000037 2019-12-07T19:34:30.624+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] generate_transactions: 1:276d5164:::smithi13140796-22:e, current size is 360448 buffers are {} 2019-12-07T19:34:30.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000053.00000000000000000038' <- 0000000053.00000000000000000038 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.625+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:30.625+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511d800 cost 7524 (1 ios * 4000 + 3524 bytes) 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511d800 onodes 0x558c57a35400,0x558c57e8f180 shared_blobs 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): new_size start 0 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_append: {} 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:276d5164:::smithi13140796-22:e resetting hinfo to logical size 0 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] generate_transactions: 1:276d5164:::smithi13140796-22:head, current size is 311296 buffers are {} 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 124429 (123370 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): new_size start 311296 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_overwrite: {} 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): to_append: {} 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] operator(): 1:276d5164:::smithi13140796-22:head resetting hinfo to logical size 311296 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written: {1:276d5164:::smithi13140796-22:e={},1:276d5164:::smithi13140796-22:head={}} 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore.extentmap(0x558c57e8f288) update 4#1:9b8adabe:::smithi13140796-6:11#25 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:e v=53'10 tt=0'0 tid=44 reqid=osd.7.0:44 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:11#25 is 55375 (54316 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written_set: {1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]} 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:e,{} 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: 1:276d5164:::smithi13140796-22:head,{} 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511d800 allocated 0x[] released 0x[] 2019-12-07T19:34:30.625+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d800 prepare 2019-12-07T19:34:30.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511d800 2019-12-07T19:34:30.625+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d800 io_done 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 53/41 ECSubWrite(tid=44, reqid=osd.7.0:44, at_version=53'10, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c5852b000 con 0x558c54c64000 2019-12-07T19:34:30.625+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'36 crt=52'34 lcod 52'34 active mbc={} ps=[13~1,19~1]] op_applied version 53'38 2019-12-07T19:34:30.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 102 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c58400 con 0x558c54c19c00 2019-12-07T19:34:30.625+0000 7f8f04181700 10 osd.7 53 dequeue_op 0x558c5511f4a0 finish 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7168 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5511e6e0 prio 196 cost 0 latency 0.000013 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.625+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 53/41 ECSubWrite(tid=44, reqid=osd.7.0:44, at_version=53'10, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c5852b800 con 0x558c54c19800 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 39936 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.4s1 53/41 ECSubWrite(tid=44, reqid=osd.7.0:44, at_version=53'10, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c5852c000 con 0x558c54c5bc00 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 72704 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 53/41 ECSubWrite(tid=44, reqid=osd.7.0:44, at_version=53'10, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c5852c800 con 0x558c54c5b400 2019-12-07T19:34:30.625+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 53/41 ECSubWrite(tid=44, reqid=osd.7.0:44, at_version=53'10, trim_to=0'0, roll_forward_to=45'8)) v2 -- 0x558c5852d000 con 0x558c54c19c00 2019-12-07T19:34:30.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 246 ==== MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=141, reqid=osd.3.0:141, at_version=53'65, trim_to=0'0, roll_forward_to=52'61)) v2 ==== 2656+0+0 (crc 0 0 0) 0x558c5831d800 con 0x558c54c5bc00 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 105472 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c54e79340 prio 127 cost 0 latency 0.000030 epoch 53 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=141, reqid=osd.3.0:141, at_version=53'65, trim_to=0'0, roll_forward_to=52'61)) v2 2019-12-07T19:34:30.625+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=141, reqid=osd.3.0:141, at_version=53'65, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.625+0000 7f8f06986700 20 snap_mapper.remove_oid 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.625+0000 7f8f06986700 20 snap_mapper._remove_oid 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 138240 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 171008 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 182311 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 270 ==== MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=102, reqid=osd.4.0:102, at_version=53'35, trim_to=0'0, roll_forward_to=52'33)) v2 ==== 3394+0+0 (unknown 0 0 0) 0x558c551dc000 con 0x558c54c5b400 2019-12-07T19:34:30.625+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c54e79600 prio 127 cost 0 latency 0.000028 epoch 53 MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=102, reqid=osd.4.0:102, at_version=53'35, trim_to=0'0, roll_forward_to=52'33)) v2 2019-12-07T19:34:30.625+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=102, reqid=osd.4.0:102, at_version=53'35, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f180 had 1 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572a00 had 1 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=141, reqid=osd.3.0:141, at_version=53'65, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 182334 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 182334 < min_flush_size 524288 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=141, reqid=osd.3.0:141, at_version=53'65, trim_to=0'0, roll_forward_to=52'61)) v2) prio 127 cost 0 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2be400~2c83e to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2be400 0x2be400~2c83e to file(ino 16 size 0x2c0000 mtime 2019-12-07T19:34:29.629297+0000 allocated 2c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~290000]) 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _allocate len 0x2ac3e from 0 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _allocate len 0x2ac3e from 1 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x30000/10000,30000,acd3f0000 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c54e79340 prio 127 cost 0 latency 0.000196 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=141, reqid=osd.3.0:141, at_version=53'65, trim_to=0'0, roll_forward_to=52'61)) v2 pg pg[1.6s2( v 53'63 (0'0,53'63] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd3f0000~30000/10000,30000,acd3f0000 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x2f0000 to include allocated 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 61 (was clean) 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2c0000 x_off 0x28e400 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x400 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'63 (0'0,53'63] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=141, reqid=osd.3.0:141, at_version=53'65, trim_to=0'0, roll_forward_to=52'61)) v2 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc3e and padding block with 0x3c2 2019-12-07T19:34:30.625+0000 7f8f06986700 20 snap_mapper.get_snaps 1:276d5164:::smithi13140796-22:e e 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x2f0000 2019-12-07T19:34:30.625+0000 7f8f06986700 20 snap_mapper.clear_snaps 1:276d5164:::smithi13140796-22:e 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 snap_mapper.remove_oid 1:6ad15c4c:::smithi13140796-31:13 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 snap_mapper._remove_oid 1:6ad15c4c:::smithi13140796-31:13 2019-12-07T19:34:30.625+0000 7f8f06986700 20 snap_mapper.clear_snaps rm OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:30.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.625+0000 7f8f06986700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:30.625+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f06986700 20 trim 53'9 (45'7) delete 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6ad15c4c:::smithi13140796-31:13 e 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 snap_mapper.clear_snaps 1:6ad15c4c:::smithi13140796-31:13 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:30.625+0000 7f8f06986700 20 trim 53'10 (45'8) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 45'8 (0'0,45'8] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log log((0'0,45'8], crt=45'8) [53'9 (45'7) delete 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'10 (45'8) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'9 (0'0,53'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 lua=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] add_log_entry 53'9 (45'7) delete 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'63 (0'0,53'63] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'63], crt=52'61) [53'64 (52'58) delete 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:30.624363+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'65 (52'59) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:30.624363+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 lua=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] add_log_entry 53'10 (45'8) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'64 (0'0,53'64] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'63 crt=52'61 lcod 53'63 active mbc={} ps=[13~1,19~1]] add_log_entry 53'64 (52'58) delete 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:30.624363+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'63 crt=52'61 lcod 53'63 active mbc={} ps=[13~1,19~1]] add_log_entry 53'65 (52'59) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:30.624363+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 lua=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log approx pg log length = 10 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'63 crt=52'61 lcod 53'63 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 65 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 lua=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.625+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'63 crt=52'61 lcod 53'63 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.625+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'64, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.625+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.625+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 170 ==== MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=66, reqid=osd.2.0:66, at_version=53'22, trim_to=0'0, roll_forward_to=52'18)) v2 ==== 1799+0+0 (crc 0 0 0) 0x558c575be800 con 0x558c54c19800 2019-12-07T19:34:30.625+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000050 epoch 53 MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=66, reqid=osd.2.0:66, at_version=53'22, trim_to=0'0, roll_forward_to=52'18)) v2 2019-12-07T19:34:30.625+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=66, reqid=osd.2.0:66, at_version=53'22, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c582ad200 seq 42 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582aef00 seq 99 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) get_onode oid 0#1:276d5164:::smithi13140796-22:e#9 key 0x808000000000000001276d51'd!smithi13140796-22!='0x000000000000000e0000000000000009'o' 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ad15c4c:::smithi13140796-31:13#40 key 0x8280000000000000016ad15c'L!smithi13140796-31!='0x00000000000000130000000000000040'o' 2019-12-07T19:34:30.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=66, reqid=osd.2.0:66, at_version=53'22, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.4s0_head 0x558c54d0f0e0) r -2 v.len 0 2019-12-07T19:34:30.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:30.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# -> 0#1:276d5164:::smithi13140796-22:e#9 2019-12-07T19:34:30.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=66, reqid=osd.2.0:66, at_version=53'22, trim_to=0'0, roll_forward_to=52'18)) v2) prio 127 cost 0 e53) pg 0x558c55141000 2019-12-07T19:34:30.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# -> 2#1:6ad15c4c:::smithi13140796-31:13#40 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13# -> 2#1:6ad15c4c:::smithi13140796-31:13#40 = 0 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 53 dequeue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000114 MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=66, reqid=osd.2.0:66, at_version=53'22, trim_to=0'0, roll_forward_to=52'18)) v2 pg pg[1.ds2( v 53'20 (0'0,53'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.4s0_head 0#1:276d5164:::smithi13140796-22:e# -> 0#1:276d5164:::smithi13140796-22:e#9 = 0 2019-12-07T19:34:30.625+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 53/41 ECSubWriteReply(tid=248, last_complete=53'36, committed=1, applied=1)) v2 -- 0x558c5841f800 con 0x558c54c64000 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:34:30.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys 2019-12-07T19:34:30.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# 2 keys = 0 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'20 (0'0,53'20] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=66, reqid=osd.2.0:66, at_version=53'22, trim_to=0'0, roll_forward_to=52'18)) v2 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.625+0000 7f8f06185700 20 snap_mapper.remove_oid 1:bf81b261:::smithi13140796-37:13 2019-12-07T19:34:30.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f06185700 20 snap_mapper._remove_oid 1:bf81b261:::smithi13140796-37:13 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.4s0_head 0#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- OBJ_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=102, reqid=osd.4.0:102, at_version=53'35, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.625+0000 7f8f06185700 20 snap_mapper.get_snaps 1:bf81b261:::smithi13140796-37:13 e 2019-12-07T19:34:30.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f06185700 20 snap_mapper.clear_snaps 1:bf81b261:::smithi13140796-37:13 2019-12-07T19:34:30.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=102, reqid=osd.4.0:102, at_version=53'35, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e53) pg 0x558c575ef000 2019-12-07T19:34:30.625+0000 7f8f06185700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:30.625+0000 7f8f06185700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:30.625+0000 7f8f07187700 10 osd.7 53 dequeue_op 0x558c54e79600 prio 127 cost 0 latency 0.000385 MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=102, reqid=osd.4.0:102, at_version=53'35, trim_to=0'0, roll_forward_to=52'33)) v2 pg pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.625+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=102, reqid=osd.4.0:102, at_version=53'35, trim_to=0'0, roll_forward_to=52'33)) v2 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'20 (0'0,53'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'20], crt=52'18) [53'21 (52'17) delete 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:30.625774+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'22 (52'18) delete 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:30.625774+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.625+0000 7f8f07187700 20 snap_mapper.update_snaps 1:dce63cc3:::smithi13140796-50:15 15 was 2019-12-07T19:34:30.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'21 (0'0,53'21] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'20 crt=52'18 lcod 53'20 active mbc={} ps=[13~1,19~1]] add_log_entry 53'21 (52'17) delete 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:30.625774+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'20 crt=52'18 lcod 53'20 active mbc={} ps=[13~1,19~1]] add_log_entry 53'22 (52'18) delete 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:30.625774+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f07187700 20 snap_mapper.get_snaps 1:dce63cc3:::smithi13140796-50:15 e,15 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'20 crt=52'18 lcod 53'20 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 22 2019-12-07T19:34:30.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f07187700 20 snap_mapper.set_snaps 1:dce63cc3:::smithi13140796-50:15 15 2019-12-07T19:34:30.625+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2eac3e 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'20 crt=52'18 lcod 53'20 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- OBJ_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:30.625+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.625+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'21, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000E_.0_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000053.00000000000000000009' <- 0000000053.00000000000000000009 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.0000000053.00000000000000000010' <- 0000000053.00000000000000000010 2019-12-07T19:34:30.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.65B8A323.13.smithi13140796-31..' <- SNA_1_000000000000000E_.2_0000000000000001.65B8A323.13.smithi13140796-31.. 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58525800 seq 53 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ad200 cost 7533 (1 ios * 4000 + 3533 bytes) 2019-12-07T19:34:30.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ad200 onodes 0x558c57f35900,0x558c58093400 shared_blobs 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore.extentmap(0x558c57f35a08) update 0#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000064' <- 0000000053.00000000000000000064 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bf81b261:::smithi13140796-37:13#15 key 0x828000000000000001bf81b2'a!smithi13140796-37!='0x00000000000000130000000000000015'o' 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000065' <- 0000000053.00000000000000000065 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:head# is 681 (485 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore.extentmap(0x558c58093508) update 0#1:276d5164:::smithi13140796-22:e#9 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:276d5164:::smithi13140796-22:e#9 is 718 (412 bytes onode + 2 bytes spanning blobs + 304 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# -> 2#1:bf81b261:::smithi13140796-37:13#15 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ad200 allocated 0x[] released 0x[] 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (61) on file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]), flushing log 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582aef00 cost 7533 (1 ios * 4000 + 3533 bytes) 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582aef00 onodes 0x558c54ce2280,0x558c57e8f900 shared_blobs 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13# -> 2#1:bf81b261:::smithi13140796-37:13#15 = 0 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad200 prepare 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce2388) update 2#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:34:30.625+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ad200 2019-12-07T19:34:30.625+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad200 io_done 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:head# is 501 (499 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 61 len 0x4a crc 0xe7a814e1) 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bf81b261:::smithi13140796-37:head#16 key 0x828000000000000001bf81b2'a!smithi13140796-37!='0xfffffffffffffffe0000000000000016'o' 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57e8fa08) update 2#1:6ad15c4c:::smithi13140796-31:13#40 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x3c000~1000 to file(ino 1 size 0x3c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ad15c4c:::smithi13140796-31:13#40 is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x3c000 0x3c000~1000 to file(ino 1 size 0x3c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:30.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# -> 2#1:bf81b261:::smithi13140796-37:head#16 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582aef00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x3d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3c000 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 lua=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] op_applied version 53'10 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head# -> 2#1:bf81b261:::smithi13140796-37:head#16 = 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aef00 prepare 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582aef00 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aef00 io_done 2019-12-07T19:34:30.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- OBJ_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] eval_repop repgather(0x558c576b6ea0 53'10 rep_tid=44 committed?=0 r=0) (no op) 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.DF18D468.13.smithi13140796-37..' <- SNA_1_000000000000000E_.2_0000000000000001.DF18D468.13.smithi13140796-37.. 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'63 crt=52'61 lcod 53'63 active mbc={} ps=[13~1,19~1]] op_applied version 53'65 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] calc_trim_to_aggressive limit = 45'8 2019-12-07T19:34:30.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:30.625+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c54e79340 finish 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000053.00000000000000000021' <- 0000000053.00000000000000000021 2019-12-07T19:34:30.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000053.00000000000000000022' <- 0000000053.00000000000000000022 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.625+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] enter Trimming/WaitRepops 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58525800 cost 6676 (1 ios * 4000 + 2676 bytes) 2019-12-07T19:34:30.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x3d000 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58525800 onodes 0x558c54d96c80,0x558c57976000 shared_blobs 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:30.625+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:30.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore.extentmap(0x558c54d96d88) update 2#1:bf81b261:::smithi13140796-37:head#16 2019-12-07T19:34:30.625+0000 7f8f07187700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:30.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 420 ==== MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=250, reqid=osd.1.0:247, at_version=0'0, trim_to=0'0, roll_forward_to=53'49)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c5852d000 con 0x558c54c64000 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:head#16 is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 271 ==== MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=103, reqid=osd.4.0:103, at_version=53'19, trim_to=0'0, roll_forward_to=45'17)) v2 ==== 2640+0+0 (unknown 0 0 0) 0x558c5852c000 con 0x558c54c5b400 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore.extentmap(0x558c57976108) update 2#1:bf81b261:::smithi13140796-37:13#15 2019-12-07T19:34:30.625+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5511f080 prio 127 cost 0 latency 0.000020 epoch 53 MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=250, reqid=osd.1.0:247, at_version=0'0, trim_to=0'0, roll_forward_to=53'49)) v2 2019-12-07T19:34:30.625+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57334f20 prio 127 cost 0 latency 0.000025 epoch 53 MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=103, reqid=osd.4.0:103, at_version=53'19, trim_to=0'0, roll_forward_to=45'17)) v2 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bf81b261:::smithi13140796-37:13#15 is 931 (412 bytes onode + 2 bytes spanning blobs + 517 bytes inline extents) 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58525800 allocated 0x[] released 0x[] 2019-12-07T19:34:30.625+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=103, reqid=osd.4.0:103, at_version=53'19, trim_to=0'0, roll_forward_to=45'17)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.625+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 52'33 (0'0,52'33] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] append_log log((0'0,52'33], crt=52'33) [53'34 (52'30) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:30.625395+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'35 (52'31) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:30.625395+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525800 prepare 2019-12-07T19:34:30.625+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.625+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58525800 2019-12-07T19:34:30.625+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525800 io_done 2019-12-07T19:34:30.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=103, reqid=osd.4.0:103, at_version=53'19, trim_to=0'0, roll_forward_to=45'17)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.625+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'34 (0'0,53'34] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'33 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] add_log_entry 53'34 (52'30) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:30.625395+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'20 crt=52'18 lcod 53'20 active mbc={} ps=[13~1,19~1]] op_applied version 53'22 2019-12-07T19:34:30.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.625+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'35 (0'0,53'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'33 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] add_log_entry 53'35 (52'31) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:30.625395+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=103, reqid=osd.4.0:103, at_version=53'19, trim_to=0'0, roll_forward_to=45'17)) v2) prio 127 cost 0 e53) pg 0x558c5513e000 2019-12-07T19:34:30.625+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'35 (0'0,53'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'33 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 35 2019-12-07T19:34:30.625+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'35 (0'0,53'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'33 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.625+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.625+0000 7f8f0317f700 10 osd.7 53 dequeue_op 0x558c57334f20 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=103, reqid=osd.4.0:103, at_version=53'19, trim_to=0'0, roll_forward_to=45'17)) v2 pg pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.626+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'34, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=103, reqid=osd.4.0:103, at_version=53'19, trim_to=0'0, roll_forward_to=45'17)) v2 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 snap_mapper.remove_oid 1:c8ce3553:::smithi13140796-29:e 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 snap_mapper._remove_oid 1:c8ce3553:::smithi13140796-29:e 2019-12-07T19:34:30.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush cnt:3 2019-12-07T19:34:30.625+0000 7f8f06185700 10 osd.7 53 dequeue_op 0x558c581cdb80 finish 2019-12-07T19:34:30.626+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.626+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58522900 seq 42 2019-12-07T19:34:30.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# 1 keys = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.625+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=250, reqid=osd.1.0:247, at_version=0'0, trim_to=0'0, roll_forward_to=53'49)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=250, reqid=osd.1.0:247, at_version=0'0, trim_to=0'0, roll_forward_to=53'49)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=250, reqid=osd.1.0:247, at_version=0'0, trim_to=0'0, roll_forward_to=53'49)) v2) prio 127 cost 0 e53) pg 0x558c57380000 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 osd.7 53 dequeue_op 0x558c5511f080 prio 127 cost 0 latency 0.000274 MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=250, reqid=osd.1.0:247, at_version=0'0, trim_to=0'0, roll_forward_to=53'49)) v2 pg pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 53/41 ECSubWrite(tid=250, reqid=osd.1.0:247, at_version=0'0, trim_to=0'0, roll_forward_to=53'49)) v2 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=52'45 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'49], crt=52'45) [] 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'46 (52'42) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:30.620496+0000 0 snaps [15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'47 (52'43) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:30.620496+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'48 (52'44) modify 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:30.621458+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'49 (52'45) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:30.621458+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 49 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.626+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c578a6300 seq 44 2019-12-07T19:34:30.626+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a6300 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a6300 onodes shared_blobs 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a6300 allocated 0x[] released 0x[] 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6300 prepare 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a6300 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6300 io_done 2019-12-07T19:34:30.626+0000 7f8f0898a700 10 osd.7 53 dequeue_op 0x558c5511f080 finish 2019-12-07T19:34:30.626+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000467 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 61 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001254741s (0.000000115s flush + 0.001254626s kv commit) 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3852 < min_flush_size 524288 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5511d800> 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d800 kv_submitted 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511d800 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511d800 finishing 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511d800 onodes 0x558c57a35400,0x558c57e8f180 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511d800 done 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511d800 [] 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35900 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093400 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 3 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd4000 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7650 < min_flush_size 524288 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce2280 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f900 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8ea00 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 11096 < min_flush_size 524288 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57454f00 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8f680 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12234 < min_flush_size 524288 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12257 < min_flush_size 524288 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12257 < min_flush_size 524288 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2eac3e~2fe1 to file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2eac3e 0x2eac3e~2fe1 to file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2c0000 x_off 0x2bac3e 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc3e 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc1f and padding block with 0x3e1 2019-12-07T19:34:30.626+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.626+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 53/41 ECSubWriteReply(tid=249, last_complete=53'38, committed=1, applied=1)) v2 -- 0x558c54c1f800 con 0x558c54c64000 2019-12-07T19:34:30.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 103 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f800 con 0x558c54c19c00 2019-12-07T19:34:30.626+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.626+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5511f1e0 prio 196 cost 0 latency 0.000016 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.626+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2edc1f 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000238508s (0.000000115s flush + 0.000238393s kv commit) 2019-12-07T19:34:30.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582ad200,0x558c582aef00,0x558c58525800,0x558c578a6300> 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad200 kv_submitted 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ad200 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad200 finishing 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ad200 onodes 0x558c57f35900,0x558c58093400 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ad200 done 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ad200 [] 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aef00 kv_submitted 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582aef00 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582aef00 finishing 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582aef00 onodes 0x558c54ce2280,0x558c57e8f900 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582aef00 done 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582aef00 [] 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525800 kv_submitted 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58525800 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525800 finishing 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58525800 onodes 0x558c54d96c80,0x558c57976000 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58525800 done 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58525800 [] 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6300 kv_submitted 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a6300 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6300 finishing 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a6300 onodes 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a6300 done 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a6300 [] 2019-12-07T19:34:30.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.626+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 53/28 ECSubWriteReply(tid=141, last_complete=53'65, committed=1, applied=1)) v2 -- 0x558c58512400 con 0x558c54c5bc00 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.626+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 snap_mapper.get_snaps 1:c8ce3553:::smithi13140796-29:e e 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 snap_mapper.clear_snaps 1:c8ce3553:::smithi13140796-29:e 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 45'17 (0'0,45'17] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] append_log log((0'0,45'17], crt=45'17) [53'18 (45'16) delete 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:30.626447+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'19 (45'17) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:30.626447+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'18 (0'0,53'18] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=45'17 crt=45'17 lcod 45'17 active mbc={} ps=[13~1,19~1]] add_log_entry 53'18 (45'16) delete 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:30.626447+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=45'17 crt=45'17 lcod 45'17 active mbc={} ps=[13~1,19~1]] add_log_entry 53'19 (45'17) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:30.626447+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=45'17 crt=45'17 lcod 45'17 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 19 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=45'17 crt=45'17 lcod 45'17 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.626+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'18, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_000000000000000E_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5810ec00 seq 61 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000053.00000000000000000034' <- 0000000053.00000000000000000034 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) get_onode oid 4#1:c8ce3553:::smithi13140796-29:e#12 key 0x848000000000000001c8ce35'S!smithi13140796-29!='0x000000000000000e0000000000000012'o' 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.3s4_head 0x558c54d0e780) r -2 v.len 0 2019-12-07T19:34:30.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# -> 4#1:c8ce3553:::smithi13140796-29:e#12 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e# -> 4#1:c8ce3553:::smithi13140796-29:e#12 = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000053.00000000000000000035' <- 0000000053.00000000000000000035 2019-12-07T19:34:30.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# 2 keys = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58522900 cost 8367 (1 ios * 4000 + 4367 bytes) 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58522900 onodes 0x558c574a0a00,0x558c574a0c80 shared_blobs 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0b08) update 1#1:dce63cc3:::smithi13140796-50:15# 2019-12-07T19:34:30.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:15# is 1120 (412 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- OBJ_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore.extentmap(0x558c574a0d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:30.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 1265 (557 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000E_.4_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58522900 allocated 0x[] released 0x[] 2019-12-07T19:34:30.626+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.626+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522900 prepare 2019-12-07T19:34:30.626+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58522900 2019-12-07T19:34:30.626+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522900 io_done 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000053.00000000000000000018' <- 0000000053.00000000000000000018 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000053.00000000000000000019' <- 0000000053.00000000000000000019 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.626+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.627+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:30.626+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'35 (0'0,53'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=52'33 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] op_applied version 53'35 2019-12-07T19:34:30.627+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810ec00 cost 7533 (1 ios * 4000 + 3533 bytes) 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810ec00 onodes 0x558c57976500,0x558c57976780 shared_blobs 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57976608) update 4#1:c8ce3553:::smithi13140796-29:e#12 2019-12-07T19:34:30.627+0000 7f8f07187700 10 osd.7 53 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:30.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:e#12 is 823 (412 bytes onode + 2 bytes spanning blobs + 409 bytes inline extents) 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57976888) update 4#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c8ce3553:::smithi13140796-29:head# is 855 (485 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810ec00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.627+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ec00 prepare 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810ec00 2019-12-07T19:34:30.627+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ec00 io_done 2019-12-07T19:34:30.627+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=45'17 crt=45'17 lcod 45'17 active mbc={} ps=[13~1,19~1]] op_applied version 53'19 2019-12-07T19:34:30.627+0000 7f8f0317f700 10 osd.7 53 dequeue_op 0x558c57334f20 finish 2019-12-07T19:34:30.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 421 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55053500 con 0x558c54c64000 2019-12-07T19:34:30.627+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f24457700 15 osd.7 53 enqueue_peering_evt 1.8s5 epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.8s0( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:30.627+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.8s0( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.627+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 53/41 ECSubWriteReply(tid=66, last_complete=53'22, committed=1, applied=1)) v2 -- 0x558c58512c00 con 0x558c54c19800 2019-12-07T19:34:30.627+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_000000000000000E_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.8s0( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.627+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:30.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 272 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58512400 con 0x558c54c5b400 2019-12-07T19:34:30.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.8s0( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e53) pg 0x558c57380000 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000053.00000000000000000039' <- 0000000053.00000000000000000039 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000053.00000000000000000040' <- 0000000053.00000000000000000040 2019-12-07T19:34:30.627+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c573358c0 prio 196 cost 0 latency 0.000016 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.627+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.8s0( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578ab500 cost 7516 (1 ios * 4000 + 3516 bytes) 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578ab500 onodes 0x558c58295180,0x558c58295400 shared_blobs 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295288) update 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:19# is 1172 (410 bytes onode + 2 bytes spanning blobs + 760 bytes inline extents) 2019-12-07T19:34:30.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 273 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58512200 con 0x558c54c5b400 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:30.627+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57335e40 prio 196 cost 0 latency 0.000012 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1497 (541 bytes onode + 2 bytes spanning blobs + 954 bytes inline extents) 2019-12-07T19:34:30.627+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578ab500 allocated 0x[] released 0x[] 2019-12-07T19:34:30.627+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab500 prepare 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578ab500 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab500 io_done 2019-12-07T19:34:30.627+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:30.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:30.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58c64f00 seq 45 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 lua=53'38 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] op_applied version 53'40 2019-12-07T19:34:30.627+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:30.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:30.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:30.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:30.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c64f00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c64f00 onodes shared_blobs 2019-12-07T19:34:30.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c64f00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64f00 prepare 2019-12-07T19:34:30.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c64f00 2019-12-07T19:34:30.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64f00 io_done 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] eval_repop repgather(0x558c576b6fc0 53'40 rep_tid=43 committed?=0 r=0) (no op) 2019-12-07T19:34:30.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] calc_trim_to_aggressive limit = 52'36 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] enter Trimming/WaitRepops 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511e580 prio 196 cost 0 latency 0.002671 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511e580 finish 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c573358c0 prio 196 cost 0 latency 0.000206 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c573358c0 finish 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.627+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 53/41 ECSubWriteReply(tid=250, last_complete=53'49, committed=1, applied=1)) v2 -- 0x558c54c1f200 con 0x558c54c64000 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511e6e0 prio 196 cost 0 latency 0.002148 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511e6e0 finish 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57335e40 prio 196 cost 0 latency 0.000227 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57335e40 finish 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511f1e0 prio 196 cost 0 latency 0.000744 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511f1e0 finish 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 171 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512a00 con 0x558c54c19800 2019-12-07T19:34:30.627+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c581cd1e0 prio 196 cost 0 latency 0.000024 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c581cd1e0 prio 196 cost 0 latency 0.000049 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.627+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c581cd1e0 finish 2019-12-07T19:34:30.627+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5220 < min_flush_size 524288 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 3 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9185 < min_flush_size 524288 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9362 < min_flush_size 524288 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976500 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976780 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57c04c80 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14027 < min_flush_size 524288 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295180 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 15153 < min_flush_size 524288 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 15176 < min_flush_size 524288 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 15176 < min_flush_size 524288 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2edc1f~3b48 to file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2edc1f 0x2edc1f~3b48 to file(ino 16 size 0x2f0000 mtime 2019-12-07T19:34:30.626665+0000 allocated 2f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2c0000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _allocate len 0x1767 from 0 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _allocate len 0x1767 from 1 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd420000 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd420000~10000/10000,10000,acd420000 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x300000 to include allocated 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 62 (was clean) 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2bdc1f 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc1f 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x767 and padding block with 0x899 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x300000 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f1767 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (62) on file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]), flushing log 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 62 len 0x4a crc 0xcefc30ab) 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x3d000~1000 to file(ino 1 size 0x3d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x3d000 0x3d000~1000 to file(ino 1 size 0x3d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x3e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3d000 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x3e000 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:30.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000027 2019-12-07T19:34:30.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 274 ==== MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=104, reqid=osd.4.0:104, at_version=53'37, trim_to=0'0, roll_forward_to=52'33)) v2 ==== 3394+0+0 (unknown 0 0 0) 0x558c5852c800 con 0x558c54c5b400 2019-12-07T19:34:30.628+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57335ce0 prio 127 cost 0 latency 0.000031 epoch 53 MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=104, reqid=osd.4.0:104, at_version=53'37, trim_to=0'0, roll_forward_to=52'33)) v2 2019-12-07T19:34:30.628+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=104, reqid=osd.4.0:104, at_version=53'37, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=104, reqid=osd.4.0:104, at_version=53'37, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=104, reqid=osd.4.0:104, at_version=53'37, trim_to=0'0, roll_forward_to=52'33)) v2) prio 127 cost 0 e53) pg 0x558c575ef000 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 62 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 53 dequeue_op 0x558c57335ce0 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=104, reqid=osd.4.0:104, at_version=53'37, trim_to=0'0, roll_forward_to=52'33)) v2 pg pg[1.bs1( v 53'35 (0'0,53'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000499906s (0.000000104s flush + 0.000499802s kv commit) 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'35 (0'0,53'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=104, reqid=osd.4.0:104, at_version=53'37, trim_to=0'0, roll_forward_to=52'33)) v2 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 snap_mapper.update_snaps 1:db85c185:::smithi13140796-18:15 15 was 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58522900,0x558c5810ec00,0x558c578ab500,0x558c58c64f00> 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522900 kv_submitted 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58522900 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522900 finishing 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58522900 onodes 0x558c574a0a00,0x558c574a0c80 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522900 done 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58522900 [] 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ec00 kv_submitted 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810ec00 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 snap_mapper.get_snaps 1:db85c185:::smithi13140796-18:15 e,15 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ec00 finishing 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810ec00 onodes 0x558c57976500,0x558c57976780 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810ec00 done 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810ec00 [] 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab500 kv_submitted 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 snap_mapper.set_snaps 1:db85c185:::smithi13140796-18:15 15 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578ab500 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab500 finishing 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578ab500 onodes 0x558c58295180,0x558c58295400 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578ab500 done 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578ab500 [] 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64f00 kv_submitted 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c64f00 2019-12-07T19:34:30.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c64f00 finishing 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c64f00 onodes 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c64f00 done 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c64f00 [] 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'35 (0'0,53'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'35], crt=52'33) [53'36 (52'32) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:30.628696+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'37 (52'33) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:30.628696+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'36 (0'0,53'36] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'35 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] add_log_entry 53'36 (52'32) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:30.628696+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'35 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] add_log_entry 53'37 (52'33) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:30.628696+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'35 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 37 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'35 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.628+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'36, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5810ef00 seq 43 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# 1 keys = 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_000000000000000E_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000053.00000000000000000036' <- 0000000053.00000000000000000036 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000053.00000000000000000037' <- 0000000053.00000000000000000037 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810ef00 cost 7559 (1 ios * 4000 + 3559 bytes) 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810ef00 onodes 0x558c57573680,0x558c57e8ec80 shared_blobs 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57573788) update 1#1:db85c185:::smithi13140796-18:15# 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:15# is 664 (412 bytes onode + 2 bytes spanning blobs + 250 bytes inline extents) 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1587 (558 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810ef00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ef00 prepare 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810ef00 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ef00 io_done 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'35 crt=52'33 lcod 52'33 active mbc={} ps=[13~1,19~1]] op_applied version 53'37 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4272 < min_flush_size 524288 2019-12-07T19:34:30.628+0000 7f8f0317f700 10 osd.7 53 dequeue_op 0x558c57335ce0 finish 2019-12-07T19:34:30.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573680 had 1 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4295 < min_flush_size 524288 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4295 < min_flush_size 524288 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f1767~10c7 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f1767 0x2f1767~10c7 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c1767 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x767 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x82e and padding block with 0x7d2 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f282e 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 275 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c580b9e00 con 0x558c54c5b400 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000115947s (0.000000089s flush + 0.000115858s kv commit) 2019-12-07T19:34:30.628+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000015 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.628+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5810ef00> 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ef00 kv_submitted 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810ef00 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810ef00 finishing 2019-12-07T19:34:30.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810ef00 onodes 0x558c57573680,0x558c57e8ec80 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810ef00 done 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:30.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810ef00 [] 2019-12-07T19:34:30.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.628+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c578d3b80 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.628+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.628+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:30.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.628+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 53/41 ECSubWriteReply(tid=102, last_complete=53'35, committed=1, applied=1)) v2 -- 0x558c57519a00 con 0x558c54c5b400 2019-12-07T19:34:30.628+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 53/41 ECSubWriteReply(tid=103, last_complete=53'19, committed=1, applied=1)) v2 -- 0x558c57518000 con 0x558c54c5b400 2019-12-07T19:34:30.628+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 53/41 ECSubWriteReply(tid=104, last_complete=53'37, committed=1, applied=1)) v2 -- 0x558c54c1e000 con 0x558c54c5b400 2019-12-07T19:34:30.628+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.629+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 172 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512000 con 0x558c54c19800 2019-12-07T19:34:30.629+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c581cc840 prio 196 cost 0 latency 0.000017 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.629+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.629+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c581cc840 prio 196 cost 0 latency 0.000046 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.629+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c581cc840 finish 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 422 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c1f200 con 0x558c54c64000 2019-12-07T19:34:30.629+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c57335b80 prio 196 cost 0 latency 0.000016 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.629+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 423 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c1f800 con 0x558c54c64000 2019-12-07T19:34:30.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000007 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.629+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57335b80 prio 196 cost 0 latency 0.000043 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.629+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.629+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57335b80 finish 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.629+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000046 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.629+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c581ccb00 finish 2019-12-07T19:34:30.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.629+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 247 ==== MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=142, reqid=osd.3.0:141, at_version=0'0, trim_to=0'0, roll_forward_to=53'65)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c579e1000 con 0x558c54c5bc00 2019-12-07T19:34:30.629+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c578d3a20 prio 127 cost 0 latency 0.000019 epoch 53 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=142, reqid=osd.3.0:141, at_version=0'0, trim_to=0'0, roll_forward_to=53'65)) v2 2019-12-07T19:34:30.629+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=142, reqid=osd.3.0:141, at_version=0'0, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=142, reqid=osd.3.0:141, at_version=0'0, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.629+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=142, reqid=osd.3.0:141, at_version=0'0, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.629+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c578d3a20 prio 127 cost 0 latency 0.000041 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=142, reqid=osd.3.0:141, at_version=0'0, trim_to=0'0, roll_forward_to=53'65)) v2 pg pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=142, reqid=osd.3.0:141, at_version=0'0, trim_to=0'0, roll_forward_to=53'65)) v2 2019-12-07T19:34:30.629+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=52'61 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'65], crt=52'61) [] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'62 (45'45) delete 1:6c205177:::smithi13140796-8:10 by unknown.0.0:0 2019-12-07T19:34:30.623102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'63 (45'46) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:30.623102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'64 (52'58) delete 1:6ad15c4c:::smithi13140796-31:13 by unknown.0.0:0 2019-12-07T19:34:30.624363+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'65 (52'59) modify 1:6ad15c4c:::smithi13140796-31:head by unknown.0.0:0 2019-12-07T19:34:30.624363+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 65 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.630+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582af200 seq 100 2019-12-07T19:34:30.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:10#3e onode 0x558c574a1b80 txc 0x558c582af200 2019-12-07T19:34:30.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:10#3e 0x0 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd260) put_ref 0x0~2000 Blob(0x558c57efd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd1f0) put_ref 0x0~2000 Blob(0x558c57efd1f0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8070) put_ref 0x0~2000 Blob(0x558c54df8070 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3ab0) put_ref 0x0~2000 Blob(0x558c577f3ab0 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd2d0) put_ref 0x0~2000 Blob(0x558c57efd2d0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd340) put_ref 0x0~2000 Blob(0x558c57efd340 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd180) put_ref 0x0~2000 Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd420) put_ref 0x0~2000 Blob(0x558c57efd420 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd490) put_ref 0x0~2000 Blob(0x558c57efd490 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd500) put_ref 0x0~1000 Blob(0x558c57efd500 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.extentmap(0x558c574a1c88) dirty_range mark inline shard dirty 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57efd260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x26f000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57efd1f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x271000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54df8070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x273000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c577f3ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x275000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57efd2d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x277000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57efd340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x279000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57efd180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27b000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57efd420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27d000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57efd490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27f000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~1000: 0x0~1000 Blob(0x558c57efd500 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x281000~1000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c575bd3b0 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:10#3e = 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13#40 onode 0x558c57e8f900 txc 0x558c582af200 2019-12-07T19:34:30.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13#40 0x0 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df10) put_ref 0x0~2000 Blob(0x558c54d3df10 blob([0x11c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57a07dc0) put_ref 0x0~2000 Blob(0x558c57a07dc0 blob([0x11e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57e63e30) put_ref 0x0~2000 Blob(0x558c57e63e30 blob([0x120000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42930) put_ref 0x0~2000 Blob(0x558c57c42930 blob([0x122000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c547402a0) put_ref 0x0~2000 Blob(0x558c547402a0 blob([0x124000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c55278f50) put_ref 0x0~2000 Blob(0x558c55278f50 blob([0x126000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c55279260) put_ref 0x0~2000 Blob(0x558c55279260 blob([0x128000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c54677880) put_ref 0x0~2000 Blob(0x558c54677880 blob([0x12a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c5523a1c0) put_ref 0x0~2000 Blob(0x558c5523a1c0 blob([0x12c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c55264070) put_ref 0x0~2000 Blob(0x558c55264070 blob([0x12e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6070) put_ref 0x0~2000 Blob(0x558c575f6070 blob([0x130000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6000) put_ref 0x0~2000 Blob(0x558c575f6000 blob([0x132000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6380) put_ref 0x0~2000 Blob(0x558c575f6380 blob([0x134000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7c00) put_ref 0x0~2000 Blob(0x558c575f7c00 blob([0x136000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7c70) put_ref 0x0~2000 Blob(0x558c575f7c70 blob([0x138000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7ce0) put_ref 0x0~1000 Blob(0x558c575f7ce0 blob([0x13a000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57e8fa08) dirty_range mark inline shard dirty 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d3df10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x11c000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57a07dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x11e000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57e63e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x120000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57c42930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x122000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c547402a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x124000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c55278f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x126000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c55279260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x128000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c54677880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x12a000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5523a1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x12c000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c55264070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x12e000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c575f6070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x130000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c575f6000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x132000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c575f6380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x134000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c575f7c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x136000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c575f7c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x138000~2000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~1000: 0x0~1000 Blob(0x558c575f7ce0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=2)))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x13a000~1000] 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54e43ea0,0x558c57538c40,0x558c57538d90,0x558c57538e70,0x558c57538f50,0x558c57539030,0x558c57539110,0x558c575391f0,0x558c575392d0,0x558c575393b0,0x558c57539490,0x558c57539570,0x558c57539650,0x558c57539730,0x558c57539810,0x558c575398f0 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ad15c4c:::smithi13140796-31:13#40 = 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582af200 cost 5586 (1 ios * 4000 + 1586 bytes) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582af200 onodes shared_blobs 0x558c54e43ea0,0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c57538c40,0x558c57538d90,0x558c57538e70,0x558c57538f50,0x558c57539030,0x558c57539110,0x558c575391f0,0x558c575392d0,0x558c575393b0,0x558c57539490,0x558c57539570,0x558c57539650,0x558c57539730,0x558c57539810,0x558c575398f0,0x558c575bd3b0 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f6 is 11 SharedBlob(0x558c54e43ea0 loaded (sbid 0x28f6 ref_map(0x11c000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f7 is 11 SharedBlob(0x558c57538c40 loaded (sbid 0x28f7 ref_map(0x11e000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f8 is 11 SharedBlob(0x558c57538d90 loaded (sbid 0x28f8 ref_map(0x120000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f9 is 11 SharedBlob(0x558c57538e70 loaded (sbid 0x28f9 ref_map(0x122000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fa is 11 SharedBlob(0x558c57538f50 loaded (sbid 0x28fa ref_map(0x124000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fb is 11 SharedBlob(0x558c57539030 loaded (sbid 0x28fb ref_map(0x126000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fc is 11 SharedBlob(0x558c57539110 loaded (sbid 0x28fc ref_map(0x128000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fd is 11 SharedBlob(0x558c575391f0 loaded (sbid 0x28fd ref_map(0x12a000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28fe is 11 SharedBlob(0x558c575392d0 loaded (sbid 0x28fe ref_map(0x12c000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ff is 11 SharedBlob(0x558c575393b0 loaded (sbid 0x28ff ref_map(0x12e000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2900 is 11 SharedBlob(0x558c57539490 loaded (sbid 0x2900 ref_map(0x130000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2901 is 11 SharedBlob(0x558c57539570 loaded (sbid 0x2901 ref_map(0x132000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2902 is 11 SharedBlob(0x558c57539650 loaded (sbid 0x2902 ref_map(0x134000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2903 is 11 SharedBlob(0x558c57539730 loaded (sbid 0x2903 ref_map(0x136000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2904 is 11 SharedBlob(0x558c57539810 loaded (sbid 0x2904 ref_map(0x138000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2905 is 11 SharedBlob(0x558c575398f0 loaded (sbid 0x2905 ref_map(0x13a000~1000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=1))) 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582af200 allocated 0x[] released 0x[] 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af200 prepare 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582af200 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af200 io_done 2019-12-07T19:34:30.630+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c578d3a20 finish 2019-12-07T19:34:30.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 424 ==== MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=252, reqid=osd.1.0:249, at_version=0'0, trim_to=0'0, roll_forward_to=53'38)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c5852b000 con 0x558c54c64000 2019-12-07T19:34:30.630+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c581ccc60 prio 127 cost 0 latency 0.000021 epoch 53 MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=252, reqid=osd.1.0:249, at_version=0'0, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:30.630+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=252, reqid=osd.1.0:249, at_version=0'0, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 425 ==== MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=251, reqid=osd.1.0:251, at_version=53'31, trim_to=0'0, roll_forward_to=52'29)) v2 ==== 3522+0+0 (crc 0 0 0) 0x558c54e3e000 con 0x558c54c64000 2019-12-07T19:34:30.630+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000027 epoch 53 MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=251, reqid=osd.1.0:251, at_version=53'31, trim_to=0'0, roll_forward_to=52'29)) v2 2019-12-07T19:34:30.630+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=251, reqid=osd.1.0:251, at_version=53'31, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=251, reqid=osd.1.0:251, at_version=53'31, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=251, reqid=osd.1.0:251, at_version=53'31, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e53) pg 0x558c54d5f000 2019-12-07T19:34:30.630+0000 7f8f05183700 10 osd.7 53 dequeue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000082 MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=251, reqid=osd.1.0:251, at_version=53'31, trim_to=0'0, roll_forward_to=52'29)) v2 pg pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.630+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=251, reqid=osd.1.0:251, at_version=53'31, trim_to=0'0, roll_forward_to=52'29)) v2 2019-12-07T19:34:30.630+0000 7f8f05183700 20 snap_mapper.update_snaps 1:ecc72bdb:::smithi13140796-10:15 15 was 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:15 e,15 2019-12-07T19:34:30.630+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:15 15 2019-12-07T19:34:30.630+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:30.630+0000 7f8f05183700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:30.630+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 52'29 (0'0,52'29] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] append_log log((0'0,52'29], crt=52'29) [53'30 (52'28) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:30.630519+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'31 (52'29) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:30.630519+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.630+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'30 (0'0,53'30] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=52'29 crt=52'29 lcod 52'29 active mbc={} ps=[13~1,19~1]] add_log_entry 53'30 (52'28) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:30.630519+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.630+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'31 (0'0,53'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=52'29 crt=52'29 lcod 52'29 active mbc={} ps=[13~1,19~1]] add_log_entry 53'31 (52'29) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:30.630519+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.630+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'31 (0'0,53'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=52'29 crt=52'29 lcod 52'29 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 31 2019-12-07T19:34:30.630+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'31 (0'0,53'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=52'29 crt=52'29 lcod 52'29 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.630+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.630+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'30, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c582a9200 seq 75 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# 1 keys = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_000000000000000E_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000053.00000000000000000030' <- 0000000053.00000000000000000030 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000053.00000000000000000031' <- 0000000053.00000000000000000031 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:30.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a9200 cost 8495 (1 ios * 4000 + 4495 bytes) 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a9200 onodes 0x558c575a2280,0x558c575a2a00 shared_blobs 2019-12-07T19:34:30.630+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2388) update 4#1:ecc72bdb:::smithi13140796-10:15# 2019-12-07T19:34:30.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:15# is 556 (412 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:30.631+0000 7f8f05183700 20 bluestore.extentmap(0x558c575a2b08) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:30.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 1121 (621 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:30.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a9200 allocated 0x[] released 0x[] 2019-12-07T19:34:30.631+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9200 prepare 2019-12-07T19:34:30.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a9200 2019-12-07T19:34:30.631+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9200 io_done 2019-12-07T19:34:30.631+0000 7f8f05183700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'31 (0'0,53'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=52'29 crt=52'29 lcod 52'29 active mbc={} ps=[13~1,19~1]] op_applied version 53'31 2019-12-07T19:34:30.631+0000 7f8f05183700 10 osd.7 53 dequeue_op 0x558c5740cc60 finish 2019-12-07T19:34:30.631+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=252, reqid=osd.1.0:249, at_version=0'0, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=252, reqid=osd.1.0:249, at_version=0'0, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e53) pg 0x558c5500a000 2019-12-07T19:34:30.631+0000 7f8f04181700 10 osd.7 53 dequeue_op 0x558c581ccc60 prio 127 cost 0 latency 0.000496 MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=252, reqid=osd.1.0:249, at_version=0'0, trim_to=0'0, roll_forward_to=53'38)) v2 pg pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.631+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.9s4 53/41 ECSubWrite(tid=252, reqid=osd.1.0:249, at_version=0'0, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:30.631+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=52'34 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'38], crt=52'34) [] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'35 (52'33) delete 1:9694a0e5:::smithi13140796-3:13 by unknown.0.0:0 2019-12-07T19:34:30.622942+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'36 (52'34) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:30.622942+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'37 (45'31) delete 1:9b8adabe:::smithi13140796-6:11 by unknown.0.0:0 2019-12-07T19:34:30.623777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'38 (45'32) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:30.623777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.631+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 38 2019-12-07T19:34:30.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.631+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.631+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c5511db00 seq 75 2019-12-07T19:34:30.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13#23 onode 0x558c54e45680 txc 0x558c5511db00 2019-12-07T19:34:30.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13#23 0x0 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57818460) put_ref 0x0~2000 Blob(0x558c57818460 blob([0x282000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c578f0bd0) put_ref 0x0~2000 Blob(0x558c578f0bd0 blob([0x284000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57f51a40) put_ref 0x0~2000 Blob(0x558c57f51a40 blob([0x286000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c578f0b60) put_ref 0x0~2000 Blob(0x558c578f0b60 blob([0x288000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57818310) put_ref 0x0~2000 Blob(0x558c57818310 blob([0x28a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57819500) put_ref 0x0~2000 Blob(0x558c57819500 blob([0x28c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57f50d90) put_ref 0x0~2000 Blob(0x558c57f50d90 blob([0x28e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57818070) put_ref 0x0~2000 Blob(0x558c57818070 blob([0x290000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57818000) put_ref 0x0~2000 Blob(0x558c57818000 blob([0x292000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57819490) put_ref 0x0~2000 Blob(0x558c57819490 blob([0x294000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c578f16c0) put_ref 0x0~2000 Blob(0x558c578f16c0 blob([0x296000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c578181c0) put_ref 0x0~2000 Blob(0x558c578181c0 blob([0x298000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c578180e0) put_ref 0x0~2000 Blob(0x558c578180e0 blob([0x29a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57819420) put_ref 0x0~2000 Blob(0x558c57819420 blob([0x29c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57db58f0) put_ref 0x0~2000 Blob(0x558c57db58f0 blob([0x29e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57db5880) put_ref 0x0~2000 Blob(0x558c57db5880 blob([0x2a0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57db5810) put_ref 0x0~2000 Blob(0x558c57db5810 blob([0x2a2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57db57a0) put_ref 0x0~2000 Blob(0x558c57db57a0 blob([0x2a4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.extentmap(0x558c54e45788) dirty_range mark inline shard dirty 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57818460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x282000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c578f0bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x284000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57f51a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x286000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c578f0b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x288000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57818310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x28a000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57819500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x28c000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57f50d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x28e000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57818070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x290000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57818000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x292000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57819490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x294000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c578f16c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x296000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c578181c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x298000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c578180e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x29a000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57819420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x29c000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57db58f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x29e000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c57db5880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a0000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57db5810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a2000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57db57a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a4000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf88c0,0x558c54cf8a10,0x558c57314ee0,0x558c57314fc0,0x558c573150a0,0x558c57315180,0x558c57315260,0x558c57315340,0x558c57315420,0x558c57315500,0x558c573155e0,0x558c573156c0,0x558c573157a0,0x558c57315880,0x558c573159d0,0x558c57315ab0,0x558c574fa230,0x558c57567570 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9694a0e5:::smithi13140796-3:13#23 = 0 2019-12-07T19:34:30.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11#25 onode 0x558c57e8f180 txc 0x558c5511db00 2019-12-07T19:34:30.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11#25 0x0 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57f67880) put_ref 0x0~2000 Blob(0x558c57f67880 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c5809ae70) put_ref 0x0~2000 Blob(0x558c5809ae70 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57314000) put_ref 0x0~2000 Blob(0x558c57314000 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57f678f0) put_ref 0x0~2000 Blob(0x558c57f678f0 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57f67960) put_ref 0x0~2000 Blob(0x558c57f67960 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57f67a40) put_ref 0x0~2000 Blob(0x558c57f67a40 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2e70) put_ref 0x0~2000 Blob(0x558c579b2e70 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2ee0) put_ref 0x0~2000 Blob(0x558c579b2ee0 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579700e0) put_ref 0x0~2000 Blob(0x558c579700e0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2150) put_ref 0x0~2000 Blob(0x558c579b2150 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2d90) put_ref 0x0~2000 Blob(0x558c579b2d90 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2e00) put_ref 0x0~2000 Blob(0x558c579b2e00 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57970070) put_ref 0x0~2000 Blob(0x558c57970070 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c576adc70) put_ref 0x0~2000 Blob(0x558c576adc70 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2cb0) put_ref 0x0~2000 Blob(0x558c579b2cb0 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2d20) put_ref 0x0~2000 Blob(0x558c579b2d20 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57970000) put_ref 0x0~2000 Blob(0x558c57970000 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c576adc00) put_ref 0x0~2000 Blob(0x558c576adc00 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2bd0) put_ref 0x0~2000 Blob(0x558c579b2bd0 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2c40) put_ref 0x0~2000 Blob(0x558c579b2c40 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f80) put_ref 0x0~2000 Blob(0x558c579b3f80 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57818bd0) put_ref 0x0~2000 Blob(0x558c57818bd0 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2af0) put_ref 0x0~2000 Blob(0x558c579b2af0 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2b60) put_ref 0x0~2000 Blob(0x558c579b2b60 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f10) put_ref 0x0~2000 Blob(0x558c579b3f10 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b21c0) put_ref 0x0~2000 Blob(0x558c579b21c0 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a10) put_ref 0x0~2000 Blob(0x558c579b2a10 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a80) put_ref 0x0~2000 Blob(0x558c579b2a80 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b3ea0) put_ref 0x0~2000 Blob(0x558c579b3ea0 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c576add50) put_ref 0x0~2000 Blob(0x558c576add50 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b2930) put_ref 0x0~2000 Blob(0x558c579b2930 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b29a0) put_ref 0x0~2000 Blob(0x558c579b29a0 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b3e30) put_ref 0x0~2000 Blob(0x558c579b3e30 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c576adce0) put_ref 0x0~2000 Blob(0x558c576adce0 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c574fdc00) put_ref 0x0~2000 Blob(0x558c574fdc00 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c574fdb90) put_ref 0x0~2000 Blob(0x558c574fdb90 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c579b3dc0) put_ref 0x0~2000 Blob(0x558c579b3dc0 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c57970f50) put_ref 0x0~2000 Blob(0x558c57970f50 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.blob(0x558c578f0af0) put_ref 0x0~1000 Blob(0x558c578f0af0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore.extentmap(0x558c57e8f288) dirty_range mark inline shard dirty 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57f67880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e3000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c5809ae70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e5000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57314000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e7000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57f678f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2e9000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57f67960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2eb000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57f67a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ed000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c579b2e70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ef000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c579b2ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f1000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c579700e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f3000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c579b2150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f5000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c579b2d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f7000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c579b2e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2f9000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57970070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fb000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c576adc70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2fd000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 173 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512c00 con 0x558c54c19800 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c579b2cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ff000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c581cc9a0 prio 196 cost 0 latency 0.000015 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c579b2d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x301000~2000] 2019-12-07T19:34:30.631+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57970000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x303000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c576adc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x305000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c579b2bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x307000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c579b2c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x489000~2000] 2019-12-07T19:34:30.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c579b3f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x48b000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c57818bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x48d000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c579b2af0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x48f000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c579b2b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c581cc9a0 prio 196 cost 0 latency 0.000045 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x491000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c579b3f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x493000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c579b21c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x495000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c581cc9a0 finish 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c579b2a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x497000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~2000: 0x0~2000 Blob(0x558c579b2a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x499000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x38000~2000: 0x0~2000 Blob(0x558c579b3ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x49b000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3a000~2000: 0x0~2000 Blob(0x558c576add50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x49d000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3c000~2000: 0x0~2000 Blob(0x558c579b2930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x49f000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3e000~2000: 0x0~2000 Blob(0x558c579b29a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4a1000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x40000~2000: 0x0~2000 Blob(0x558c579b3e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4a3000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x42000~2000: 0x0~2000 Blob(0x558c576adce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4a5000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x44000~2000: 0x0~2000 Blob(0x558c574fdc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4a7000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x46000~2000: 0x0~2000 Blob(0x558c574fdb90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4a9000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x48000~2000: 0x0~2000 Blob(0x558c579b3dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4ab000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4a000~2000: 0x0~2000 Blob(0x558c57970f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4ad000~2000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~1000: 0x0~1000 Blob(0x558c578f0af0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x4af000~1000] 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54e1c000,0x558c54e1c0e0,0x558c54e1c4d0,0x558c54e1c5b0,0x558c54e1c690,0x558c54e1c770,0x558c54e1c850,0x558c54e1c930,0x558c54e1ca10,0x558c54e1caf0,0x558c54e1cbd0,0x558c54e1ccb0,0x558c54e1cd90,0x558c54e1ce70,0x558c54e1cf50,0x558c54e1d030,0x558c54e1d110,0x558c574e0e00,0x558c574e1ce0,0x558c57880f50,0x558c57881180,0x558c57881340,0x558c57881420,0x558c578de3f0,0x558c578de4d0,0x558c578de690,0x558c578de930,0x558c578deaf0,0x558c578ded20,0x558c578dee00,0x558c578deee0,0x558c578defc0,0x558c578df0a0,0x558c578df180,0x558c578f1730,0x558c578f17a0,0x558c57bea070,0x558c57dcac40,0x558c57dcb6c0 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:11#25 = 0 2019-12-07T19:34:30.631+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5511db00 cost 5585 (1 ios * 4000 + 1585 bytes) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5511db00 onodes shared_blobs 0x558c54cf88c0,0x558c54cf8a10,0x558c54e1c000,0x558c54e1c0e0,0x558c54e1c4d0,0x558c54e1c5b0,0x558c54e1c690,0x558c54e1c770,0x558c54e1c850,0x558c54e1c930,0x558c54e1ca10,0x558c54e1caf0,0x558c54e1cbd0,0x558c54e1ccb0,0x558c54e1cd90,0x558c54e1ce70,0x558c54e1cf50,0x558c54e1d030,0x558c54e1d110,0x558c57314ee0,0x558c57314fc0,0x558c573150a0,0x558c57315180,0x558c57315260,0x558c57315340,0x558c57315420,0x558c57315500,0x558c573155e0,0x558c573156c0,0x558c573157a0,0x558c57315880,0x558c573159d0,0x558c57315ab0,0x558c574e0e00,0x558c574e1ce0,0x558c574fa230,0x558c57567570,0x558c57880f50,0x558c57881180,0x558c57881340,0x558c57881420,0x558c578de3f0,0x558c578de4d0,0x558c578de690,0x558c578de930,0x558c578deaf0,0x558c578ded20,0x558c578dee00,0x558c578deee0,0x558c578defc0,0x558c578df0a0,0x558c578df180,0x558c578f1730,0x558c578f17a0,0x558c57bea070,0x558c57dcac40,0x558c57dcb6c0 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e4 is 11 SharedBlob(0x558c54cf88c0 loaded (sbid 0x28e4 ref_map(0x282000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e5 is 11 SharedBlob(0x558c54cf8a10 loaded (sbid 0x28e5 ref_map(0x284000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ba is 11 SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bb is 11 SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c7 is 11 SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c6 is 11 SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c5 is 11 SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c4 is 11 SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bd is 11 SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bc is 11 SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c8 is 11 SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c9 is 11 SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ca is 11 SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cb is 11 SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cc is 11 SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cd is 11 SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ce is 11 SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cf is 11 SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d0 is 11 SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f3 is 11 SharedBlob(0x558c57314ee0 loaded (sbid 0x28f3 ref_map(0x2a0000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f2 is 11 SharedBlob(0x558c57314fc0 loaded (sbid 0x28f2 ref_map(0x29e000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f1 is 11 SharedBlob(0x558c573150a0 loaded (sbid 0x28f1 ref_map(0x29c000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f0 is 11 SharedBlob(0x558c57315180 loaded (sbid 0x28f0 ref_map(0x29a000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ef is 11 SharedBlob(0x558c57315260 loaded (sbid 0x28ef ref_map(0x298000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ee is 11 SharedBlob(0x558c57315340 loaded (sbid 0x28ee ref_map(0x296000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ed is 11 SharedBlob(0x558c57315420 loaded (sbid 0x28ed ref_map(0x294000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ec is 11 SharedBlob(0x558c57315500 loaded (sbid 0x28ec ref_map(0x292000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e9 is 11 SharedBlob(0x558c573155e0 loaded (sbid 0x28e9 ref_map(0x28c000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e8 is 11 SharedBlob(0x558c573156c0 loaded (sbid 0x28e8 ref_map(0x28a000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e7 is 11 SharedBlob(0x558c573157a0 loaded (sbid 0x28e7 ref_map(0x288000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e6 is 11 SharedBlob(0x558c57315880 loaded (sbid 0x28e6 ref_map(0x286000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f4 is 11 SharedBlob(0x558c573159d0 loaded (sbid 0x28f4 ref_map(0x2a2000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28f5 is 11 SharedBlob(0x558c57315ab0 loaded (sbid 0x28f5 ref_map(0x2a4000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c1 is 11 SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c0 is 11 SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28eb is 11 SharedBlob(0x558c574fa230 loaded (sbid 0x28eb ref_map(0x290000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ea is 11 SharedBlob(0x558c57567570 loaded (sbid 0x28ea ref_map(0x28e000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b0 is 11 SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ad is 11 SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28af is 11 SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ae is 11 SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b1 is 11 SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b3 is 11 SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b2 is 11 SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b5 is 11 SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b4 is 11 SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c2 is 11 SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c3 is 11 SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b6 is 11 SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b7 is 11 SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b8 is 11 SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b9 is 11 SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bf is 11 SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28be is 11 SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ac is 11 SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ab is 11 SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28aa is 11 SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=1))) 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5511db00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511db00 prepare 2019-12-07T19:34:30.631+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5511db00 2019-12-07T19:34:30.631+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511db00 io_done 2019-12-07T19:34:30.631+0000 7f8f04181700 10 osd.7 53 dequeue_op 0x558c581ccc60 finish 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1898 < min_flush_size 524288 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a1b80 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f900 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6474 < min_flush_size 524288 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2280 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9115 < min_flush_size 524288 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45680 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8f180 had 1 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9138 < min_flush_size 524288 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9138 < min_flush_size 524288 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f282e~23b2 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f282e 0x2f282e~23b2 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c282e 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x82e 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbe0 and padding block with 0x420 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f4be0 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000162919s (0.000000092s flush + 0.000162827s kv commit) 2019-12-07T19:34:30.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582af200,0x558c582a9200,0x558c5511db00> 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af200 kv_submitted 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582af200 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af200 finishing 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582af200 onodes 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582af200 done 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582af200 [] 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.631+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9200 kv_submitted 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a9200 2019-12-07T19:34:30.631+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 53/28 ECSubWriteReply(tid=142, last_complete=53'65, committed=1, applied=1)) v2 -- 0x558c58512600 con 0x558c54c5bc00 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9200 finishing 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a9200 onodes 0x558c575a2280,0x558c575a2a00 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a9200 done 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a9200 [] 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511db00 kv_submitted 2019-12-07T19:34:30.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5511db00 2019-12-07T19:34:30.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5511db00 finishing 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5511db00 onodes 2019-12-07T19:34:30.631+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 53/41 ECSubWriteReply(tid=251, last_complete=53'31, committed=1, applied=1)) v2 -- 0x558c58513600 con 0x558c54c64000 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5511db00 done 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5511db00 [] 2019-12-07T19:34:30.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.631+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.631+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 53/41 ECSubWriteReply(tid=252, last_complete=53'38, committed=1, applied=1)) v2 -- 0x558c57950000 con 0x558c54c64000 2019-12-07T19:34:30.631+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 426 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c54c64000 2019-12-07T19:34:30.632+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5740c9a0 prio 196 cost 0 latency 0.000016 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.632+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 427 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c55053500 con 0x558c54c64000 2019-12-07T19:34:30.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.632+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5740c9a0 prio 196 cost 0 latency 0.000056 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.632+0000 7f8f24457700 15 osd.7 53 enqueue_peering_evt 1.9s4 epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:30.632+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.632+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5740c9a0 finish 2019-12-07T19:34:30.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e53) pg 0x558c5500a000 2019-12-07T19:34:30.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 428 ==== MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=253, reqid=osd.1.0:253, at_version=53'33, trim_to=0'0, roll_forward_to=52'29)) v2 ==== 2656+0+0 (crc 0 0 0) 0x558c5831c800 con 0x558c54c64000 2019-12-07T19:34:30.632+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:30.632+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c5740d340 prio 127 cost 0 latency 0.000026 epoch 53 MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=253, reqid=osd.1.0:253, at_version=53'33, trim_to=0'0, roll_forward_to=52'29)) v2 2019-12-07T19:34:30.632+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=253, reqid=osd.1.0:253, at_version=53'33, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.632+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.632+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:30.632+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:30.632+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57956000 seq 76 2019-12-07T19:34:30.632+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:30.632+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:30.632+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:30.632+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:30.632+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57956000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.632+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57956000 onodes shared_blobs 2019-12-07T19:34:30.632+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57956000 allocated 0x[] released 0x[] 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=253, reqid=osd.1.0:253, at_version=53'33, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.632+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956000 prepare 2019-12-07T19:34:30.632+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57956000 2019-12-07T19:34:30.632+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956000 io_done 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=253, reqid=osd.1.0:253, at_version=53'33, trim_to=0'0, roll_forward_to=52'29)) v2) prio 127 cost 0 e53) pg 0x558c54d5f000 2019-12-07T19:34:30.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 53 dequeue_op 0x558c5740d340 prio 127 cost 0 latency 0.000097 MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=253, reqid=osd.1.0:253, at_version=53'33, trim_to=0'0, roll_forward_to=52'29)) v2 pg pg[1.7s4( v 53'31 (0'0,53'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'31 (0'0,53'31] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=253, reqid=osd.1.0:253, at_version=53'33, trim_to=0'0, roll_forward_to=52'29)) v2 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 snap_mapper.remove_oid 1:ec1285ab:::smithi13140796-19:10 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 snap_mapper._remove_oid 1:ec1285ab:::smithi13140796-19:10 2019-12-07T19:34:30.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ec1285ab:::smithi13140796-19:10 e 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 snap_mapper.clear_snaps 1:ec1285ab:::smithi13140796-19:10 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f4be0~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f4be0 0x2f4be0~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'31 (0'0,53'31] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'31], crt=52'29) [53'32 (45'22) delete 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:30.633192+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'33 (45'23) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:30.633192+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c4be0 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbe0 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5d and padding block with 0xfa3 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'32 (0'0,53'32] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'31 crt=52'29 lcod 53'31 active mbc={} ps=[13~1,19~1]] add_log_entry 53'32 (45'22) delete 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:30.633192+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'31 crt=52'29 lcod 53'31 active mbc={} ps=[13~1,19~1]] add_log_entry 53'33 (45'23) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:30.633192+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'31 crt=52'29 lcod 53'31 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 33 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'31 crt=52'29 lcod 53'31 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.632+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'32, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57958000 seq 76 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ec1285ab:::smithi13140796-19:10#20 key 0x848000000000000001ec1285ab21736d'ithi13140796-19!='0x00000000000000100000000000000020'o' 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# -> 4#1:ec1285ab:::smithi13140796-19:10#20 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10# -> 4#1:ec1285ab:::smithi13140796-19:10#20 = 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.632+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 248 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512600 con 0x558c54c5bc00 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.632+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000012 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.632+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- OBJ_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.4_0000000000000001.73841A5D.10.smithi13140796-19..' <- SNA_1_000000000000000E_.4_0000000000000001.73841A5D.10.smithi13140796-19.. 2019-12-07T19:34:30.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:30.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000053.00000000000000000032' <- 0000000053.00000000000000000032 2019-12-07T19:34:30.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000053.00000000000000000033' <- 0000000053.00000000000000000033 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f505d 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57958000 cost 7533 (1 ios * 4000 + 3533 bytes) 2019-12-07T19:34:30.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57958000 onodes 0x558c54e44000,0x558c574a0500 shared_blobs 2019-12-07T19:34:30.632+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c578d2c60 prio 196 cost 0 latency 0.000041 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:30.632+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1318204 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1498 (485 bytes onode + 2 bytes spanning blobs + 1011 bytes inline extents) 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore.extentmap(0x558c574a0608) update 4#1:ec1285ab:::smithi13140796-19:10#20 2019-12-07T19:34:30.632+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=42, last_complete=53'38, committed=1, applied=1)) v2 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000240155s (0.000000091s flush + 0.000240064s kv commit) 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:10#20 is 688 (412 bytes onode + 2 bytes spanning blobs + 274 bytes inline extents) 2019-12-07T19:34:30.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57958000 allocated 0x[] released 0x[] 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57956000> 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956000 kv_submitted 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958000 prepare 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57956000 2019-12-07T19:34:30.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57958000 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958000 io_done 2019-12-07T19:34:30.632+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:e v=53'38 tt=0'0 tid=42 reqid=osd.7.0:42 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:30.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956000 finishing 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57956000 onodes 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57956000 done 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:30.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57956000 [] 2019-12-07T19:34:30.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.632+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=52'36 crt=52'36 lcod 51'34 mlcod 51'34 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] repop_all_committed: repop tid 42 all committed 2019-12-07T19:34:30.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'31 crt=52'29 lcod 53'31 active mbc={} ps=[13~1,19~1]] op_applied version 53'33 2019-12-07T19:34:30.633+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] eval_repop repgather(0x558c576b6360 53'38 rep_tid=42 committed?=1 r=0) (no op) 2019-12-07T19:34:30.633+0000 7f8f0117b700 10 osd.7 53 dequeue_op 0x558c5740d340 finish 2019-12-07T19:34:30.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] commit: repgather(0x558c576b6360 53'38 rep_tid=42 committed?=1 r=0) 2019-12-07T19:34:30.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.633+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:173 2019-12-07T19:34:30.633+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] removing repgather(0x558c576b6360 53'38 rep_tid=42 committed?=1 r=0) 2019-12-07T19:34:30.633+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] q front is repgather(0x558c576b6360 53'38 rep_tid=42 committed?=1 r=0) 2019-12-07T19:34:30.633+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] remove_repop repgather(0x558c576b6360 53'38 rep_tid=42 committed?=1 r=0) 2019-12-07T19:34:30.633+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4633 < min_flush_size 524288 2019-12-07T19:34:30.633+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:e v=53'38 tt=0'0 tid=42 reqid=osd.7.0:42 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:e=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0500 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795a000 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4656 < min_flush_size 524288 2019-12-07T19:34:30.633+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4656 < min_flush_size 524288 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f505d~1230 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f505d 0x2f505d~1230 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c505d 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5d 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.633+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x28d and padding block with 0xd73 2019-12-07T19:34:30.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 249 ==== MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=143, reqid=osd.3.0:143, at_version=53'67, trim_to=0'0, roll_forward_to=53'65)) v2 ==== 2656+0+0 (crc 0 0 0) 0x558c579e2000 con 0x558c54c5bc00 2019-12-07T19:34:30.633+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c578d31e0 prio 127 cost 0 latency 0.000026 epoch 53 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=143, reqid=osd.3.0:143, at_version=53'67, trim_to=0'0, roll_forward_to=53'65)) v2 2019-12-07T19:34:30.633+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=143, reqid=osd.3.0:143, at_version=53'67, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=143, reqid=osd.3.0:143, at_version=53'67, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=143, reqid=osd.3.0:143, at_version=53'67, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c578d31e0 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=143, reqid=osd.3.0:143, at_version=53'67, trim_to=0'0, roll_forward_to=53'65)) v2 pg pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=143, reqid=osd.3.0:143, at_version=53'67, trim_to=0'0, roll_forward_to=53'65)) v2 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 snap_mapper.remove_oid 1:6ee4dfe7:::smithi13140796-47:e 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 snap_mapper._remove_oid 1:6ee4dfe7:::smithi13140796-47:e 2019-12-07T19:34:30.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6ee4dfe7:::smithi13140796-47:e e 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 snap_mapper.clear_snaps 1:6ee4dfe7:::smithi13140796-47:e 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'65 (0'0,53'65] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'65], crt=53'65) [53'66 (45'49) delete 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:30.633777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'67 (45'50) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:30.633777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'66 (0'0,53'66] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'65 crt=53'65 lcod 53'65 active mbc={} ps=[13~1,19~1]] add_log_entry 53'66 (45'49) delete 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:30.633777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'67 (0'0,53'67] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'65 crt=53'65 lcod 53'65 active mbc={} ps=[13~1,19~1]] add_log_entry 53'67 (45'50) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:30.633777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'67 (0'0,53'67] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'65 crt=53'65 lcod 53'65 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 67 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'67 (0'0,53'67] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'65 crt=53'65 lcod 53'65 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.633+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'66, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582af500 seq 101 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6ee4dfe7:::smithi13140796-47:e#42 key 0x8280000000000000016ee4dfe721736d'ithi13140796-47!='0x000000000000000e0000000000000042'o' 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# -> 2#1:6ee4dfe7:::smithi13140796-47:e#42 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e# -> 2#1:6ee4dfe7:::smithi13140796-47:e#42 = 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- OBJ_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6772BF7E.e.smithi13140796-47..' <- SNA_1_000000000000000E_.2_0000000000000001.6772BF7E.e.smithi13140796-47.. 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000066' <- 0000000053.00000000000000000066 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000067' <- 0000000053.00000000000000000067 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582af500 cost 7531 (1 ios * 4000 + 3531 bytes) 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582af500 onodes 0x558c54e12280,0x558c54e12780 shared_blobs 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) update 2#1:6ee4dfe7:::smithi13140796-47:e#42 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:e#42 is 1069 (412 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12888) update 2#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6ee4dfe7:::smithi13140796-47:head# is 1142 (485 bytes onode + 2 bytes spanning blobs + 655 bytes inline extents) 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582af500 allocated 0x[] released 0x[] 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af500 prepare 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582af500 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af500 io_done 2019-12-07T19:34:30.633+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'67 (0'0,53'67] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'65 crt=53'65 lcod 53'65 active mbc={} ps=[13~1,19~1]] op_applied version 53'67 2019-12-07T19:34:30.633+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c578d31e0 finish 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f628d 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000498586s (0.000000079s flush + 0.000498507s kv commit) 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57958000> 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958000 kv_submitted 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57958000 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958000 finishing 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57958000 onodes 0x558c54e44000,0x558c574a0500 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4656 < min_flush_size 524288 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57958000 done 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57958000 [] 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12280 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12780 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8ec80 had 1 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4679 < min_flush_size 524288 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4679 < min_flush_size 524288 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f628d~1247 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f628d 0x2f628d~1247 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c628d 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x28d 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4d4 and padding block with 0xb2c 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f74d4 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000161681s (0.000000099s flush + 0.000161582s kv commit) 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582af500> 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af500 kv_submitted 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582af500 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af500 finishing 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582af500 onodes 0x558c54e12280,0x558c54e12780 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582af500 done 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582af500 [] 2019-12-07T19:34:30.633+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 53/41 ECSubWriteReply(tid=253, last_complete=53'33, committed=1, applied=1)) v2 -- 0x558c58513800 con 0x558c54c64000 2019-12-07T19:34:30.633+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.633+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 53/28 ECSubWriteReply(tid=143, last_complete=53'67, committed=1, applied=1)) v2 -- 0x558c58512800 con 0x558c54c5bc00 2019-12-07T19:34:30.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.633+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.634+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 250 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512800 con 0x558c54c5bc00 2019-12-07T19:34:30.634+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c5511fe40 prio 196 cost 0 latency 0.000016 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.634+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.634+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 251 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c54c5bc00 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.634+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c5511fb80 prio 196 cost 0 latency 0.000009 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.634+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511fe40 prio 196 cost 0 latency 0.000042 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=43, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:19 v=53'40 tt=0'0 tid=43 reqid=osd.7.0:43 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c5511fb80 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'38 crt=52'36 lcod 52'36 mlcod 52'36 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] repop_all_committed: repop tid 43 all committed 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] eval_repop repgather(0x558c576b6fc0 53'40 rep_tid=43 committed?=1 r=0) (no op) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=44, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] commit: repgather(0x558c576b6fc0 53'40 rep_tid=43 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] handle_sub_write_reply Calling on_all_commit on Op(1:276d5164:::smithi13140796-22:e v=53'10 tt=0'0 tid=44 reqid=osd.7.0:44 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:30.634+0000 7f8f0297e700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:174 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 luod=45'8 crt=45'8 lcod 42'6 mlcod 42'6 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] repop_all_committed: repop tid 44 all committed 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] removing repgather(0x558c576b6fc0 53'40 rep_tid=43 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] q front is repgather(0x558c576b6fc0 53'40 rep_tid=43 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] eval_repop repgather(0x558c576b6ea0 53'10 rep_tid=44 committed?=1 r=0) (no op) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] commit: repgather(0x558c576b6ea0 53'10 rep_tid=44 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] exit Trimming/WaitRepops 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.634+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:138 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] removing repgather(0x558c576b6ea0 53'10 rep_tid=44 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] q front is repgather(0x558c576b6ea0 53'10 rep_tid=44 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 53 queueing pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] for snaptrim 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] exit Trimming/WaitRepops 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=53) prio 5 cost 1048576 e53) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.634+0000 7f8f0297e700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:175 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] remove_repop repgather(0x558c576b6fc0 53'40 rep_tid=43 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=53'40 tt=0'0 tid=43 reqid=osd.7.0:43 roll_forward_to=52'36 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 53 queueing pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] for snaptrim 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=53) prio 5 cost 1048576 e53) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [13~1,19~1] 2019-12-07T19:34:30.634+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] publish_stats_to_osd 53:139 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=45 reqid=osd.7.0:43 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] remove_repop repgather(0x558c576b6ea0 53'10 rep_tid=44 committed?=1 r=0) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:e v=53'10 tt=0'0 tid=44 reqid=osd.7.0:44 roll_forward_to=45'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:276d5164:::smithi13140796-22:e=[],1:276d5164:::smithi13140796-22:head=[]}) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=45 reqid=osd.7.0:43 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: starting commit on Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=46 reqid=osd.7.0:44 roll_forward_to=53'10 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 53/28 ECSubWrite(tid=45, reqid=osd.7.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c57a44000 con 0x558c54c64000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: op: Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=46 reqid=osd.7.0:44 roll_forward_to=53'10 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:30.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 53/28 ECSubWrite(tid=45, reqid=osd.7.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c57a44800 con 0x558c54c19800 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:30.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 53/28 ECSubWrite(tid=45, reqid=osd.7.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c57a43000 con 0x558c54c5bc00 2019-12-07T19:34:30.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 53/28 ECSubWrite(tid=45, reqid=osd.7.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c58154000 con 0x558c54c5b400 2019-12-07T19:34:30.634+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 53/28 ECSubWrite(tid=45, reqid=osd.7.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c58157000 con 0x558c54c19c00 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=52'36 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log log((0'0,53'40], crt=52'36) [] 2019-12-07T19:34:30.634+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.4s2 53/41 ECSubWrite(tid=46, reqid=osd.7.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=53'10)) v2 -- 0x558c5852d800 con 0x558c54c64000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] rollforward: entry=53'37 (45'29) delete 1:3a5a3f5b:::smithi13140796-23:e by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.634+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.4s4 53/41 ECSubWrite(tid=46, reqid=osd.7.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=53'10)) v2 -- 0x558c58532000 con 0x558c54c19800 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] rollforward: entry=53'38 (45'30) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:30.623412+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.634+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.4s1 53/41 ECSubWrite(tid=46, reqid=osd.7.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=53'10)) v2 -- 0x558c58532800 con 0x558c54c5bc00 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] rollforward: entry=53'39 (52'35) modify 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.634+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.4s5 53/41 ECSubWrite(tid=46, reqid=osd.7.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=53'10)) v2 -- 0x558c58533000 con 0x558c54c5b400 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] rollforward: entry=53'40 (52'36) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:30.624359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log approx pg log length = 40 2019-12-07T19:34:30.634+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.4s3 53/41 ECSubWrite(tid=46, reqid=osd.7.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=53'10)) v2 -- 0x558c58533800 con 0x558c54c19c00 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.634+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=45'8 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log log((0'0,53'10], crt=45'8) [] 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c578ab800 seq 54 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] rollforward: entry=53'9 (45'7) delete 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e#25 onode 0x558c57f35b80 txc 0x558c578ab800 2019-12-07T19:34:30.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e#25 0x0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4a10) put_ref 0x0~2000 Blob(0x558c57ff4a10 blob([0x339000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff49a0) put_ref 0x0~2000 Blob(0x558c57ff49a0 blob([0x33b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff48c0) put_ref 0x0~2000 Blob(0x558c57ff48c0 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4930) put_ref 0x0~2000 Blob(0x558c57ff4930 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] rollforward: entry=53'10 (45'8) modify 1:276d5164:::smithi13140796-22:head by unknown.0.0:0 2019-12-07T19:34:30.625953+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4a80) put_ref 0x0~2000 Blob(0x558c57ff4a80 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4af0) put_ref 0x0~2000 Blob(0x558c57ff4af0 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4bd0) put_ref 0x0~2000 Blob(0x558c57ff4bd0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4c40) put_ref 0x0~2000 Blob(0x558c57ff4c40 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log approx pg log length = 10 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4cb0) put_ref 0x0~2000 Blob(0x558c57ff4cb0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4d20) put_ref 0x0~2000 Blob(0x558c57ff4d20 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4d90) put_ref 0x0~2000 Blob(0x558c57ff4d90 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff50a0) put_ref 0x0~1000 Blob(0x558c57ff50a0 blob([0x34f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881ce0) put_ref 0x0~2000 Blob(0x558c57881ce0 blob([0x377000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881c70) put_ref 0x0~2000 Blob(0x558c57881c70 blob([0x379000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881c00) put_ref 0x0~2000 Blob(0x558c57881c00 blob([0x37b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881b90) put_ref 0x0~2000 Blob(0x558c57881b90 blob([0x37d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881b20) put_ref 0x0~2000 Blob(0x558c57881b20 blob([0x37f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881ab0) put_ref 0x0~2000 Blob(0x558c57881ab0 blob([0x381000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881a40) put_ref 0x0~2000 Blob(0x558c57881a40 blob([0x383000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578819d0) put_ref 0x0~2000 Blob(0x558c578819d0 blob([0x385000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881960) put_ref 0x0~2000 Blob(0x558c57881960 blob([0x387000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578818f0) put_ref 0x0~2000 Blob(0x558c578818f0 blob([0x389000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880a80) put_ref 0x0~2000 Blob(0x558c57880a80 blob([0x38b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880a10) put_ref 0x0~2000 Blob(0x558c57880a10 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c582ad500 seq 43 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881ea0) put_ref 0x0~2000 Blob(0x558c57881ea0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881f10) put_ref 0x0~2000 Blob(0x558c57881f10 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880d90) put_ref 0x0~2000 Blob(0x558c57880d90 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57880cb0) put_ref 0x0~2000 Blob(0x558c57880cb0 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.4s0_head 0#1:276d5164:::smithi13140796-22:e#9 onode 0x558c58093400 txc 0x558c582ad500 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881e30) put_ref 0x0~2000 Blob(0x558c57881e30 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.4s0_head 0#1:276d5164:::smithi13140796-22:e#9 0x0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881dc0) put_ref 0x0~2000 Blob(0x558c57881dc0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57881d50) put_ref 0x0~2000 Blob(0x558c57881d50 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1ea0) put_ref 0x0~2000 Blob(0x558c578f1ea0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a4ee0) put_ref 0x0~2000 Blob(0x558c580a4ee0 blob([0x97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1f80) put_ref 0x0~2000 Blob(0x558c578f1f80 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1e30) put_ref 0x0~2000 Blob(0x558c578f1e30 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1dc0) put_ref 0x0~2000 Blob(0x558c578f1dc0 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a4e70) put_ref 0x0~2000 Blob(0x558c580a4e70 blob([0x99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1d50) put_ref 0x0~2000 Blob(0x558c578f1d50 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578dfdc0) put_ref 0x0~2000 Blob(0x558c578dfdc0 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a4540) put_ref 0x0~2000 Blob(0x558c580a4540 blob([0x9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578dfe30) put_ref 0x0~2000 Blob(0x558c578dfe30 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c578dff80) put_ref 0x0~2000 Blob(0x558c578dff80 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a4e00) put_ref 0x0~2000 Blob(0x558c580a4e00 blob([0x9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c577f36c0) put_ref 0x0~2000 Blob(0x558c577f36c0 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a4f50) put_ref 0x0~2000 Blob(0x558c580a4f50 blob([0x9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c575bdc70) put_ref 0x0~2000 Blob(0x558c575bdc70 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57dcad20) put_ref 0x0~2000 Blob(0x558c57dcad20 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a4fc0) put_ref 0x0~2000 Blob(0x558c580a4fc0 blob([0xa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c54d75810) put_ref 0x0~2000 Blob(0x558c54d75810 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5110) put_ref 0x0~2000 Blob(0x558c57ff5110 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a50a0) put_ref 0x0~2000 Blob(0x558c580a50a0 blob([0xa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5180) put_ref 0x0~2000 Blob(0x558c57ff5180 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff51f0) put_ref 0x1000~1000 Blob(0x558c57ff51f0 blob([!~1000,0x3b9000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a5110) put_ref 0x0~2000 Blob(0x558c580a5110 blob([0xa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) dirty_range mark shard 0x23000 dirty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57f35c88) dirty_range mark shard 0x47000 dirty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57ff4a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map(0x339000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a5180) put_ref 0x0~2000 Blob(0x558c580a5180 blob([0xa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x339000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x339000~2000] from SharedBlob(0x558c57396230 loaded (sbid 0x284a ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x339000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a51f0) put_ref 0x0~2000 Blob(0x558c580a51f0 blob([0xa9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57ff49a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map(0x33b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33b000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.blob(0x558c580a5260) put_ref 0x0~2000 Blob(0x558c580a5260 blob([0xab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x33b000~2000] from SharedBlob(0x558c5757e8c0 loaded (sbid 0x284b ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x33b000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore.extentmap(0x558c58093508) dirty_range mark inline shard dirty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57ff48c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map(0x33d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33d000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x33d000~2000] from SharedBlob(0x558c578f18f0 loaded (sbid 0x284c ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x33d000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c580a4ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map(0x97000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57ff4930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map(0x33f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x97000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x33f000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x33f000~2000] from SharedBlob(0x558c578f19d0 loaded (sbid 0x284d ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x33f000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x97000~2000] from SharedBlob(0x558c54e568c0 loaded (sbid 0x288b ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57ff4a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map(0x341000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x97000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x341000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x341000~2000] from SharedBlob(0x558c578f1ab0 loaded (sbid 0x284e ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x341000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57ff4af0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map(0x343000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c580a4e70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map(0x99000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x343000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x99000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x99000~2000] from SharedBlob(0x558c54e56b60 loaded (sbid 0x288c ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x343000~2000] from SharedBlob(0x558c578f1b90 loaded (sbid 0x284f ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x99000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x343000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57ff4bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map(0x345000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x345000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c580a4540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map(0x9b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x345000~2000] from SharedBlob(0x558c578f1c70 loaded (sbid 0x2850 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x345000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x9b000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57ff4c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map(0x347000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x347000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x9b000~2000] from SharedBlob(0x558c54e56a10 loaded (sbid 0x288d ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x347000~2000] from SharedBlob(0x558c54d75e30 loaded (sbid 0x2851 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x9b000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x347000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57ff4cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map(0x349000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x349000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c580a4e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map(0x9d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x349000~2000] from SharedBlob(0x558c574d2850 loaded (sbid 0x2852 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x9d000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x349000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x9d000~2000] from SharedBlob(0x558c54e56af0 loaded (sbid 0x288e ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57ff4d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map(0x34b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x9d000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34b000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x34b000~2000] from SharedBlob(0x558c5742bc00 loaded (sbid 0x2853 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x34b000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57ff4d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map(0x34d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34d000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x34d000~2000] from SharedBlob(0x558c575bd9d0 loaded (sbid 0x2854 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c580a4f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map(0x9f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x34d000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x9f000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~1000: 0x0~1000 Blob(0x558c57ff50a0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map(0x34f000~1000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x34f000~1000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x9f000~2000] from SharedBlob(0x558c54e563f0 loaded (sbid 0x288f ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x34f000~1000] from SharedBlob(0x558c578dfea0 loaded (sbid 0x2855 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x9f000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x34f000~1000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x17000~2000: 0x0~2000 Blob(0x558c57881ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map(0x377000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c580a4fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map(0xa1000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x377000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa1000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x377000~2000] from SharedBlob(0x558c5794d420 loaded (sbid 0x2869 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x377000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xa1000~2000] from SharedBlob(0x558c54e42af0 loaded (sbid 0x2890 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xa1000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x19000~2000: 0x0~2000 Blob(0x558c57881c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map(0x379000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x379000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x379000~2000] from SharedBlob(0x558c5794d500 loaded (sbid 0x286a ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x379000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c580a50a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map(0xa3000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa3000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1b000~2000: 0x0~2000 Blob(0x558c57881c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map(0x37b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x37b000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xa3000~2000] from SharedBlob(0x558c54e43d50 loaded (sbid 0x2891 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x37b000~2000] from SharedBlob(0x558c5794d5e0 loaded (sbid 0x286b ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xa3000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x37b000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1d000~2000: 0x0~2000 Blob(0x558c57881b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map(0x37d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x37d000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c580a5110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map(0xa5000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x37d000~2000] from SharedBlob(0x558c5794d6c0 loaded (sbid 0x286c ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa5000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x37d000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xa5000~2000] from SharedBlob(0x558c54e42d20 loaded (sbid 0x2892 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1f000~2000: 0x0~2000 Blob(0x558c57881b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map(0x37f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x37f000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xa5000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x37f000~2000] from SharedBlob(0x558c5794d7a0 loaded (sbid 0x286d ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x37f000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x21000~2000: 0x0~2000 Blob(0x558c57881ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map(0x381000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c580a5180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map(0xa7000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x381000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa7000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x381000~2000] from SharedBlob(0x558c5794d880 loaded (sbid 0x286e ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xa7000~2000] from SharedBlob(0x558c54e56c40 loaded (sbid 0x2893 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x381000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xa7000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x23000~2000: 0x0~2000 Blob(0x558c57881a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map(0x383000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c580a51f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map(0xa9000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x383000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa9000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x383000~2000] from SharedBlob(0x558c5794d960 loaded (sbid 0x286f ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xa9000~2000] from SharedBlob(0x558c54e56d20 loaded (sbid 0x2894 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x383000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xa9000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x25000~2000: 0x0~2000 Blob(0x558c578819d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map(0x385000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c580a5260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map(0xab000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x385000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xab000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x385000~2000] from SharedBlob(0x558c5794da40 loaded (sbid 0x2870 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xab000~2000] from SharedBlob(0x558c54e56e00 loaded (sbid 0x2895 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x385000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xab000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x27000~2000: 0x0~2000 Blob(0x558c57881960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map(0x387000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x387000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x387000~2000] from SharedBlob(0x558c5794db20 loaded (sbid 0x2871 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x387000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x29000~2000: 0x0~2000 Blob(0x558c578818f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map(0x389000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x389000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x389000~2000] from SharedBlob(0x558c5794dc00 loaded (sbid 0x2872 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54e42af0,0x558c54e42d20,0x558c54e43d50,0x558c54e563f0,0x558c54e568c0,0x558c54e56a10,0x558c54e56af0,0x558c54e56b60,0x558c54e56c40,0x558c54e56d20,0x558c54e56e00 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x389000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2b000~2000: 0x0~2000 Blob(0x558c57880a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map(0x38b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x38b000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x38b000~2000] from SharedBlob(0x558c5794dce0 loaded (sbid 0x2873 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x38b000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.4s0_head 0#1:276d5164:::smithi13140796-22:e#9 = 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2d000~2000: 0x0~2000 Blob(0x558c57880a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map(0x38d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x38d000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x38d000~2000] from SharedBlob(0x558c5794ddc0 loaded (sbid 0x2874 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x38d000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2f000~2000: 0x0~2000 Blob(0x558c57881ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map(0x38f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x38f000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x38f000~2000] from SharedBlob(0x558c5794dea0 loaded (sbid 0x2875 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x38f000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x31000~2000: 0x0~2000 Blob(0x558c57881f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map(0x391000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x391000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x391000~2000] from SharedBlob(0x558c5794df80 loaded (sbid 0x2876 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x391000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x33000~2000: 0x0~2000 Blob(0x558c57880d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map(0x393000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x393000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x393000~2000] from SharedBlob(0x558c5792c070 loaded (sbid 0x2877 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x393000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x35000~2000: 0x0~2000 Blob(0x558c57880cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map(0x395000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x395000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x395000~2000] from SharedBlob(0x558c5792c150 loaded (sbid 0x2878 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x395000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ad500 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x37000~2000: 0x0~2000 Blob(0x558c57881e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map(0x397000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x397000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x397000~2000] from SharedBlob(0x558c5792c230 loaded (sbid 0x2879 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x397000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ad500 onodes shared_blobs 0x558c54e42af0,0x558c54e42d20,0x558c54e43d50,0x558c54e563f0,0x558c54e568c0,0x558c54e56a10,0x558c54e56af0,0x558c54e56b60,0x558c54e56c40,0x558c54e56d20,0x558c54e56e00 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x39000~2000: 0x0~2000 Blob(0x558c57881dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map(0x399000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x399000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2890 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x399000~2000] from SharedBlob(0x558c5792c310 loaded (sbid 0x287a ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x399000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2892 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3b000~2000: 0x0~2000 Blob(0x558c57881d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map(0x39b000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2891 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x39b000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x39b000~2000] from SharedBlob(0x558c5792c3f0 loaded (sbid 0x287b ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288f is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x39b000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288b is empty 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288d is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3d000~2000: 0x0~2000 Blob(0x558c578f1ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map(0x39d000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288e is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x39d000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288c is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x39d000~2000] from SharedBlob(0x558c5792c4d0 loaded (sbid 0x287c ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x39d000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2893 is empty 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2894 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3f000~2000: 0x0~2000 Blob(0x558c578f1f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map(0x39f000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x39f000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2895 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x39f000~2000] from SharedBlob(0x558c5792c5b0 loaded (sbid 0x287d ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x39f000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x41000~2000: 0x0~2000 Blob(0x558c578f1e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map(0x3a1000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ad500 allocated 0x[] released 0x[97000~16000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a1000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3a1000~2000] from SharedBlob(0x558c5792c690 loaded (sbid 0x287e ref_map())) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x97000~16000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3a1000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 10 freelist release 0x97000~16000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x43000~2000: 0x0~2000 Blob(0x558c578f1dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map(0x3a3000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a3000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3a3000~2000] from SharedBlob(0x558c546777a0 loaded (sbid 0x287f ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3a3000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x45000~2000: 0x0~2000 Blob(0x558c578f1d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map(0x3a5000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a5000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3a5000~2000] from SharedBlob(0x558c573967e0 loaded (sbid 0x2880 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3a5000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x47000~2000: 0x0~2000 Blob(0x558c578dfdc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map(0x3a7000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a7000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3a7000~2000] from SharedBlob(0x558c5792d180 loaded (sbid 0x2881 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3a7000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x49000~2000: 0x0~2000 Blob(0x558c578dfe30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map(0x3a9000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3a9000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3a9000~2000] from SharedBlob(0x558c5792d260 loaded (sbid 0x2882 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3a9000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad500 prepare 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4b000~2000: 0x0~2000 Blob(0x558c578dff80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map(0x3ab000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ad500 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3ab000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad500 io_done 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3ab000~2000] from SharedBlob(0x558c5792d340 loaded (sbid 0x2883 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3ab000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4d000~2000: 0x0~2000 Blob(0x558c577f36c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map(0x3ad000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3ad000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3ad000~2000] from SharedBlob(0x558c5792d420 loaded (sbid 0x2884 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3ad000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4f000~2000: 0x0~2000 Blob(0x558c575bdc70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map(0x3af000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3af000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3af000~2000] from SharedBlob(0x558c5792d500 loaded (sbid 0x2885 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3af000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c5511fb80 finish 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x51000~2000: 0x0~2000 Blob(0x558c57dcad20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map(0x3b1000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b1000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3b1000~2000] from SharedBlob(0x558c5792d5e0 loaded (sbid 0x2886 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3b1000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x53000~2000: 0x0~2000 Blob(0x558c54d75810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map(0x3b3000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b3000~2000] 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3b3000~2000] from SharedBlob(0x558c5792d6c0 loaded (sbid 0x2887 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3b3000~2000 2019-12-07T19:34:30.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=53) prio 5 cost 1048576 e53) queued 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x55000~2000: 0x0~2000 Blob(0x558c57ff5110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map(0x3b5000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b5000~2000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3b5000~2000] from SharedBlob(0x558c5792d7a0 loaded (sbid 0x2888 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3b5000~2000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x57000~2000: 0x0~2000 Blob(0x558c57ff5180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map(0x3b7000~2000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b7000~2000] 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3b7000~2000] from SharedBlob(0x558c5792d880 loaded (sbid 0x2889 ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3b7000~2000 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x59000~1000: 0x1000~1000 Blob(0x558c57ff51f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map(0x3b9000~1000=1)))) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3b9000~1000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x3b9000~1000] from SharedBlob(0x558c5792d960 loaded (sbid 0x288a ref_map())) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x3b9000~1000 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:30.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1382 < min_flush_size 524288 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x23000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x47000 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c546777a0,0x558c54d75e30,0x558c57396230,0x558c573967e0,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70,0x558c5792c070,0x558c5792c150,0x558c5792c230,0x558c5792c310,0x558c5792c3f0,0x558c5792c4d0,0x558c5792c5b0,0x558c5792c690,0x558c5792d180,0x558c5792d260,0x558c5792d340,0x558c5792d420,0x558c5792d500,0x558c5792d5e0,0x558c5792d6c0,0x558c5792d7a0,0x558c5792d880,0x558c5792d960,0x558c5794d420,0x558c5794d500,0x558c5794d5e0,0x558c5794d6c0,0x558c5794d7a0,0x558c5794d880,0x558c5794d960,0x558c5794da40,0x558c5794db20,0x558c5794dc00,0x558c5794dce0,0x558c5794ddc0,0x558c5794dea0,0x558c5794df80 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:e#25 = 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578ab800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578ab800 onodes shared_blobs 0x558c546777a0,0x558c54d75e30,0x558c57396230,0x558c573967e0,0x558c5742bc00,0x558c574d2850,0x558c5757e8c0,0x558c575bd9d0,0x558c578dfea0,0x558c578f18f0,0x558c578f19d0,0x558c578f1ab0,0x558c578f1b90,0x558c578f1c70,0x558c5792c070,0x558c5792c150,0x558c5792c230,0x558c5792c310,0x558c5792c3f0,0x558c5792c4d0,0x558c5792c5b0,0x558c5792c690,0x558c5792d180,0x558c5792d260,0x558c5792d340,0x558c5792d420,0x558c5792d500,0x558c5792d5e0,0x558c5792d6c0,0x558c5792d7a0,0x558c5792d880,0x558c5792d960,0x558c5794d420,0x558c5794d500,0x558c5794d5e0,0x558c5794d6c0,0x558c5794d7a0,0x558c5794d880,0x558c5794d960,0x558c5794da40,0x558c5794db20,0x558c5794dc00,0x558c5794dce0,0x558c5794ddc0,0x558c5794dea0,0x558c5794df80 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093400 had 1 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287f is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2851 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284a is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1405 < min_flush_size 524288 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2880 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2853 is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1405 < min_flush_size 524288 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2852 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284b is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2854 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2855 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284c is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284d is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284e is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x284f is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2850 is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2877 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2878 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2879 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287a is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287b is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287c is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287d is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x287e is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f74d4~57d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2881 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2882 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2883 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2884 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2885 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2886 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2887 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2888 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2889 is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f74d4 0x2f74d4~57d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x288a is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2869 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286a is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286b is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286c is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286d is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286e is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x286f is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c74d4 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2870 is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4d4 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2871 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2872 is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2873 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2874 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2875 is empty 2019-12-07T19:34:30.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa51 and padding block with 0x5af 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2876 is empty 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578ab800 allocated 0x[] released 0x[339000~17000,377000~43000] 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x339000~17000 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 freelist release 0x339000~17000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x377000~43000 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 freelist release 0x377000~43000 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 freelist _xor first_key 0x300000 last_key 0x380000 2019-12-07T19:34:30.634+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab800 prepare 2019-12-07T19:34:30.634+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578ab800 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab800 io_done 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511fe40 finish 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=53) prio 5 cost 1048576 e53) pg 0x558c57676000 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=53) prio 5 cost 1048576 e53) queued 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=53) prio 5 cost 1048576 e53) pg 0x558c548ab000 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap e 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap e 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c550476c0) upper_bound after SNA_1_000000000000000E_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.4' 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54e88500) upper_bound after SNA_1_000000000000000E_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.0_0000000000000001.C' 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c550476c0) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54e88500) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000E_.0_0000000000000001.4) returns 0 SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:30.635+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000E_.0_0000000000000001.C) returns 0 SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[e~1] ps=[13~1,19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[13~1,19~1]] SnapTrimmer state: adding snap e to purged_snaps 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[13~1,19~1]] SnapTrimmer state: adding snap e to purged_snaps 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:30.635+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~1]] SnapTrimmer state: purged_snaps now [e~1,13~1,19~1], snap_trimq now [] 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~1]] SnapTrimmer state: purged_snaps now [e~1,13~1,19~1], snap_trimq now [] 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c578abb00 seq 44 2019-12-07T19:34:30.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c582ad800 seq 55 2019-12-07T19:34:30.635+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:30.635+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578abb00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578abb00 onodes shared_blobs 2019-12-07T19:34:30.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582ad800 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578abb00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582ad800 onodes shared_blobs 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582ad800 allocated 0x[] released 0x[] 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578abb00 prepare 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578abb00 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578abb00 io_done 2019-12-07T19:34:30.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad800 prepare 2019-12-07T19:34:30.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582ad800 2019-12-07T19:34:30.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad800 io_done 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~1]] share_pg_info 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~1]] share_pg_info 2019-12-07T19:34:30.635+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c54e06300 con 0x558c54c64000 2019-12-07T19:34:30.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e53/53) v1 -- 0x558c576df500 con 0x558c54c64000 2019-12-07T19:34:30.635+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c55052000 con 0x558c54c19800 2019-12-07T19:34:30.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e53/53) v1 -- 0x558c57453100 con 0x558c54c19800 2019-12-07T19:34:30.635+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c55055800 con 0x558c54c5bc00 2019-12-07T19:34:30.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e53/53) v1 -- 0x558c5829e700 con 0x558c54c5bc00 2019-12-07T19:34:30.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e53/53) v1 -- 0x558c5829ee00 con 0x558c54c5b400 2019-12-07T19:34:30.635+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c58536000 con 0x558c54c5b400 2019-12-07T19:34:30.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e53/53) v1 -- 0x558c5829f500 con 0x558c54c19c00 2019-12-07T19:34:30.635+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 -- 0x558c58536700 con 0x558c54c19c00 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~1]] exit Trimming 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~1]] exit Trimming 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:30.635+0000 7f8f06986700 15 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:176 2019-12-07T19:34:30.635+0000 7f8f0297e700 15 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:140 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] enter NotTrimming 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] enter NotTrimming 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:30.635+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:30.635+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] snap_trimmer complete 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f7a51 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000418720s (0.000000077s flush + 0.000418643s kv commit) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582ad500> 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1455 < min_flush_size 524288 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad500 kv_submitted 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ad500 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2048 < min_flush_size 524288 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad500 finishing 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ad500 onodes 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ad500 done 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578abb00 kv_queued 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ad500 [97000~16000] 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x97000~16000 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e56e00) put 0x558c54e56e00 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56e18 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57cccbd0 space 0x558c54e56e18 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57cccc60 space 0x558c54e56e18 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e56d20) put 0x558c54e56d20 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56d38 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57cccab0 space 0x558c54e56d38 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57cccb40 space 0x558c54e56d38 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e56c40) put 0x558c54e56c40 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56c58 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57ccc990 space 0x558c54e56c58 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57ccca20 space 0x558c54e56c58 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e56b60) put 0x558c54e56b60 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56b78 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c580cc900 space 0x558c54e56b78 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57a090e0 space 0x558c54e56b78 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e56af0) put 0x558c54e56af0 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56b08 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57a09200 space 0x558c54e56b08 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57a09290 space 0x558c54e56b08 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e56a10) put 0x558c54e56a10 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56a28 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54e14b40 space 0x558c54e56a28 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57a09e60 space 0x558c54e56a28 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e568c0) put 0x558c54e568c0 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e568d8 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c551a97a0 space 0x558c54e568d8 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c580cc870 space 0x558c54e568d8 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e563f0) put 0x558c54e563f0 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e56408 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57b87cb0 space 0x558c54e56408 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 2 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57b87d40 space 0x558c54e56408 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35b80 had 1 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e43d50) put 0x558c54e43d50 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e43d68 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57b87ef0 space 0x558c54e43d68 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3174 < min_flush_size 524288 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57ccc7e0 space 0x558c54e43d68 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e42d20) put 0x558c54e42d20 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42d38 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57ccc870 space 0x558c54e42d38 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57ccc900 space 0x558c54e42d38 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54e42af0) put 0x558c54e42af0 removing self from set 0x558c54d0f1b0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54e42b08 in 0x558c53a59260) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57b87dd0 space 0x558c54e42b08 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57b87e60 space 0x558c54e42b08 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4300 < min_flush_size 524288 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4323 < min_flush_size 524288 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4323 < min_flush_size 524288 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f7a51~10e3 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f7a51 0x2f7a51~10e3 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c7a51 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa51 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb34 and padding block with 0x4cc 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.635+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 174 ==== MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=67, reqid=osd.2.0:66, at_version=0'0, trim_to=0'0, roll_forward_to=53'22)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5852b800 con 0x558c54c19800 2019-12-07T19:34:30.635+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000019 epoch 53 MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=67, reqid=osd.2.0:66, at_version=0'0, trim_to=0'0, roll_forward_to=53'22)) v2 2019-12-07T19:34:30.635+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=67, reqid=osd.2.0:66, at_version=0'0, trim_to=0'0, roll_forward_to=53'22)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f8b34 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.635+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000237589s (0.000000111s flush + 0.000237478s kv commit) 2019-12-07T19:34:30.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c578ab800,0x558c578abb00,0x558c582ad800> 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab800 kv_submitted 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578ab800 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab800 finishing 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578ab800 onodes 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578ab800 done 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ad800 kv_submitted 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578ab800 [339000~17000,377000~43000] 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x339000~17000 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x377000~43000 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794df80) put 0x558c5794df80 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794df98 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a7290 space 0x558c5794df98 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794dea0) put 0x558c5794dea0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794deb8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a7200 space 0x558c5794deb8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794ddc0) put 0x558c5794ddc0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794ddd8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a7170 space 0x558c5794ddd8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794dce0) put 0x558c5794dce0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794dcf8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a70e0 space 0x558c5794dcf8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794dc00) put 0x558c5794dc00 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794dc18 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a7050 space 0x558c5794dc18 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794db20) put 0x558c5794db20 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794db38 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6fc0 space 0x558c5794db38 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794da40) put 0x558c5794da40 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794da58 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6f30 space 0x558c5794da58 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794d960) put 0x558c5794d960 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d978 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6ea0 space 0x558c5794d978 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794d880) put 0x558c5794d880 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d898 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6e10 space 0x558c5794d898 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794d7a0) put 0x558c5794d7a0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d7b8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6d80 space 0x558c5794d7b8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794d6c0) put 0x558c5794d6c0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d6d8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6cf0 space 0x558c5794d6d8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794d5e0) put 0x558c5794d5e0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d5f8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6c60 space 0x558c5794d5f8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794d500) put 0x558c5794d500 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d518 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6bd0 space 0x558c5794d518 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794d420) put 0x558c5794d420 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d438 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6b40 space 0x558c5794d438 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d960) put 0x558c5792d960 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d978 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6a20 space 0x558c5792d978 0x1000~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d880) put 0x558c5792d880 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d898 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6990 space 0x558c5792d898 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d7a0) put 0x558c5792d7a0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d7b8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6900 space 0x558c5792d7b8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d6c0) put 0x558c5792d6c0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d6d8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6870 space 0x558c5792d6d8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d5e0) put 0x558c5792d5e0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d5f8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a67e0 space 0x558c5792d5f8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d500) put 0x558c5792d500 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d518 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6750 space 0x558c5792d518 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d420) put 0x558c5792d420 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d438 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a66c0 space 0x558c5792d438 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d340) put 0x558c5792d340 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d358 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6630 space 0x558c5792d358 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d260) put 0x558c5792d260 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d278 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a65a0 space 0x558c5792d278 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792d180) put 0x558c5792d180 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792d198 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6510 space 0x558c5792d198 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c690) put 0x558c5792c690 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c6a8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6480 space 0x558c5792c6a8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c5b0) put 0x558c5792c5b0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c5c8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a63f0 space 0x558c5792c5c8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c4d0) put 0x558c5792c4d0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c4e8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6360 space 0x558c5792c4e8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c3f0) put 0x558c5792c3f0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c408 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a62d0 space 0x558c5792c408 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c310) put 0x558c5792c310 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c328 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6240 space 0x558c5792c328 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c230) put 0x558c5792c230 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c248 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a61b0 space 0x558c5792c248 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c150) put 0x558c5792c150 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c168 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6120 space 0x558c5792c168 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5792c070) put 0x558c5792c070 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c088 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6090 space 0x558c5792c088 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578f1c70) put 0x558c578f1c70 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1c88 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb7ef0 space 0x558c578f1c88 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578f1b90) put 0x558c578f1b90 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1ba8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb7e60 space 0x558c578f1ba8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578f1ab0) put 0x558c578f1ab0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1ac8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb7dd0 space 0x558c578f1ac8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578f19d0) put 0x558c578f19d0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f19e8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb7d40 space 0x558c578f19e8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578f18f0) put 0x558c578f18f0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1908 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c57982120 space 0x558c578f1908 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578dfea0) put 0x558c578dfea0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dfeb8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb79e0 space 0x558c578dfeb8 0x0~1000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c575bd9d0) put 0x558c575bd9d0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575bd9e8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb73b0 space 0x558c575bd9e8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5757e8c0) put 0x558c5757e8c0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e8d8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb7a70 space 0x558c5757e8d8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574d2850) put 0x558c574d2850 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d2868 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb6870 space 0x558c574d2868 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5742bc00) put 0x558c5742bc00 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742bc18 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb7cb0 space 0x558c5742bc18 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c573967e0) put 0x558c573967e0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573967f8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54cb7b00 space 0x558c573967f8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57396230) put 0x558c57396230 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396248 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c54e14750 space 0x558c57396248 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d75e30) put 0x558c54d75e30 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75e48 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6000 space 0x558c54d75e48 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c546777a0) put 0x558c546777a0 removing self from set 0x558c54d380d0 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c546777b8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 _rm buffer(0x558c579a6ab0 space 0x558c546777b8 0x0~2000 clean) 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578abb00 kv_submitted 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578abb00 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578abb00 finishing 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578abb00 onodes 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578abb00 done 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578abb00 [] 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad800 kv_submitted 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582ad800 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582ad800 finishing 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582ad800 onodes 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582ad800 done 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582ad800 [] 2019-12-07T19:34:30.635+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.635+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=67, reqid=osd.2.0:66, at_version=0'0, trim_to=0'0, roll_forward_to=53'22)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=67, reqid=osd.2.0:66, at_version=0'0, trim_to=0'0, roll_forward_to=53'22)) v2) prio 127 cost 0 e53) pg 0x558c55141000 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 53 dequeue_op 0x558c54e79e40 prio 127 cost 0 latency 0.000527 MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=67, reqid=osd.2.0:66, at_version=0'0, trim_to=0'0, roll_forward_to=53'22)) v2 pg pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 53/41 ECSubWrite(tid=67, reqid=osd.2.0:66, at_version=0'0, trim_to=0'0, roll_forward_to=53'22)) v2 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=52'18 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'22], crt=52'18) [] 2019-12-07T19:34:30.636+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 104 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58513800 con 0x558c54c19c00 2019-12-07T19:34:30.636+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c57f3edc0 prio 196 cost 0 latency 0.000016 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.636+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'19 (52'15) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:30.623424+0000 0 snaps [15,1a,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'20 (52'16) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:30.623424+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.636+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'21 (52'17) delete 1:bf81b261:::smithi13140796-37:13 by unknown.0.0:0 2019-12-07T19:34:30.625774+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.636+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.636+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'22 (52'18) delete 1:bf81b261:::smithi13140796-37:head by unknown.0.0:0 2019-12-07T19:34:30.625774+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.636+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57f3edc0 prio 196 cost 0 latency 0.000064 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 22 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.636+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.636+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57f3edc0 finish 2019-12-07T19:34:30.636+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.636+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 252 ==== MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=144, reqid=osd.3.0:144, at_version=53'69, trim_to=0'0, roll_forward_to=53'65)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c58533000 con 0x558c54c5bc00 2019-12-07T19:34:30.636+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c5511e840 prio 127 cost 0 latency 0.000031 epoch 53 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=144, reqid=osd.3.0:144, at_version=53'69, trim_to=0'0, roll_forward_to=53'65)) v2 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:30.636+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=144, reqid=osd.3.0:144, at_version=53'69, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58525b00 seq 54 2019-12-07T19:34:30.636+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13#15 onode 0x558c57976000 txc 0x558c58525b00 2019-12-07T19:34:30.636+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13#15 0x0 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c574d2f50) put_ref 0x0~2000 Blob(0x558c574d2f50 blob([0x181000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c57f90fc0) put_ref 0x0~2000 Blob(0x558c57f90fc0 blob([0x183000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c57f91180) put_ref 0x0~2000 Blob(0x558c57f91180 blob([0x185000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c5750ff80) put_ref 0x0~2000 Blob(0x558c5750ff80 blob([0x187000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f05b0) put_ref 0x0~2000 Blob(0x558c578f05b0 blob([0x189000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c57f51110) put_ref 0x0~2000 Blob(0x558c57f51110 blob([0x18b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f0460) put_ref 0x0~2000 Blob(0x558c578f0460 blob([0x18d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f04d0) put_ref 0x0~2000 Blob(0x558c578f04d0 blob([0x18f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f12d0) put_ref 0x0~2000 Blob(0x558c578f12d0 blob([0x191000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c57f51180) put_ref 0x0~2000 Blob(0x558c57f51180 blob([0x193000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f0380) put_ref 0x0~2000 Blob(0x558c578f0380 blob([0x195000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f03f0) put_ref 0x0~2000 Blob(0x558c578f03f0 blob([0x197000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f1260) put_ref 0x0~2000 Blob(0x558c578f1260 blob([0x199000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c57f511f0) put_ref 0x0~2000 Blob(0x558c57f511f0 blob([0x19b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f02a0) put_ref 0x0~2000 Blob(0x558c578f02a0 blob([0x19d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f0310) put_ref 0x0~2000 Blob(0x558c578f0310 blob([0x19f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f11f0) put_ref 0x0~2000 Blob(0x558c578f11f0 blob([0x1a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c57f51260) put_ref 0x0~2000 Blob(0x558c57f51260 blob([0x1a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.blob(0x558c578f01c0) put_ref 0x0~1000 Blob(0x558c578f01c0 blob([0x1a5000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore.extentmap(0x558c57976108) dirty_range mark inline shard dirty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574d2f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map(0x181000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x181000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x181000~2000] from SharedBlob(0x558c55278a80 loaded (sbid 0x28d1 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x181000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57f90fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map(0x183000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x183000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x183000~2000] from SharedBlob(0x558c57538700 loaded (sbid 0x28d2 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x183000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57f91180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map(0x185000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x185000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x185000~2000] from SharedBlob(0x558c57538850 loaded (sbid 0x28d3 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x185000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c5750ff80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map(0x187000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x187000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x187000~2000] from SharedBlob(0x558c57538930 loaded (sbid 0x28d4 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x187000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c578f05b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map(0x189000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x189000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x189000~2000] from SharedBlob(0x558c57538a10 loaded (sbid 0x28d5 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x189000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57f51110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map(0x18b000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x18b000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x18b000~2000] from SharedBlob(0x558c575832d0 loaded (sbid 0x28d6 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x18b000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c578f0460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map(0x18d000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x18d000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x18d000~2000] from SharedBlob(0x558c575831f0 loaded (sbid 0x28d7 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x18d000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c578f04d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map(0x18f000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x18f000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x18f000~2000] from SharedBlob(0x558c57583650 loaded (sbid 0x28d8 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x18f000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c578f12d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map(0x191000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x191000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x191000~2000] from SharedBlob(0x558c57583730 loaded (sbid 0x28d9 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x191000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57f51180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map(0x193000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x193000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x193000~2000] from SharedBlob(0x558c57583810 loaded (sbid 0x28da ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x193000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c578f0380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map(0x195000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x195000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x195000~2000] from SharedBlob(0x558c575838f0 loaded (sbid 0x28db ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x195000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c578f03f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map(0x197000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x197000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x197000~2000] from SharedBlob(0x558c54d3c540 loaded (sbid 0x28dc ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x197000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c578f1260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map(0x199000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x199000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x199000~2000] from SharedBlob(0x558c575839d0 loaded (sbid 0x28dd ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x199000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57f511f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map(0x19b000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x19b000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x19b000~2000] from SharedBlob(0x558c57583ab0 loaded (sbid 0x28de ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x19b000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c578f02a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map(0x19d000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x19d000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x19d000~2000] from SharedBlob(0x558c57583b90 loaded (sbid 0x28df ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x19d000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c578f0310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map(0x19f000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x19f000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x19f000~2000] from SharedBlob(0x558c57583c70 loaded (sbid 0x28e0 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x19f000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c578f11f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map(0x1a1000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1a1000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x1a1000~2000] from SharedBlob(0x558c57583d50 loaded (sbid 0x28e1 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x1a1000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57f51260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map(0x1a3000~2000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1a3000~2000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x1a3000~2000] from SharedBlob(0x558c57583e30 loaded (sbid 0x28e2 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x1a3000~2000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~1000: 0x0~1000 Blob(0x558c578f01c0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map(0x1a5000~1000=1)))) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1a5000~1000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x1a5000~1000] from SharedBlob(0x558c57583f10 loaded (sbid 0x28e3 ref_map())) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x1a5000~1000 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3c540,0x558c55278a80,0x558c57538700,0x558c57538850,0x558c57538930,0x558c57538a10,0x558c575831f0,0x558c575832d0,0x558c57583650,0x558c57583730,0x558c57583810,0x558c575838f0,0x558c575839d0,0x558c57583ab0,0x558c57583b90,0x558c57583c70,0x558c57583d50,0x558c57583e30,0x558c57583f10 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bf81b261:::smithi13140796-37:13#15 = 0 2019-12-07T19:34:30.636+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head#16 onode 0x558c54d96c80 txc 0x558c58525b00 2019-12-07T19:34:30.636+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head#16 0x0 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bf81b261:::smithi13140796-37:head#16 = 0 2019-12-07T19:34:30.636+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58525b00 cost 5587 (1 ios * 4000 + 1587 bytes) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58525b00 onodes shared_blobs 0x558c54d3c540,0x558c55278a80,0x558c57538700,0x558c57538850,0x558c57538930,0x558c57538a10,0x558c575831f0,0x558c575832d0,0x558c57583650,0x558c57583730,0x558c57583810,0x558c575838f0,0x558c575839d0,0x558c57583ab0,0x558c57583b90,0x558c57583c70,0x558c57583d50,0x558c57583e30,0x558c57583f10 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28dc is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d1 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d2 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d3 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d4 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d5 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d7 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d6 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d8 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d9 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28da is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28db is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28dd is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28de is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28df is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e0 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e1 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e2 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28e3 is empty 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58525b00 allocated 0x[] released 0x[181000~25000] 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x181000~25000 2019-12-07T19:34:30.636+0000 7f8f06185700 10 freelist release 0x181000~25000 2019-12-07T19:34:30.636+0000 7f8f06185700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525b00 prepare 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58525b00 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525b00 io_done 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 53 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=144, reqid=osd.3.0:144, at_version=53'69, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=144, reqid=osd.3.0:144, at_version=53'69, trim_to=0'0, roll_forward_to=53'65)) v2) prio 127 cost 0 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c5511e840 prio 127 cost 0 latency 0.000379 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=144, reqid=osd.3.0:144, at_version=53'69, trim_to=0'0, roll_forward_to=53'65)) v2 pg pg[1.6s2( v 53'67 (0'0,53'67] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'67 (0'0,53'67] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=144, reqid=osd.3.0:144, at_version=53'69, trim_to=0'0, roll_forward_to=53'65)) v2 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6336c6f2:::smithi13140796-16:17 15 was 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96c80 had 1 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976000 had 1 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1555 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1555 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f8b34~613 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f8b34 0x2f8b34~613 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6336c6f2:::smithi13140796-16:17 e,15 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c8b34 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb34 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x147 and padding block with 0xeb9 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6336c6f2:::smithi13140796-16:17 15 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:30.636+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 175 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57453100 con 0x558c54c19800 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'67 (0'0,53'67] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'67], crt=53'65) [53'68 (52'60) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:30.636089+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'69 (52'61) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:30.636089+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.636+0000 7f8f24c58700 15 osd.7 53 enqueue_peering_evt 1.ds2 epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.ds0( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'68 (0'0,53'68] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'67 crt=53'65 lcod 53'67 active mbc={} ps=[13~1,19~1]] add_log_entry 53'68 (52'60) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:30.636089+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.636+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.ds0( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'67 crt=53'65 lcod 53'67 active mbc={} ps=[13~1,19~1]] add_log_entry 53'69 (52'61) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:30.636089+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'67 crt=53'65 lcod 53'67 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 69 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'67 crt=53'65 lcod 53'67 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.636+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'68, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f9147 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000158331s (0.000000135s flush + 0.000158196s kv commit) 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58525b00> 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525b00 kv_submitted 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58525b00 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58525b00 finishing 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58525b00 onodes 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58525b00 done 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58525b00 [181000~25000] 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x181000~25000 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583f10) put 0x558c57583f10 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583f28 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c578ac990 space 0x558c57583f28 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583e30) put 0x558c57583e30 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583e48 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c578ac870 space 0x558c57583e48 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c578ac900 space 0x558c57583e48 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582af800 seq 102 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583d50) put 0x558c57583d50 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.ds0( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583d68 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671e60 space 0x558c57583d68 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.ds0( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e53) pg 0x558c55141000 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671ef0 space 0x558c57583d68 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583c70) put 0x558c57583c70 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583c88 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671d40 space 0x558c57583c88 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# 1 keys = 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671dd0 space 0x558c57583c88 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583b90) put 0x558c57583b90 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583ba8 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671c20 space 0x558c57583ba8 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 2(0) info: 1.ds0( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671cb0 space 0x558c57583ba8 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583ab0) put 0x558c57583ab0 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583ac8 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671b00 space 0x558c57583ac8 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671b90 space 0x558c57583ac8 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c575839d0) put 0x558c575839d0 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575839e8 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c576719e0 space 0x558c575839e8 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671a70 space 0x558c575839e8 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c575838f0) put 0x558c575838f0 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583908 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c576717a0 space 0x558c57583908 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671830 space 0x558c57583908 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583810) put 0x558c57583810 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583828 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671680 space 0x558c57583828 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671710 space 0x558c57583828 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583730) put 0x558c57583730 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583748 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671560 space 0x558c57583748 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c576715f0 space 0x558c57583748 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57583650) put 0x558c57583650 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583668 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5853a000 seq 55 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671440 space 0x558c57583668 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c576714d0 space 0x558c57583668 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c575832d0) put 0x558c575832d0 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575832e8 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_000000000000000E_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671200 space 0x558c575832e8 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671290 space 0x558c575832e8 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c575831f0) put 0x558c575831f0 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57583208 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671320 space 0x558c57583208 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c576713b0 space 0x558c57583208 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57538a10) put 0x558c57538a10 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000068' <- 0000000053.00000000000000000068 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538a28 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c576710e0 space 0x558c57538a28 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671170 space 0x558c57538a28 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000053.00000000000000000069' <- 0000000053.00000000000000000069 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853a000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57538930) put 0x558c57538930 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538948 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853a000 onodes shared_blobs 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57670fc0 space 0x558c57538948 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853a000 allocated 0x[] released 0x[] 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671050 space 0x558c57538948 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57538850) put 0x558c57538850 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538868 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57b87560 space 0x558c57538868 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57670f30 space 0x558c57538868 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a000 prepare 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57538700) put 0x558c57538700 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853a000 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57538718 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a000 io_done 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582af800 cost 7559 (1 ios * 4000 + 3559 bytes) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57b87440 space 0x558c57538718 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582af800 onodes 0x558c54ce3900,0x558c57f8e500 shared_blobs 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57b874d0 space 0x558c57538718 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c55278a80) put 0x558c55278a80 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55278a98 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c573d9560 space 0x558c55278a98 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c551a9d40 space 0x558c55278a98 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3c540) put 0x558c54d3c540 removing self from set 0x558c54d0f570 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1579 (558 bytes onode + 2 bytes spanning blobs + 1019 bytes inline extents) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c558 in 0x558c53a59dc0) _clear 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c576718c0 space 0x558c54d3c558 0x0~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 _rm buffer(0x558c57671950 space 0x558c54d3c558 0x1000~1000 clean) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e608) update 2#1:6336c6f2:::smithi13140796-16:17# 2019-12-07T19:34:30.636+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 53/41 ECSubWriteReply(tid=67, last_complete=53'22, committed=1, applied=1)) v2 -- 0x558c58512e00 con 0x558c54c19800 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:17# is 637 (412 bytes onode + 2 bytes spanning blobs + 223 bytes inline extents) 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582af800 allocated 0x[] released 0x[] 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af800 prepare 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582af800 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af800 io_done 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f9147~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f9147 0x2f9147~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'67 crt=53'65 lcod 53'67 active mbc={} ps=[13~1,19~1]] op_applied version 53'69 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c9147 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x147 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5c4 and padding block with 0xa3c 2019-12-07T19:34:30.636+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c5511e840 finish 2019-12-07T19:34:30.636+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 105 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59200 con 0x558c54c19c00 2019-12-07T19:34:30.636+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c57f3f600 prio 196 cost 0 latency 0.000013 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.636+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.636+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.636+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.636+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2f95c4 2019-12-07T19:34:30.636+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57f3f600 prio 196 cost 0 latency 0.000059 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.636+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.636+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57f3f600 finish 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000139645s (0.000000120s flush + 0.000139525s kv commit) 2019-12-07T19:34:30.636+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853a000> 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4237 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a000 kv_submitted 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853a000 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a000 finishing 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853a000 onodes 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853a000 done 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853a000 [] 2019-12-07T19:34:30.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.636+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e500 had 1 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4260 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4260 < min_flush_size 524288 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2f95c4~10a4 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2f95c4 0x2f95c4~10a4 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2c95c4 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5c4 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x668 and padding block with 0x998 2019-12-07T19:34:30.637+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:30.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fa668 2019-12-07T19:34:30.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000160295s (0.000000094s flush + 0.000160201s kv commit) 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582af800> 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af800 kv_submitted 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582af800 2019-12-07T19:34:30.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582af800 finishing 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582af800 onodes 0x558c54ce3900,0x558c57f8e500 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582af800 done 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:30.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582af800 [] 2019-12-07T19:34:30.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.637+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.637+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 53/28 ECSubWriteReply(tid=144, last_complete=53'69, committed=1, applied=1)) v2 -- 0x558c58513e00 con 0x558c54c5bc00 2019-12-07T19:34:30.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.637+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 276 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58513e00 con 0x558c54c5b400 2019-12-07T19:34:30.637+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c5511e9a0 prio 196 cost 0 latency 0.000015 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.637+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.637+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 277 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c1e000 con 0x558c54c5b400 2019-12-07T19:34:30.637+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511e9a0 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.637+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57efe2c0 prio 196 cost 0 latency 0.000010 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.637+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.637+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c5511e9a0 finish 2019-12-07T19:34:30.637+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.637+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.637+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.637+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.637+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.637+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57efe2c0 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.637+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.637+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57efe2c0 finish 2019-12-07T19:34:30.637+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.638+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 429 ==== MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=254, reqid=osd.1.0:254, at_version=53'21, trim_to=0'0, roll_forward_to=45'19)) v2 ==== 2656+0+0 (crc 0 0 0) 0x558c58533800 con 0x558c54c64000 2019-12-07T19:34:30.638+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c57f3fa20 prio 127 cost 0 latency 0.000032 epoch 53 MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=254, reqid=osd.1.0:254, at_version=53'21, trim_to=0'0, roll_forward_to=45'19)) v2 2019-12-07T19:34:30.638+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=254, reqid=osd.1.0:254, at_version=53'21, trim_to=0'0, roll_forward_to=45'19)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.639+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 278 ==== MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=105, reqid=osd.4.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=53'19)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c58532800 con 0x558c54c5b400 2019-12-07T19:34:30.639+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57eff340 prio 127 cost 0 latency 0.000030 epoch 53 MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=105, reqid=osd.4.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=53'19)) v2 2019-12-07T19:34:30.639+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=105, reqid=osd.4.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=53'19)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.639+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 279 ==== MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=106, reqid=osd.4.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=53'37)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c57a43000 con 0x558c54c5b400 2019-12-07T19:34:30.639+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57eff1e0 prio 127 cost 0 latency 0.000021 epoch 53 MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=106, reqid=osd.4.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=53'37)) v2 2019-12-07T19:34:30.639+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=106, reqid=osd.4.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=105, reqid=osd.4.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=53'19)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=105, reqid=osd.4.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=53'19)) v2) prio 127 cost 0 e53) pg 0x558c5513e000 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 osd.7 53 dequeue_op 0x558c57eff340 prio 127 cost 0 latency 0.000164 MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=105, reqid=osd.4.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=53'19)) v2 pg pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=106, reqid=osd.4.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=106, reqid=osd.4.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e53) pg 0x558c575ef000 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.3s4 53/41 ECSubWrite(tid=105, reqid=osd.4.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=53'19)) v2 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=45'17 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'19], crt=45'17) [] 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 53 dequeue_op 0x558c57eff1e0 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=106, reqid=osd.4.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=53'37)) v2 pg pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'18 (45'16) delete 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:34:30.626447+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.bs1 53/41 ECSubWrite(tid=106, reqid=osd.4.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=53'37)) v2 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=52'33 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'37], crt=52'33) [] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'19 (45'17) modify 1:c8ce3553:::smithi13140796-29:head by unknown.0.0:0 2019-12-07T19:34:30.626447+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'34 (52'30) modify 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:30.625395+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 19 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'35 (52'31) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:30.625395+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'36 (52'32) modify 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:30.628696+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'37 (52'33) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:30.628696+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 37 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.639+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.639+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5810f200 seq 62 2019-12-07T19:34:30.639+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e#12 onode 0x558c57976500 txc 0x558c5810f200 2019-12-07T19:34:30.639+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e#12 0x0 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57971ce0) put_ref 0x0~2000 Blob(0x558c57971ce0 blob([0xe1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c576ac3f0) put_ref 0x0~2000 Blob(0x558c576ac3f0 blob([0xe3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58522c00 seq 44 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf81c0) put_ref 0x0~2000 Blob(0x558c54cf81c0 blob([0xe5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57971e30) put_ref 0x0~2000 Blob(0x558c57971e30 blob([0xe7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c576ac540) put_ref 0x0~2000 Blob(0x558c576ac540 blob([0xe9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3ba40) put_ref 0x0~2000 Blob(0x558c57f3ba40 blob([0xeb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58522c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58522c00 onodes shared_blobs 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bb20) put_ref 0x0~2000 Blob(0x558c57f3bb20 blob([0xed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58522c00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bb90) put_ref 0x0~2000 Blob(0x558c57f3bb90 blob([0xef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522c00 prepare 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58522c00 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bc00) put_ref 0x0~2000 Blob(0x558c57f3bc00 blob([0xf1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522c00 io_done 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bc70) put_ref 0x0~2000 Blob(0x558c57f3bc70 blob([0xf3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 53 dequeue_op 0x558c57eff1e0 finish 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bce0) put_ref 0x0~2000 Blob(0x558c57f3bce0 blob([0xf5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bd50) put_ref 0x0~2000 Blob(0x558c57f3bd50 blob([0xf7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bdc0) put_ref 0x0~2000 Blob(0x558c57f3bdc0 blob([0xf9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3be30) put_ref 0x0~2000 Blob(0x558c57f3be30 blob([0xfb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bea0) put_ref 0x0~1000 Blob(0x558c57f3bea0 blob([0xfd000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57976608) dirty_range mark inline shard dirty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57971ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map(0xe1000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe1000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xe1000~2000] from SharedBlob(0x558c574e0c40 loaded (sbid 0x285a ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xe1000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c576ac3f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map(0xe3000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe3000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xe3000~2000] from SharedBlob(0x558c54d3c310 loaded (sbid 0x285b ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xe3000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54cf81c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map(0xe5000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe5000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xe5000~2000] from SharedBlob(0x558c54d3c4d0 loaded (sbid 0x285c ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xe5000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57971e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map(0xe7000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe7000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xe7000~2000] from SharedBlob(0x558c574e2000 loaded (sbid 0x285d ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xe7000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c576ac540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map(0xe9000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xe9000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xe9000~2000] from SharedBlob(0x558c574e20e0 loaded (sbid 0x285e ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xe9000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57f3ba40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map(0xeb000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xeb000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xeb000~2000] from SharedBlob(0x558c574e21c0 loaded (sbid 0x285f ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xeb000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57f3bb20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map(0xed000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xed000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xed000~2000] from SharedBlob(0x558c574e22a0 loaded (sbid 0x2860 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xed000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57f3bb90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map(0xef000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xef000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xef000~2000] from SharedBlob(0x558c574e2380 loaded (sbid 0x2861 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xef000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57f3bc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map(0xf1000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf1000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xf1000~2000] from SharedBlob(0x558c574e2460 loaded (sbid 0x2862 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xf1000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57f3bc70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map(0xf3000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf3000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xf3000~2000] from SharedBlob(0x558c574e2540 loaded (sbid 0x2863 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xf3000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57f3bce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map(0xf5000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf5000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xf5000~2000] from SharedBlob(0x558c574e2620 loaded (sbid 0x2864 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xf5000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57f3bd50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map(0xf7000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf7000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xf7000~2000] from SharedBlob(0x558c574e2700 loaded (sbid 0x2865 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xf7000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57f3bdc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map(0xf9000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xf9000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xf9000~2000] from SharedBlob(0x558c574e27e0 loaded (sbid 0x2866 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xf9000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57f3be30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map(0xfb000~2000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xfb000~2000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xfb000~2000] from SharedBlob(0x558c574e28c0 loaded (sbid 0x2867 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xfb000~2000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~1000: 0x0~1000 Blob(0x558c57f3bea0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map(0xfd000~1000=1)))) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xfd000~1000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0xfd000~1000] from SharedBlob(0x558c574e29a0 loaded (sbid 0x2868 ref_map())) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0xfd000~1000 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3c310,0x558c54d3c4d0,0x558c574e0c40,0x558c574e2000,0x558c574e20e0,0x558c574e21c0,0x558c574e22a0,0x558c574e2380,0x558c574e2460,0x558c574e2540,0x558c574e2620,0x558c574e2700,0x558c574e27e0,0x558c574e28c0,0x558c574e29a0 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.3s4_head 4#1:c8ce3553:::smithi13140796-29:e#12 = 0 2019-12-07T19:34:30.639+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:30.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810f200 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810f200 onodes shared_blobs 0x558c54d3c310,0x558c54d3c4d0,0x558c574e0c40,0x558c574e2000,0x558c574e20e0,0x558c574e21c0,0x558c574e22a0,0x558c574e2380,0x558c574e2460,0x558c574e2540,0x558c574e2620,0x558c574e2700,0x558c574e27e0,0x558c574e28c0,0x558c574e29a0 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285b is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285c is empty 2019-12-07T19:34:30.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285a is empty 2019-12-07T19:34:30.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285d is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285e is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x285f is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2860 is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2861 is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2862 is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2863 is empty 2019-12-07T19:34:30.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2864 is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2865 is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2866 is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2867 is empty 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2868 is empty 2019-12-07T19:34:30.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fa668~471 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810f200 allocated 0x[] released 0x[e1000~1d000] 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0xe1000~1d000 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 freelist release 0xe1000~1d000 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:34:30.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fa668 0x2fa668~471 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f200 prepare 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810f200 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f200 io_done 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2ca668 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x668 2019-12-07T19:34:30.639+0000 7f8f0317f700 10 osd.7 53 dequeue_op 0x558c57eff340 finish 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xad9 and padding block with 0x527 2019-12-07T19:34:30.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.639+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=254, reqid=osd.1.0:254, at_version=53'21, trim_to=0'0, roll_forward_to=45'19)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.639+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=254, reqid=osd.1.0:254, at_version=53'21, trim_to=0'0, roll_forward_to=45'19)) v2) prio 127 cost 0 e53) pg 0x558c5500d000 2019-12-07T19:34:30.639+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 280 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c5829e700 con 0x558c54c5b400 2019-12-07T19:34:30.639+0000 7f8f04982700 10 osd.7 53 dequeue_op 0x558c57f3fa20 prio 127 cost 0 latency 0.001060 MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=254, reqid=osd.1.0:254, at_version=53'21, trim_to=0'0, roll_forward_to=45'19)) v2 pg pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.639+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=254, reqid=osd.1.0:254, at_version=53'21, trim_to=0'0, roll_forward_to=45'19)) v2 2019-12-07T19:34:30.639+0000 7f8f23c56700 15 osd.7 53 enqueue_peering_evt 1.bs1 epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:30.639+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.639+0000 7f8f04982700 20 snap_mapper.remove_oid 1:0064fe1d:::smithi13140796-44:e 2019-12-07T19:34:30.639+0000 7f8f04982700 20 snap_mapper._remove_oid 1:0064fe1d:::smithi13140796-44:e 2019-12-07T19:34:30.639+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.639+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:30.639+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.639+0000 7f8f04982700 20 snap_mapper.get_snaps 1:0064fe1d:::smithi13140796-44:e e 2019-12-07T19:34:30.639+0000 7f8f04982700 20 snap_mapper.clear_snaps 1:0064fe1d:::smithi13140796-44:e 2019-12-07T19:34:30.639+0000 7f8f04982700 20 snap_mapper.clear_snaps rm OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f04982700 20 snap_mapper._remove_oid rm SNA_1_000000000000000E_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e53) pg 0x558c575ef000 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:30.639+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 45'19 (0'0,45'19] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] append_log log((0'0,45'19], crt=45'19) [53'20 (45'18) delete 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:30.635462+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,53'21 (45'19) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:30.635462+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:30.639+0000 7f8f07187700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.639+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:30.639+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'20 (0'0,53'20] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=45'19 crt=45'19 lcod 45'19 active mbc={} ps=[13~1,19~1]] add_log_entry 53'20 (45'18) delete 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:30.635462+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58522f00 seq 45 2019-12-07T19:34:30.639+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=45'19 crt=45'19 lcod 45'19 active mbc={} ps=[13~1,19~1]] add_log_entry 53'21 (45'19) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:30.635462+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:30.639+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=45'19 crt=45'19 lcod 45'19 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 21 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:30.639+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=45'19 crt=45'19 lcod 45'19 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:30.639+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58522f00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58522f00 onodes shared_blobs 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58522f00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.639+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 53'20, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522f00 prepare 2019-12-07T19:34:30.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58522f00 2019-12-07T19:34:30.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522f00 io_done 2019-12-07T19:34:30.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.639+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:30.639+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c58c65200 seq 44 2019-12-07T19:34:30.639+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) get_onode oid 3#1:0064fe1d:::smithi13140796-44:e#14 key 0x8380000000000000010064fe1d21736d'ithi13140796-44!='0x000000000000000e0000000000000014'o' 2019-12-07T19:34:30.639+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.0s3_head 0x558c53a71c20) r -2 v.len 0 2019-12-07T19:34:30.639+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# -> 3#1:0064fe1d:::smithi13140796-44:e#14 2019-12-07T19:34:30.639+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e# -> 3#1:0064fe1d:::smithi13140796-44:e#14 = 0 2019-12-07T19:34:30.639+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys 2019-12-07T19:34:30.639+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# 2 keys = 0 2019-12-07T19:34:30.639+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) 2019-12-07T19:34:30.639+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:30.639+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.639+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 281 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c5829e700 con 0x558c54c5b400 2019-12-07T19:34:30.639+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- OBJ_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:30.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.640+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.3_0000000000000001.0062F78B.e.smithi13140796-44..' <- SNA_1_000000000000000E_.3_0000000000000001.0062F78B.e.smithi13140796-44.. 2019-12-07T19:34:30.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:30.640+0000 7f8f23c56700 15 osd.7 53 enqueue_peering_evt 1.3s4 epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:30.640+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2faad9 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.640+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000053.00000000000000000020' <- 0000000053.00000000000000000020 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.0000000053.00000000000000000021' <- 0000000053.00000000000000000021 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 430 ==== MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=255, reqid=osd.1.0:253, at_version=0'0, trim_to=0'0, roll_forward_to=53'33)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5852d800 con 0x558c54c64000 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:30.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c65200 cost 7549 (1 ios * 4000 + 3549 bytes) 2019-12-07T19:34:30.640+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000022 epoch 53 MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=255, reqid=osd.1.0:253, at_version=0'0, trim_to=0'0, roll_forward_to=53'33)) v2 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c65200 onodes 0x558c575bb900,0x558c57f34000 shared_blobs 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore.extentmap(0x558c575bba08) update 3#1:0064fe1d:::smithi13140796-44:head# 2019-12-07T19:34:30.640+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=255, reqid=osd.1.0:253, at_version=0'0, trim_to=0'0, roll_forward_to=53'33)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:head# is 500 (498 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore.extentmap(0x558c57f34108) update 3#1:0064fe1d:::smithi13140796-44:e#14 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 3#1:0064fe1d:::smithi13140796-44:e#14 is 1039 (412 bytes onode + 2 bytes spanning blobs + 625 bytes inline extents) 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c65200 allocated 0x[] released 0x[] 2019-12-07T19:34:30.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65200 prepare 2019-12-07T19:34:30.640+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c65200 2019-12-07T19:34:30.640+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65200 io_done 2019-12-07T19:34:30.640+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 lua=45'19 crt=45'19 lcod 45'19 active mbc={} ps=[13~1,19~1]] op_applied version 53'21 2019-12-07T19:34:30.640+0000 7f8f04982700 10 osd.7 53 dequeue_op 0x558c57f3fa20 finish 2019-12-07T19:34:30.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000001 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000555014s (0.000000124s flush + 0.000554890s kv commit) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1430 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976500 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2556 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6564 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575bb900 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34000 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9900 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6587 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6587 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2faad9~19bb to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2faad9 0x2faad9~19bb to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2caad9 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xad9 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x494 and padding block with 0xb6c 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58522c00> 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522c00 kv_submitted 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58522c00 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522c00 finishing 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58522c00 onodes 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522c00 done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522f00 kv_submitted 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58522c00 [] 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fc494 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000180142s (0.000000109s flush + 0.000180033s kv commit) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5810f200,0x558c58522f00,0x558c58c65200> 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f200 kv_submitted 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810f200 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f200 finishing 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810f200 onodes 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e53) pg 0x558c5513e000 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810f200 done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810f200 [e1000~1d000] 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0xe1000~1d000 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e29a0) put 0x558c574e29a0 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e29b8 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf4d0 space 0x558c574e29b8 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e28c0) put 0x558c574e28c0 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e28d8 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf3b0 space 0x558c574e28d8 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf440 space 0x558c574e28d8 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e27e0) put 0x558c574e27e0 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e27f8 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf290 space 0x558c574e27f8 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf320 space 0x558c574e27f8 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e2700) put 0x558c574e2700 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2718 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf170 space 0x558c574e2718 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf200 space 0x558c574e2718 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e2620) put 0x558c574e2620 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2638 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf050 space 0x558c574e2638 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdf0e0 space 0x558c574e2638 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e2540) put 0x558c574e2540 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2558 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c551a9a70 space 0x558c574e2558 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c551a9560 space 0x558c574e2558 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e2460) put 0x558c574e2460 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2478 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdef30 space 0x558c574e2478 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdefc0 space 0x558c574e2478 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e2380) put 0x558c574e2380 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2398 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdee10 space 0x558c574e2398 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdeea0 space 0x558c574e2398 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e22a0) put 0x558c574e22a0 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5810f500 seq 63 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e22b8 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdecf0 space 0x558c574e22b8 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cded80 space 0x558c574e22b8 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e21c0) put 0x558c574e21c0 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e21d8 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdfef0 space 0x558c574e21d8 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdec60 space 0x558c574e21d8 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e20e0) put 0x558c574e20e0 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e20f8 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c548dea20 space 0x558c574e20f8 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c54cdfe60 space 0x558c574e20f8 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e2000) put 0x558c574e2000 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2018 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c548de900 space 0x558c574e2018 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c548de990 space 0x558c574e2018 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574e0c40) put 0x558c574e0c40 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0c58 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c548de480 space 0x558c574e0c58 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c551a8d80 space 0x558c574e0c58 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3c4d0) put 0x558c54d3c4d0 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c4e8 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c548de7e0 space 0x558c54d3c4e8 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810f500 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c548de870 space 0x558c54d3c4e8 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810f500 onodes shared_blobs 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3c310) put 0x558c54d3c310 removing self from set 0x558c54d0e850 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3c328 in 0x558c53a596c0) _clear 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810f500 allocated 0x[] released 0x[] 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c573d8d80 space 0x558c54d3c328 0x0~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 _rm buffer(0x558c548de750 space 0x558c54d3c328 0x1000~1000 clean) 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522f00 kv_submitted 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58522f00 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58522f00 finishing 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58522f00 onodes 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58522f00 done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58522f00 [] 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f500 prepare 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65200 kv_submitted 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c65200 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810f500 2019-12-07T19:34:30.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f500 io_done 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65200 finishing 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c65200 onodes 0x558c575bb900,0x558c57f34000 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c65200 done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c65200 [] 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.640+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 53/18 ECSubWriteReply(tid=254, last_complete=53'21, committed=1, applied=1)) v2 -- 0x558c57518800 con 0x558c54c64000 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fc494~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fc494 0x2fc494~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2cc494 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x494 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x911 and padding block with 0x6ef 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fc911 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000095528s (0.000000095s flush + 0.000095433s kv commit) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5810f500> 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.640+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 431 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c576df500 con 0x558c54c64000 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f500 kv_submitted 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810f500 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f500 finishing 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810f500 onodes 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810f500 done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810f500 [] 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.640+0000 7f8f24457700 15 osd.7 53 enqueue_peering_evt 1.7s4 epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.7s0( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:30.640+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.7s0( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.7s0( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.7s0( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e53) pg 0x558c54d5f000 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.7s0( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57958300 seq 77 2019-12-07T19:34:30.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57958300 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57958300 onodes shared_blobs 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57958300 allocated 0x[] released 0x[] 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958300 prepare 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57958300 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958300 io_done 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=255, reqid=osd.1.0:253, at_version=0'0, trim_to=0'0, roll_forward_to=53'33)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=255, reqid=osd.1.0:253, at_version=0'0, trim_to=0'0, roll_forward_to=53'33)) v2) prio 127 cost 0 e53) pg 0x558c54d5f000 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 53 dequeue_op 0x558c57f3ef20 prio 127 cost 0 latency 0.000683 MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=255, reqid=osd.1.0:253, at_version=0'0, trim_to=0'0, roll_forward_to=53'33)) v2 pg pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fc911~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fc911 0x2fc911~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2cc911 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x911 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd8e and padding block with 0x272 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.7s4 53/41 ECSubWrite(tid=255, reqid=osd.1.0:253, at_version=0'0, trim_to=0'0, roll_forward_to=53'33)) v2 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=52'29 active mbc={} ps=[e~1,13~1,19~1]] append_log log((0'0,53'33], crt=52'29) [] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=53'30 (52'28) modify 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:30.630519+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fcd8e 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=53'31 (52'29) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:30.630519+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000072042s (0.000000074s flush + 0.000071968s kv commit) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=53'32 (45'22) delete 1:ec1285ab:::smithi13140796-19:10 by unknown.0.0:0 2019-12-07T19:34:30.633192+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57958300> 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958300 kv_submitted 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57958300 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958300 finishing 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57958300 onodes 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57958300 done 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=53'33 (45'23) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:30.633192+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57958300 [] 2019-12-07T19:34:30.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.640+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] append_log approx pg log length = 33 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.640+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57958600 seq 78 2019-12-07T19:34:30.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10#20 onode 0x558c574a0500 txc 0x558c57958600 2019-12-07T19:34:30.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10#20 0x0 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c5792df80) put_ref 0x0~2000 Blob(0x558c5792df80 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c574d23f0) put_ref 0x0~2000 Blob(0x558c574d23f0 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c574d3f80) put_ref 0x0~2000 Blob(0x558c574d3f80 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c5792df10) put_ref 0x0~2000 Blob(0x558c5792df10 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c5792d0a0) put_ref 0x0~2000 Blob(0x558c5792d0a0 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57566fc0) put_ref 0x0~2000 Blob(0x558c57566fc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c575677a0) put_ref 0x0~2000 Blob(0x558c575677a0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57567730) put_ref 0x0~2000 Blob(0x558c57567730 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c575676c0) put_ref 0x0~2000 Blob(0x558c575676c0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.blob(0x558c57567650) put_ref 0x0~1000 Blob(0x558c57567650 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore.extentmap(0x558c574a0608) dirty_range mark inline shard dirty 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c5792df80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c574d23f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x86000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c574d3f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x88000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c5792df10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x8a000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c5792d0a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x8c000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57566fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x8e000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c575677a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x90000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57567730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x92000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c575676c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x94000~2000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~1000: 0x0~1000 Blob(0x558c57567650 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x96000~1000] 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf9650,0x558c54d3d110,0x558c54e42fc0,0x558c54e430a0,0x558c54e43180,0x558c54e43260,0x558c54e43340,0x558c54e43420,0x558c54e43500,0x558c55265960 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:10#20 = 0 2019-12-07T19:34:30.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57958600 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57958600 onodes shared_blobs 0x558c54cf9650,0x558c54d3d110,0x558c54e42fc0,0x558c54e430a0,0x558c54e43180,0x558c54e43260,0x558c54e43340,0x558c54e43420,0x558c54e43500,0x558c55265960 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a0 is 11 SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a1 is 11 SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a3 is 11 SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a4 is 11 SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a5 is 11 SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a6 is 11 SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a7 is 11 SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a8 is 11 SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a9 is 11 SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a2 is 11 SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=1))) 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57958600 allocated 0x[] released 0x[] 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958600 prepare 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57958600 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958600 io_done 2019-12-07T19:34:30.640+0000 7f8f0117b700 10 osd.7 53 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1461 < min_flush_size 524288 2019-12-07T19:34:30.640+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 53/41 ECSubWriteReply(tid=106, last_complete=53'37, committed=1, applied=1)) v2 -- 0x558c55251e00 con 0x558c54c5b400 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:30.641+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 53/41 ECSubWriteReply(tid=105, last_complete=53'19, committed=1, applied=1)) v2 -- 0x558c53a5e600 con 0x558c54c5b400 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0500 had 1 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1484 < min_flush_size 524288 2019-12-07T19:34:30.641+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1484 < min_flush_size 524288 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fcd8e~5cc to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fcd8e 0x2fcd8e~5cc to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2ccd8e 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd8e 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x35a and padding block with 0xca6 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fd35a 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000144665s (0.000000093s flush + 0.000144572s kv commit) 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57958600> 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958600 kv_submitted 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57958600 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958600 finishing 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57958600 onodes 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57958600 done 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57958600 [] 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.641+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 53/41 ECSubWriteReply(tid=255, last_complete=53'33, committed=1, applied=1)) v2 -- 0x558c58513a00 con 0x558c54c64000 2019-12-07T19:34:30.641+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 432 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c576df500 con 0x558c54c64000 2019-12-07T19:34:30.641+0000 7f8f24457700 15 osd.7 53 enqueue_peering_evt 1.as5 epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:30.641+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.641+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.641+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 433 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518800 con 0x558c54c64000 2019-12-07T19:34:30.641+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e53) pg 0x558c57e8b000 2019-12-07T19:34:30.641+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c57f3f4a0 prio 196 cost 0 latency 0.000012 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.641+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.641+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 434 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58513600 con 0x558c54c64000 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.641+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c57334dc0 prio 196 cost 0 latency 0.000008 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.641+0000 7f8f07988700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.641+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.641+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:30.641+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:30.641+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58110600 seq 18 2019-12-07T19:34:30.641+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57f3f4a0 prio 196 cost 0 latency 0.000041 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.641+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:30.641+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:30.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.641+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:30.641+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57f3f4a0 finish 2019-12-07T19:34:30.641+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:30.641+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58110600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:30.641+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58110600 onodes shared_blobs 2019-12-07T19:34:30.641+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58110600 allocated 0x[] released 0x[] 2019-12-07T19:34:30.641+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110600 prepare 2019-12-07T19:34:30.641+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58110600 2019-12-07T19:34:30.641+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110600 io_done 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.641+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fd35a~495 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fd35a 0x2fd35a~495 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2cd35a 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x35a 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7ef and padding block with 0x811 2019-12-07T19:34:30.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fd7ef 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000132351s (0.000000122s flush + 0.000132229s kv commit) 2019-12-07T19:34:30.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58110600> 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110600 kv_submitted 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58110600 2019-12-07T19:34:30.641+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57334dc0 prio 196 cost 0 latency 0.000207 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110600 finishing 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58110600 onodes 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58110600 done 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58110600 [] 2019-12-07T19:34:30.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.641+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.641+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57334dc0 finish 2019-12-07T19:34:30.641+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 253 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c53a5e600 con 0x558c54c5bc00 2019-12-07T19:34:30.641+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57efeb00 prio 196 cost 0 latency 0.000013 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.641+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.641+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 254 ==== MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=145, reqid=osd.3.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=53'69)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c58154000 con 0x558c54c5bc00 2019-12-07T19:34:30.641+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57efec60 prio 127 cost 0 latency 0.000017 epoch 53 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=145, reqid=osd.3.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=53'69)) v2 2019-12-07T19:34:30.641+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=145, reqid=osd.3.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=53'69)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.641+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57efeb00 prio 196 cost 0 latency 0.000039 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=145, reqid=osd.3.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=53'69)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.641+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=145, reqid=osd.3.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=53'69)) v2) prio 127 cost 0 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.641+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c57efec60 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=145, reqid=osd.3.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=53'69)) v2 pg pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.6s2 53/28 ECSubWrite(tid=145, reqid=osd.3.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=53'69)) v2 2019-12-07T19:34:30.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'65 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'69], crt=53'65) [] 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'66 (45'49) delete 1:6ee4dfe7:::smithi13140796-47:e by unknown.0.0:0 2019-12-07T19:34:30.633777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'67 (45'50) modify 1:6ee4dfe7:::smithi13140796-47:head by unknown.0.0:0 2019-12-07T19:34:30.633777+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'68 (52'60) modify 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:30.636089+0000 0 snaps [15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'69 (52'61) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:30.636089+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 69 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.642+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.642+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 255 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518000 con 0x558c54c5bc00 2019-12-07T19:34:30.642+0000 7f8f23c56700 15 osd.7 53 enqueue_op 0x558c57eff8c0 prio 196 cost 0 latency 0.000014 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.642+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:30.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c582afb00 seq 103 2019-12-07T19:34:30.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e#42 onode 0x558c54e12280 txc 0x558c582afb00 2019-12-07T19:34:30.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e#42 0x0 2019-12-07T19:34:30.642+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57eff8c0 prio 196 cost 0 latency 0.000055 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) put_ref 0x0~2000 Blob(0x558c574d28c0 blob([0x22b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) put_ref 0x0~2000 Blob(0x558c54d750a0 blob([0x22d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c577f39d0) put_ref 0x0~2000 Blob(0x558c577f39d0 blob([0x22f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57314310) put_ref 0x0~2000 Blob(0x558c57314310 blob([0x231000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57eff8c0 finish 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574d39d0) put_ref 0x0~2000 Blob(0x558c574d39d0 blob([0x233000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) put_ref 0x0~2000 Blob(0x558c574d27e0 blob([0x235000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) put_ref 0x0~2000 Blob(0x558c574d2770 blob([0x237000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c573144d0) put_ref 0x0~2000 Blob(0x558c573144d0 blob([0x239000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) put_ref 0x0~2000 Blob(0x558c57566e00 blob([0x23b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574e0690) put_ref 0x0~2000 Blob(0x558c574e0690 blob([0x23d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) put_ref 0x0~2000 Blob(0x558c574d30a0 blob([0x23f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf82a0) put_ref 0x0~2000 Blob(0x558c54cf82a0 blob([0x241000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) put_ref 0x0~2000 Blob(0x558c574d3030 blob([0x243000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) put_ref 0x0~2000 Blob(0x558c575bd6c0 blob([0x245000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) put_ref 0x0~2000 Blob(0x558c5742ad90 blob([0x247000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3a40) put_ref 0x0~2000 Blob(0x558c574d3a40 blob([0x249000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) put_ref 0x0~2000 Blob(0x558c5742ae70 blob([0x24b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57396cb0) put_ref 0x0~2000 Blob(0x558c57396cb0 blob([0x24d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) put_ref 0x0~2000 Blob(0x558c575bc700 blob([0x24f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) put_ref 0x0~2000 Blob(0x558c575bc620 blob([0x251000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57396d20) put_ref 0x0~2000 Blob(0x558c57396d20 blob([0x253000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc540) put_ref 0x0~2000 Blob(0x558c575bc540 blob([0x255000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57396fc0) put_ref 0x0~2000 Blob(0x558c57396fc0 blob([0x257000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57397030) put_ref 0x0~2000 Blob(0x558c57397030 blob([0x259000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54e12388) dirty_range mark inline shard dirty 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574d28c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22b000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d750a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22d000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c577f39d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x22f000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57314310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x231000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c574d39d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x233000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x235000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c574d2770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x237000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c573144d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x239000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57566e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23b000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c574e0690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23d000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x23f000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c54cf82a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x241000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c574d3030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x243000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x245000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x247000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c574d3a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x249000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24b000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57396cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24d000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c575bc700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x24f000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c575bc620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x251000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c57396d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x253000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c575bc540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x255000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c57396fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x257000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c57397030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=2)))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x259000~2000] 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6ee4dfe7:::smithi13140796-47:e#42 = 0 2019-12-07T19:34:30.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582afb00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582afb00 onodes shared_blobs 0x558c54d3caf0,0x558c54d3d650,0x558c54d3d6c0,0x558c54d3d880,0x558c54e56700,0x558c5742ad20,0x558c5742ae00,0x558c5742aee0,0x558c5742afc0,0x558c5742b0a0,0x558c5742b180,0x558c574e0620,0x558c574f1810,0x558c574fdea0,0x558c574fdf80,0x558c575bc070,0x558c575bc150,0x558c575bc230,0x558c575bc310,0x558c575bc3f0,0x558c575bc4d0,0x558c575bc5b0,0x558c575bc690,0x558c575bc770 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558c54d3caf0 loaded (sbid 0x281e ref_map(0x22b000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558c54d3d650 loaded (sbid 0x282a ref_map(0x243000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558c54d3d6c0 loaded (sbid 0x2829 ref_map(0x241000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558c54d3d880 loaded (sbid 0x2828 ref_map(0x23f000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558c54e56700 loaded (sbid 0x281f ref_map(0x22d000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558c5742ad20 loaded (sbid 0x2820 ref_map(0x22f000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558c5742ae00 loaded (sbid 0x2821 ref_map(0x231000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558c5742aee0 loaded (sbid 0x2822 ref_map(0x233000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558c5742afc0 loaded (sbid 0x2823 ref_map(0x235000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558c5742b0a0 loaded (sbid 0x2824 ref_map(0x237000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558c5742b180 loaded (sbid 0x2825 ref_map(0x239000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558c574e0620 loaded (sbid 0x282c ref_map(0x247000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558c574f1810 loaded (sbid 0x282b ref_map(0x245000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558c574fdea0 loaded (sbid 0x2826 ref_map(0x23b000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558c574fdf80 loaded (sbid 0x2827 ref_map(0x23d000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558c575bc070 loaded (sbid 0x282d ref_map(0x249000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558c575bc150 loaded (sbid 0x282e ref_map(0x24b000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558c575bc230 loaded (sbid 0x282f ref_map(0x24d000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558c575bc310 loaded (sbid 0x2830 ref_map(0x24f000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558c575bc3f0 loaded (sbid 0x2831 ref_map(0x251000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558c575bc4d0 loaded (sbid 0x2832 ref_map(0x253000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558c575bc5b0 loaded (sbid 0x2833 ref_map(0x255000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558c575bc690 loaded (sbid 0x2834 ref_map(0x257000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558c575bc770 loaded (sbid 0x2835 ref_map(0x259000~2000=1))) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582afb00 allocated 0x[] released 0x[] 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582afb00 prepare 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582afb00 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582afb00 io_done 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 osd.7 53 dequeue_op 0x558c57efec60 finish 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1797 < min_flush_size 524288 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12280 had 1 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1820 < min_flush_size 524288 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1820 < min_flush_size 524288 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fd7ef~71c to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fd7ef 0x2fd7ef~71c to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2cd7ef 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7ef 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf0b and padding block with 0xf5 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fdf0b 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107460s (0.000000118s flush + 0.000107342s kv commit) 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582afb00> 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582afb00 kv_submitted 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582afb00 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582afb00 finishing 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582afb00 onodes 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582afb00 done 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582afb00 [] 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.642+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 53/28 ECSubWriteReply(tid=145, last_complete=53'69, committed=1, applied=1)) v2 -- 0x558c57951400 con 0x558c54c5bc00 2019-12-07T19:34:30.642+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.642+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 256 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c5829e700 con 0x558c54c5bc00 2019-12-07T19:34:30.642+0000 7f8f23c56700 15 osd.7 53 enqueue_peering_evt 1.6s2 epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:30.642+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s)) prio 255 cost 10 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 62.116337411s ub 62.116337411s int 16s) 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5853e000 seq 104 2019-12-07T19:34:30.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853e000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853e000 onodes shared_blobs 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853e000 allocated 0x[] released 0x[] 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e000 prepare 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853e000 2019-12-07T19:34:30.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e000 io_done 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fdf0b~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fdf0b 0x2fdf0b~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2cdf0b 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf0b 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x388 and padding block with 0xc78 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fe388 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000107771s (0.000000092s flush + 0.000107679s kv commit) 2019-12-07T19:34:30.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853e000> 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e000 kv_submitted 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853e000 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e000 finishing 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853e000 onodes 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853e000 done 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853e000 [] 2019-12-07T19:34:30.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.642+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 435 ==== MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=256, reqid=osd.1.0:254, at_version=0'0, trim_to=0'0, roll_forward_to=53'21)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57a44000 con 0x558c54c64000 2019-12-07T19:34:30.643+0000 7f8f24457700 15 osd.7 53 enqueue_op 0x558c573354a0 prio 127 cost 0 latency 0.000020 epoch 53 MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=256, reqid=osd.1.0:254, at_version=0'0, trim_to=0'0, roll_forward_to=53'21)) v2 2019-12-07T19:34:30.643+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=256, reqid=osd.1.0:254, at_version=0'0, trim_to=0'0, roll_forward_to=53'21)) v2) prio 127 cost 0 e53) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=256, reqid=osd.1.0:254, at_version=0'0, trim_to=0'0, roll_forward_to=53'21)) v2) prio 127 cost 0 e53) queued 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGOpItem(op=MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=256, reqid=osd.1.0:254, at_version=0'0, trim_to=0'0, roll_forward_to=53'21)) v2) prio 127 cost 0 e53) pg 0x558c5500d000 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 53 dequeue_op 0x558c573354a0 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=256, reqid=osd.1.0:254, at_version=0'0, trim_to=0'0, roll_forward_to=53'21)) v2 pg pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.0s3 53/18 ECSubWrite(tid=256, reqid=osd.1.0:254, at_version=0'0, trim_to=0'0, roll_forward_to=53'21)) v2 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=45'19 active mbc={} ps=[13~1,19~1]] append_log log((0'0,53'21], crt=45'19) [] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'20 (45'18) delete 1:64fe1d00:::smithi13140796-44:e by unknown.0.0:0 2019-12-07T19:34:30.635462+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[13~1,19~1]] rollforward: entry=53'21 (45'19) modify 1:64fe1d00:::smithi13140796-44:head by unknown.0.0:0 2019-12-07T19:34:30.635462+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[13~1,19~1]] append_log approx pg log length = 21 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:30.643+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:30.643+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c58c65500 seq 45 2019-12-07T19:34:30.643+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e#14 onode 0x558c57f34000 txc 0x558c58c65500 2019-12-07T19:34:30.643+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e#14 0x0 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c574fba40) put_ref 0x0~2000 Blob(0x558c574fba40 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c574fb9d0) put_ref 0x0~2000 Blob(0x558c574fb9d0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c574fdd50) put_ref 0x0~2000 Blob(0x558c574fdd50 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c574fb810) put_ref 0x0~2000 Blob(0x558c574fb810 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c574fbab0) put_ref 0x0~2000 Blob(0x558c574fbab0 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57efdce0) put_ref 0x0~2000 Blob(0x558c57efdce0 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57efddc0) put_ref 0x0~2000 Blob(0x558c57efddc0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57efde30) put_ref 0x0~2000 Blob(0x558c57efde30 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57efdea0) put_ref 0x0~2000 Blob(0x558c57efdea0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57efdf10) put_ref 0x0~2000 Blob(0x558c57efdf10 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57efdf80) put_ref 0x0~2000 Blob(0x558c57efdf80 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a000) put_ref 0x0~2000 Blob(0x558c57f3a000 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a070) put_ref 0x0~2000 Blob(0x558c57f3a070 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a0e0) put_ref 0x0~2000 Blob(0x558c57f3a0e0 blob([0x1e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a150) put_ref 0x0~2000 Blob(0x558c57f3a150 blob([0x1eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a1c0) put_ref 0x0~2000 Blob(0x558c57f3a1c0 blob([0x1ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a230) put_ref 0x0~2000 Blob(0x558c57f3a230 blob([0x1ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a2a0) put_ref 0x0~2000 Blob(0x558c57f3a2a0 blob([0x1f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a310) put_ref 0x0~2000 Blob(0x558c57f3a310 blob([0x1f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a380) put_ref 0x0~2000 Blob(0x558c57f3a380 blob([0x1f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a3f0) put_ref 0x0~2000 Blob(0x558c57f3a3f0 blob([0x1f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a460) put_ref 0x0~2000 Blob(0x558c57f3a460 blob([0x1f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a4d0) put_ref 0x0~1000 Blob(0x558c57f3a4d0 blob([0x1fb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore.extentmap(0x558c57f34108) dirty_range mark inline shard dirty 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574fba40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1cf000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c574fb9d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d1000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c574fdd50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d3000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574fb810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d5000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c574fbab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d7000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57efdce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1d9000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57efddc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1db000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57efde30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1dd000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57efdea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1df000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57efdf10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e1000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57efdf80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e3000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57f3a000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e5000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57f3a070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e7000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57f3a0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1e9000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57f3a150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1eb000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c57f3a1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1ed000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57f3a230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1ef000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57f3a2a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f1000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c57f3a310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f3000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c57f3a380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f5000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c57f3a3f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f7000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c57f3a460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1f9000~2000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~1000: 0x0~1000 Blob(0x558c57f3a4d0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=2)))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x1fb000~1000] 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.0s3_head 3#1:0064fe1d:::smithi13140796-44:e#14 = 0 2019-12-07T19:34:30.643+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c65500 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c65500 onodes shared_blobs 0x558c54d3d730,0x558c5742bab0,0x558c5742bb90,0x558c5742bc70,0x558c5742bd50,0x558c5742be30,0x558c5742bf10,0x558c574fa000,0x558c574fa0e0,0x558c574fa1c0,0x558c574fa2a0,0x558c574fa380,0x558c574fa460,0x558c574fa540,0x558c574fa620,0x558c574fa700,0x558c574fa7e0,0x558c574fa8c0,0x558c574fa9a0,0x558c574faa80,0x558c574fab60,0x558c57583340,0x558c57583570 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558c54d3d730 loaded (sbid 0x2801 ref_map(0x1cf000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558c5742bab0 loaded (sbid 0x2804 ref_map(0x1d5000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558c5742bb90 loaded (sbid 0x2805 ref_map(0x1d7000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558c5742bc70 loaded (sbid 0x2806 ref_map(0x1d9000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558c5742bd50 loaded (sbid 0x2807 ref_map(0x1db000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558c5742be30 loaded (sbid 0x2808 ref_map(0x1dd000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558c5742bf10 loaded (sbid 0x2809 ref_map(0x1df000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558c574fa000 loaded (sbid 0x280a ref_map(0x1e1000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558c574fa0e0 loaded (sbid 0x280b ref_map(0x1e3000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558c574fa1c0 loaded (sbid 0x280c ref_map(0x1e5000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558c574fa2a0 loaded (sbid 0x280d ref_map(0x1e7000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558c574fa380 loaded (sbid 0x280e ref_map(0x1e9000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558c574fa460 loaded (sbid 0x280f ref_map(0x1eb000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558c574fa540 loaded (sbid 0x2810 ref_map(0x1ed000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558c574fa620 loaded (sbid 0x2811 ref_map(0x1ef000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558c574fa700 loaded (sbid 0x2812 ref_map(0x1f1000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558c574fa7e0 loaded (sbid 0x2813 ref_map(0x1f3000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558c574fa8c0 loaded (sbid 0x2814 ref_map(0x1f5000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558c574fa9a0 loaded (sbid 0x2815 ref_map(0x1f7000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558c574faa80 loaded (sbid 0x2816 ref_map(0x1f9000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558c574fab60 loaded (sbid 0x2817 ref_map(0x1fb000~1000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558c57583340 loaded (sbid 0x2802 ref_map(0x1d1000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558c57583570 loaded (sbid 0x2803 ref_map(0x1d3000~2000=1))) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c65500 allocated 0x[] released 0x[] 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65500 prepare 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c65500 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65500 io_done 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 53 dequeue_op 0x558c573354a0 finish 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1773 < min_flush_size 524288 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f34000 had 1 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1796 < min_flush_size 524288 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1796 < min_flush_size 524288 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fe388~704 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fe388 0x2fe388~704 to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2ce388 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x388 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa8c and padding block with 0x574 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fea8c 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000130354s (0.000000068s flush + 0.000130286s kv commit) 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58c65500> 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65500 kv_submitted 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c65500 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65500 finishing 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c65500 onodes 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c65500 done 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c65500 [] 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.643+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.0s0 53/18 ECSubWriteReply(tid=256, last_complete=53'21, committed=1, applied=1)) v2 -- 0x558c575b8400 con 0x558c54c64000 2019-12-07T19:34:30.643+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 436 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e53/53) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c576df500 con 0x558c54c64000 2019-12-07T19:34:30.643+0000 7f8f24457700 15 osd.7 53 enqueue_peering_evt 1.0s3 epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:30.643+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e53) pg 0x558c5500d000 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:30.643+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~1] 2019-12-07T19:34:30.643+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c58c65800 seq 46 2019-12-07T19:34:30.643+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c65800 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c65800 onodes shared_blobs 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c65800 allocated 0x[] released 0x[] 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65800 prepare 2019-12-07T19:34:30.643+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c65800 2019-12-07T19:34:30.643+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65800 io_done 2019-12-07T19:34:30.643+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fea8c~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fea8c 0x2fea8c~47d to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2d0000 x_off 0x2cea8c 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa8c 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf09 and padding block with 0xf7 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x2fef09 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:30.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000091375s (0.000000071s flush + 0.000091304s kv commit) 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:30.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58c65800> 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65800 kv_submitted 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c65800 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65800 finishing 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c65800 onodes 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c65800 done 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c65800 [] 2019-12-07T19:34:30.643+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:30.643+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:30.644+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:30.644+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 176 ==== MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512e00 con 0x558c54c19800 2019-12-07T19:34:30.644+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c54e78420 prio 196 cost 0 latency 0.000014 epoch 53 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.644+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.644+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 177 ==== MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d000 con 0x558c54c19800 2019-12-07T19:34:30.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.644+0000 7f8f24c58700 15 osd.7 53 enqueue_op 0x558c57f3f8c0 prio 196 cost 0 latency 0.000006 epoch 53 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.644+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) 2019-12-07T19:34:30.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c57676000 2019-12-07T19:34:30.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.644+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c54e78420 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 pg pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) queued 2019-12-07T19:34:30.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGOpItem(op=MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2) prio 196 cost 0 e53) pg 0x558c548ab000 2019-12-07T19:34:30.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 53/28 ECSubWriteReply(tid=45, last_complete=53'40, committed=1, applied=1)) v2 2019-12-07T19:34:30.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=45 reqid=osd.7.0:43 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:30.644+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57f3f8c0 prio 196 cost 0 latency 0.000033 MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 pg pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] 2019-12-07T19:34:30.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:30.644+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWriteReply(1.4s0 53/41 ECSubWriteReply(tid=46, last_complete=53'10, committed=1, applied=1)) v2 2019-12-07T19:34:30.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:30.644+0000 7f8f0297e700 10 osd.7 53 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:30.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.644+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] try_finish_rmw: Op(1:276d5164:::smithi13140796-22:e v=0'0 tt=0'0 tid=46 reqid=osd.7.0:44 roll_forward_to=53'10 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:30.644+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:30.644+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:30.644+0000 7f8f06986700 10 osd.7 53 dequeue_op 0x558c57f3f8c0 finish 2019-12-07T19:34:30.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.656+0000 7f8f0998c700 15 osd.7 53 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:30.656+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) queued 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:30.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.5 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.4 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.1 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.3 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until readable_until[_ub] 62.092697193s (sent 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.cs3 pg_lease(ru 62.092697193s ub 70.092849942s int 16s) e53/53) v1 -- 0x558c5763c800 con 0x558c54c64000 2019-12-07T19:34:30.656+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.cs4 pg_lease(ru 62.092697193s ub 70.092849942s int 16s) e53/53) v1 -- 0x558c5795d200 con 0x558c54c19800 2019-12-07T19:34:30.656+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.cs5 pg_lease(ru 62.092697193s ub 70.092849942s int 16s) e53/53) v1 -- 0x558c5501ce00 con 0x558c54c5bc00 2019-12-07T19:34:30.656+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.cs2 pg_lease(ru 62.092697193s ub 70.092849942s int 16s) e53/53) v1 -- 0x558c58114a00 con 0x558c54c5b400 2019-12-07T19:34:30.656+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.cs1 pg_lease(ru 62.092697193s ub 70.092849942s int 16s) e53/53) v1 -- 0x558c58114c00 con 0x558c54c19c00 2019-12-07T19:34:30.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 282 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 70.092849942s) e53/53) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c57ede5a0 con 0x558c54c5b400 2019-12-07T19:34:30.656+0000 7f8f23c56700 15 osd.7 53 enqueue_peering_evt 1.cs0 epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.4 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 106 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 70.092849942s) e53/53) v1 ==== 42+0+0 (crc 0 0 0) 0x558c585f9c20 con 0x558c54c19c00 2019-12-07T19:34:30.656+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.4 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.656+0000 7f8f24457700 15 osd.7 53 enqueue_peering_evt 1.cs0 epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.5 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.5 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.4 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.4 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) pg 0x558c57676000 2019-12-07T19:34:30.656+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.4 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.5 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.5 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.4 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.1 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.3 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until readable_until[_ub] 62.092697193s (sent 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:30.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.5 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) pg 0x558c57676000 2019-12-07T19:34:30.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 257 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 70.092849942s) e53/53) v1 ==== 42+0+0 (crc 0 0 0) 0x558c57ede5a0 con 0x558c54c5bc00 2019-12-07T19:34:30.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 437 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 70.092849942s) e53/53) v1 ==== 42+0+0 (crc 0 0 0) 0x558c585f9c20 con 0x558c54c64000 2019-12-07T19:34:30.656+0000 7f8f23c56700 15 osd.7 53 enqueue_peering_evt 1.cs0 epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.3 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.5 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.3 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.656+0000 7f8f24457700 15 osd.7 53 enqueue_peering_evt 1.cs0 epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.1 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 178 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 70.092849942s) e53/53) v1 ==== 42+0+0 (crc 0 0 0) 0x558c575cd0e0 con 0x558c54c19800 2019-12-07T19:34:30.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.1 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f24c58700 15 osd.7 53 enqueue_peering_evt 1.cs0 epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.2 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.2 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.3 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.4 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.1 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.3 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until readable_until[_ub] 62.092697193s (sent 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:30.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.1 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.3 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) pg 0x558c57676000 2019-12-07T19:34:30.656+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.3 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.4 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.1 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.3 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until readable_until[_ub] 62.092697193s (sent 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:30.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.2 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.1 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) pg 0x558c57676000 2019-12-07T19:34:30.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.1 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.4 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.1 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.2 ruub 62.092697193s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.3 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until readable_until[_ub] 62.092697193s (sent 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:30.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:30.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.2 pg_lease_ack(ruub 70.092849942s)) prio 255 cost 10 e53) pg 0x558c57676000 2019-12-07T19:34:30.656+0000 7f8f06986700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MLeaseAck epoch 53 from osd.2 pg_lease_ack(ruub 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.4 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.1 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.2 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until peer osd.3 ruub 70.092849942s 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recalc_readable_until readable_until[_ub] 70.092849942s (sent 70.092849942s) 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:30.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:30.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:30.658+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 258 ==== pg_lease(1.6s2 pg_lease(ru 62.116337411s ub 70.116514081s int 16s) e53/53) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5501ce00 con 0x558c54c5bc00 2019-12-07T19:34:30.658+0000 7f8f23c56700 15 osd.7 53 enqueue_peering_evt 1.6s2 epoch_sent: 53 epoch_requested: 53 MLease epoch 53 from osd.3 pg_lease(ru 62.116337411s ub 70.116514081s int 16s) 2019-12-07T19:34:30.658+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLease epoch 53 from osd.3 pg_lease(ru 62.116337411s ub 70.116514081s int 16s)) prio 255 cost 10 e53) 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLease epoch 53 from osd.3 pg_lease(ru 62.116337411s ub 70.116514081s int 16s)) prio 255 cost 10 e53) queued 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 53 epoch_requested: 53 MLease epoch 53 from osd.3 pg_lease(ru 62.116337411s ub 70.116514081s int 16s)) prio 255 cost 10 e53) pg 0x558c54d5c000 2019-12-07T19:34:30.658+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 53 epoch_requested: 53 MLease epoch 53 from osd.3 pg_lease(ru 62.116337411s ub 70.116514081s int 16s) 2019-12-07T19:34:30.658+0000 7f8f05984700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] proc_lease pg_lease(ru 62.116337411s ub 70.116514081s int 16s) 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] peer_clock_delta_ub 0.022420715s -> ru 62.093916696s 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] proc_lease readable_until now 62.093916696s 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] peer_clock_delta_lb 0.021836395s -> ruub 70.094677686s 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] proc_lease readable_until_ub now 70.094677686s 2019-12-07T19:34:30.658+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.6s0 pg_lease_ack(ruub 70.116514081s) e53/53) v1 -- 0x558c576dd860 con 0x558c54c5bc00 2019-12-07T19:34:30.658+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:30.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:30.683+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:30.683+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:30.683+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 42 0x2000~1000 2019-12-07T19:34:30.683+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 91b3c8a9 2019-12-07T19:34:30.683+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 42 0x1b000~1000 2019-12-07T19:34:30.683+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 5434a025 2019-12-07T19:34:30.683+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.683+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:30.683+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 10 osd.7 53 tick_without_osd_lock 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f568000/0x40000000/0x1680000000, data 0xabf2d1/0xa96000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 25 osd.7 53 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875883+0000 last_rx_front 2019-12-07T19:34:27.875916+0000 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 25 osd.7 53 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875931+0000 last_rx_front 2019-12-07T19:34:27.875870+0000 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 25 osd.7 53 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875893+0000 last_rx_front 2019-12-07T19:34:27.875903+0000 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 25 osd.7 53 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875766+0000 last_rx_front 2019-12-07T19:34:27.875813+0000 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 25 osd.7 53 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875796+0000 last_rx_front 2019-12-07T19:34:27.875837+0000 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 20 osd.7 53 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 10 osd.7 53 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 20 osd.7 53 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:30.724+0000 7f8f1fbbe700 10 osd.7 53 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:30.733+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.745+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 16 ==== osd_ping(ping e53 up_from 13 ping_stamp 2019-12-07T19:34:30.745663+0000/54.217382031s send_stamp 54.217382031s delta_ub 0.036588053s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5763c800 con 0x558c552d1c00 2019-12-07T19:34:30.745+0000 7f8f24457700 20 osd.7 53 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036091876s,0.036588053s]) 2019-12-07T19:34:30.745+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e53 up_from 13 ping_stamp 2019-12-07T19:34:30.745663+0000/54.217382031s send_stamp 54.181290155s delta_ub 18446744073.673459740s) v5 -- 0x558c58114c00 con 0x558c552d1c00 2019-12-07T19:34:30.745+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 16 ==== osd_ping(ping e53 up_from 13 ping_stamp 2019-12-07T19:34:30.745663+0000/54.217382031s send_stamp 54.217382031s delta_ub 0.036588053s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c552d1800 2019-12-07T19:34:30.745+0000 7f8f24c58700 20 osd.7 53 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036045360s,0.036588053s]) 2019-12-07T19:34:30.745+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e53 up_from 13 ping_stamp 2019-12-07T19:34:30.745663+0000/54.217382031s send_stamp 54.181336671s delta_ub 18446744073.673506256s) v5 -- 0x558c58333e00 con 0x558c552d1800 2019-12-07T19:34:30.784+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.834+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.856+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 15 ==== osd_ping(ping e53 up_from 13 ping_stamp 2019-12-07T19:34:30.856649+0000/54.313572181s send_stamp 54.313572181s delta_ub 0.021792164s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5501ce00 con 0x558c54c19000 2019-12-07T19:34:30.856+0000 7f8f23c56700 20 osd.7 53 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021290124s,0.021792164s]) 2019-12-07T19:34:30.856+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e53 up_from 13 ping_stamp 2019-12-07T19:34:30.856649+0000/54.313572181s send_stamp 54.292282057s delta_ub 18446744073.688261492s) v5 -- 0x558c58114a00 con 0x558c54c19000 2019-12-07T19:34:30.856+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 15 ==== osd_ping(ping e53 up_from 13 ping_stamp 2019-12-07T19:34:30.856649+0000/54.313572181s send_stamp 54.313572181s delta_ub 0.021792164s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5501ce00 con 0x558c54c18800 2019-12-07T19:34:30.856+0000 7f8f23c56700 20 osd.7 53 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021205738s,0.021792164s]) 2019-12-07T19:34:30.856+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e53 up_from 13 ping_stamp 2019-12-07T19:34:30.856649+0000/54.313572181s send_stamp 54.292366443s delta_ub 18446744073.688345878s) v5 -- 0x558c57951400 con 0x558c54c18800 2019-12-07T19:34:30.884+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.933+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:30.984+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.034+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.084+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.134+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.184+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.234+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.284+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.319+0000 7f8f21451700 10 osd.7 53 tick 2019-12-07T19:34:31.319+0000 7f8f21451700 10 osd.7 53 do_waiters -- start 2019-12-07T19:34:31.319+0000 7f8f21451700 10 osd.7 53 do_waiters -- finish 2019-12-07T19:34:31.319+0000 7f8f21451700 20 osd.7 53 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:31.334+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.384+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.434+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.484+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.534+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.569+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9f400 0x558c552d6000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:31.569+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9f400 0x558c552d6000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:31.569+0000 7f8f24457700 10 osd.7 53 ms_handle_authentication new session 0x558c57bfa1a0 con 0x558c54c9f400 entity client.admin addr 2019-12-07T19:34:31.569+0000 7f8f24457700 10 osd.7 53 ms_handle_authentication session 0x558c57bfa1a0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:31.569+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3777706531 conn(0x558c54c9f400 0x558c552d6000 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4520 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:31.569+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4520 172.21.15.131:0/3777706531 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c585f9c20 con 0x558c54c9f400 2019-12-07T19:34:31.569+0000 7f8f164bb700 20 osd.7 53 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:31.569+0000 7f8f164bb700 10 osd.7 53 do_waiters -- start 2019-12-07T19:34:31.569+0000 7f8f164bb700 10 osd.7 53 do_waiters -- finish 2019-12-07T19:34:31.569+0000 7f8f164bb700 20 osd.7 53 _dispatch 0x558c585f9c20 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:31.569+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3777706531 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce1a40 con 0x558c54c9f400 2019-12-07T19:34:31.585+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1239722 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.592+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4520 172.21.15.131:0/3777706531 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c585f9e00 con 0x558c54c9f400 2019-12-07T19:34:31.592+0000 7f8f164bb700 20 osd.7 53 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:31.592+0000 7f8f164bb700 10 osd.7 53 do_waiters -- start 2019-12-07T19:34:31.592+0000 7f8f164bb700 10 osd.7 53 do_waiters -- finish 2019-12-07T19:34:31.592+0000 7f8f164bb700 20 osd.7 53 _dispatch 0x558c585f9e00 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:31.593+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/3777706531 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce1c00 con 0x558c54c9f400 2019-12-07T19:34:31.595+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3777706531 conn(0x558c54c9f400 msgr2=0x558c552d6000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 72 2019-12-07T19:34:31.595+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3777706531 conn(0x558c54c9f400 msgr2=0x558c552d6000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:31.595+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3777706531 conn(0x558c54c9f400 0x558c552d6000 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:31.595+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/3777706531 conn(0x558c54c9f400 0x558c552d6000 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:31.595+0000 7f8f164bb700 2 osd.7 53 ms_handle_reset con 0x558c54c9f400 session 0x558c57bfa1a0 2019-12-07T19:34:31.617+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 30 ==== osd_map(53..54 src has 1..54) v4 ==== 1286+0+0 (crc 0 0 0) 0x558c576c1680 con 0x558c5476d000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 osd.7 53 OSD::ms_dispatch: osd_map(53..54 src has 1..54) v4 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 osd.7 53 do_waiters -- start 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 osd.7 53 do_waiters -- finish 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 osd.7 53 _dispatch 0x558c576c1680 osd_map(53..54 src has 1..54) v4 2019-12-07T19:34:31.617+0000 7f8f164bb700 3 osd.7 53 handle_osd_map epochs [53,54], i have 53, src has [1,54] 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 osd.7 53 handle_osd_map got inc map for epoch 54 2019-12-07T19:34:31.617+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3294e826:::osdmap.53:0# 0x0~0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5829c230 blob([0xb48000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5829c2a0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809aa10 blob([!~1000,0xb49000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c579712d0 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3294e826:::osdmap.53:0# 0x0~1464 = 5220 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 osd.7 53 add_map_bl 53 5220 bytes 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 osd.7 53 got_full_map 54, nothing requested 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 osd.7 53 handle_osd_map pg_num_history pg_num_history(e54 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {54={}} 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 osd.7 53 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e54 [1,54] lci=[13,54]) 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c582b6300 seq 45 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c087972f:::inc_osdmap.54:0# key 0x7f7fffffffffffffffc08797'/!inc_osdmap.54!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c087972f:::inc_osdmap.54:0# 0x0~269 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1253 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c087972f:::inc_osdmap.54:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6300 1 blobs 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x97000~1000/1000,1000,0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x97000~1000] 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ea540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578de540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ea540 blob([0x97000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de540 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c583ea540) get_ref 0x0~269 Blob(0x558c583ea540 blob([0x97000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de540 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c583ea540) get_ref init 0x1000, 1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c583ea540 blob([0x97000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c578de540 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c578de558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b008) dirty_range mark inline shard dirty 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c087972f:::inc_osdmap.54:0# 0x0~269 = 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3874e826:::osdmap.54:0# key 0x7f7fffffffffffffff3874e8'&!osdmap.54!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3874e826:::osdmap.54:0# 0x0~1464 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1254 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3874e826:::osdmap.54:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6300 2 blobs 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x98000~2000/1000,2000,0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x98000~2000] 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809aaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a2770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809aaf0 blob([0x98000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2770 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c5809aaf0) get_ref 0x0~1000 Blob(0x558c5809aaf0 blob([0x98000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2770 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c5809aaf0) get_ref init 0x1000, 1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5809aaf0 blob([0x98000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580a2770 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c580a2788 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7880 blob([!~1000,0x99000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec230 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7880) get_ref 0x1000~464 Blob(0x558c580d7880 blob([!~1000,0x99000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c580ec230 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7880) get_ref init 0x2000, 1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c580d7880 blob([!~1000,0x99000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c580ec230 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c580ec248 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3874e826:::osdmap.54:0# 0x0~1464 = 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6300 0 blobs 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6300 0 blobs 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b6300 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b6300 onodes 0x558c53a91900,0x558c55276c80,0x558c5795af00,0x558c5795b180 shared_blobs 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b008) update #-1:c087972f:::inc_osdmap.54:0# 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b008) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c087972f:::inc_osdmap.54:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b288) update #-1:3874e826:::osdmap.54:0# 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b288) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3874e826:::osdmap.54:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b6300 allocated 0x[97000~3000] released 0x[] 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 freelist allocate 0x97000~3000 2019-12-07T19:34:31.617+0000 7f8f164bb700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6300 prepare 2019-12-07T19:34:31.617+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c582b6300 2019-12-07T19:34:31.617+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6300 aio_wait 2019-12-07T19:34:31.617+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b6300 2019-12-07T19:34:31.617+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6300 io_done 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4343 < min_flush_size 524288 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795af00 had 1 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795b180 had 1 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8859 < min_flush_size 524288 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8859 < min_flush_size 524288 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x2fef09~229b to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x2fef09 0x2fef09~229b to file(ino 16 size 0x300000 mtime 2019-12-07T19:34:30.628953+0000 allocated 300000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2d0000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _allocate len 0x11a4 from 0 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _allocate len 0x11a4 from 1 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd430000 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd430000~10000/10000,10000,acd430000 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x310000 to include allocated 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 63 (was clean) 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2cef09 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf09 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1a4 and padding block with 0xe5c 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x310000 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3011a4 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (63) on file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]), flushing log 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 63 len 0x4a crc 0xf383b20a) 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x3e000~1000 to file(ino 1 size 0x3e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x3e000 0x3e000~1000 to file(ino 1 size 0x3e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x3f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3e000 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x3f000 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:31.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000021 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 63 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000370048s (0.000003717s flush + 0.000366331s kv commit) 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b6300> 2019-12-07T19:34:31.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c57f53a40> 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6300 kv_submitted 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b6300 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c582b6300 osr 0x558c54ea4b40 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 43 0x1b000~1000 crc 7b545522 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 43 0x2000~1000 crc 5912821f 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2900 deferred_cleanup 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d2900 onodes 0x558c53a91900,0x558c55276c80,0x558c58294f00,0x558c58295680 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7 53 _committed_osd_maps 54..54 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809bac8 in 0x558c53a59340) _finish_write discard buffer(0x558c58b85560 space 0x558c5809bac8 0x0~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7 53 advance to epoch 54 (<= last 54 <= newest_map 54) 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b83828 in 0x558c53a59340) _finish_write discard buffer(0x558c58b85680 space 0x558c58b83828 0x1000~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b83908 in 0x558c53a59340) _finish_write discard buffer(0x558c58b855f0 space 0x558c58b83908 0x0~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2900 done 2019-12-07T19:34:31.617+0000 7f8f141a1700 7 osd.7 54 consume_map version 54 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829bb00 deferred_cleanup 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d2900 [] 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.8s5 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.0s3 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.1s1 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.9s4 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829bb00 deferred_cleanup 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.as5 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5829bb00 onodes 0x558c53a91900,0x558c55276c80,0x558c58295b80,0x558c5851e000 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.2s0 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829c398 in 0x558c53a59340) _finish_write discard buffer(0x558c582bafc0 space 0x558c5829c398 0x0~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.bs1 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829d9e8 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8eb40 space 0x558c5829d9e8 0x1000~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.3s4 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58b836d8 in 0x558c53a59340) _finish_write discard buffer(0x558c582bb050 space 0x558c58b836d8 0x0~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.cs0 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829bb00 done 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.4s0 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2c00 deferred_cleanup 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.5s4 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5829bb00 [] 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.ds2 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.es4 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.6s2 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2c00 deferred_cleanup 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.fs5 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d2c00 onodes 0x558c53a91900,0x558c55276c80,0x558c5851e280,0x558c5851e500 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 identify_splits_and_merges 1.7s4 e53 to e54 pg_nums {18=16} 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579712e8 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8ee10 space 0x558c579712e8 0x1000~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 54 prune_pg_created 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809ab78 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8ecf0 space 0x558c5809ab78 0x0~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829c2b8 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8ed80 space 0x558c5829c2b8 0x0~1000 writing nocache) 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:0.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2c00 done 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6300 deferred_queued 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d2c00 [] 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:31.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:1.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:2.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:3.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:4.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:31.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:5.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:6.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7:7.consume_map 54 (was 53) 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:31.617+0000 7f8f141a1700 10 osd.7 54 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:31.617+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.8s5 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.617+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.0s3 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c57380000 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.1s1 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.9s4 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c5500d000 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c5736a000 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 53 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.as5 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.2s0 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.bs1 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c5500a000 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.3s4 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 53 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.cs0 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.4s0 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 53 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.5s4 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.ds2 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 53 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c57676000 2019-12-07T19:34:31.618+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.es4 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.6s2 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.fs5 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f141a1700 15 osd.7 54 enqueue_peering_evt 1.7s4 epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f141a1700 7 osd.7 54 activate_map version 54 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~1]] state: Active advmap 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] on_active_advmap new removed_snaps [1a~1], snap_trimq now [1a~1] 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c57522000 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:31.618+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c57e8b000 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] state: Active: handling ActMap 2019-12-07T19:34:31.618+0000 7f8f0918b700 20 osd.7 54 agent_entry empty queue 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] Active: kicking snap trim 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c57383000 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] exit NotTrimming 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:31.618+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:177 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f0297e700 7 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] state: handle ActMap primary 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:178 2019-12-07T19:34:31.618+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c548ab000 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c57e88000 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 53 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c55141000 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~1]] state: Active advmap 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] on_active_advmap new removed_snaps [1a~1], snap_trimq now [1a~1] 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c54d5f000 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 53 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~1]] state: Active advmap 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 53 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] on_active_advmap new removed_snaps [1a~1], snap_trimq now [1a~1] 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] state: Active: handling ActMap 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 53 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] Active: kicking snap trim 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] exit NotTrimming 2019-12-07T19:34:31.618+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming 2019-12-07T19:34:31.618+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:141 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f0297e700 7 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] state: handle ActMap primary 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:142 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] state: Active: handling ActMap 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] Active: kicking snap trim 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c575ef000 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] exit NotTrimming 2019-12-07T19:34:31.618+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming/WaitReservation 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming: trimming 1a 2019-12-07T19:34:31.618+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f03980700 15 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:153 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f03980700 7 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] state: handle ActMap primary 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 54:179 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c5513e000 2019-12-07T19:34:31.618+0000 7f8f03980700 15 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 53:154 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 54 queueing pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] for snaptrim 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=54) prio 5 cost 1048576 e54) 2019-12-07T19:34:31.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 53 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:31.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 54:180 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=54) prio 5 cost 1048576 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c54d98000 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=54) prio 5 cost 1048576 e54) pg 0x558c57676000 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming: trimming 1a 2019-12-07T19:34:31.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 1a 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 54:143 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 54 queueing pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] for snaptrim 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=54) prio 5 cost 1048576 e54) 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57bcc800) upper_bound after SNA_1_000000000000001A_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.0_0000000000000001.C' 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 54:144 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57bcc800) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=54) prio 5 cost 1048576 e54) queued 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 NullEvt) prio 255 cost 10 e54) pg 0x558c54d5c000 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=54) prio 5 cost 1048576 e54) pg 0x558c548ab000 2019-12-07T19:34:31.618+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000001A_.0_0000000000000001.C) returns 0 SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 1a 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map: 54 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 53 -> 54 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~1]] SnapTrimmer state: adding snap 1a to purged_snaps 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 53 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] state: Started advmap 2019-12-07T19:34:31.618+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:31.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] plpg_on_pool_change 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] hit_set_clear 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] agent_stop 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~2]] SnapTrimmer state: purged_snaps now [e~1,13~1,19~2], snap_trimq now [] 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57bcc880) upper_bound after SNA_1_000000000000001A_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.0_0000000000000001.4' 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 54 advance_pg new pool opts old pool opts 2019-12-07T19:34:31.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c582adb00 seq 56 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] handle_activate_map: 54 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] activate_map 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57bcc880) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' 2019-12-07T19:34:31.618+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] activate_map: Not dirtying info: last_persisted is 53 while current is 54 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 NullEvt 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000001A_.0_0000000000000001.4) returns 0 SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:31.618+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:31.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:31.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582adb00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582adb00 onodes shared_blobs 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582adb00 allocated 0x[] released 0x[] 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~1]] SnapTrimmer state: adding snap 1a to purged_snaps 2019-12-07T19:34:31.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582adb00 prepare 2019-12-07T19:34:31.618+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582adb00 2019-12-07T19:34:31.618+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582adb00 io_done 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~2]] share_pg_info 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~2]] SnapTrimmer state: purged_snaps now [e~1,13~1,19~2], snap_trimq now [] 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c57bd2000 seq 45 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:31.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e54/54) v1 -- 0x558c5829fc00 con 0x558c54c64000 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3011a4~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3011a4 0x3011a4~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd2000 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd2000 onodes shared_blobs 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd2000 allocated 0x[] released 0x[] 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d11a4 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1a4 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x639 and padding block with 0x9c7 2019-12-07T19:34:31.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e54/54) v1 -- 0x558c582a0300 con 0x558c54c19800 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2000 prepare 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd2000 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2000 io_done 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~2]] share_pg_info 2019-12-07T19:34:31.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e54/54) v1 -- 0x558c582a0a00 con 0x558c54c5bc00 2019-12-07T19:34:31.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c58536e00 con 0x558c54c64000 2019-12-07T19:34:31.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e54/54) v1 -- 0x558c582a1100 con 0x558c54c5b400 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x301639 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c58537500 con 0x558c54c19800 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e54/54) v1 -- 0x558c582a1800 con 0x558c54c19c00 2019-12-07T19:34:31.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c58537c00 con 0x558c54c5bc00 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000084617s (0.000000102s flush + 0.000084515s kv commit) 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim ps=[e~1,13~1,19~2]] exit Trimming 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c58538300 con 0x558c54c5b400 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~2] 2019-12-07T19:34:31.618+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c58538a00 con 0x558c54c19c00 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f06986700 15 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] publish_stats_to_osd 54:181 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x301639~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] enter NotTrimming 2019-12-07T19:34:31.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x301639 0x301639~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d1639 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x639 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xace and padding block with 0x532 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,13~1,19~2]] exit Trimming 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:31.618+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] snap_trimmer complete 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~2] 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f0297e700 15 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] publish_stats_to_osd 54:145 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582adb00> 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582adb00 kv_submitted 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] enter NotTrimming 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582adb00 2019-12-07T19:34:31.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582adb00 finishing 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582adb00 onodes 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582adb00 done 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:31.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582adb00 [] 2019-12-07T19:34:31.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:31.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f0297e700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] snap_trimmer complete 2019-12-07T19:34:31.618+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: NotTrimming: trimming 1a 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[1a~1] ps=[e~1,13~1,19~1]] exit Trimming/WaitReservation 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 54:155 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:31.618+0000 7f8f0b990700 10 osd.7 54 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[1a~1] ps=[e~1,13~1,19~1]] for snaptrim 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=54) prio 5 cost 1048576 e54) 2019-12-07T19:34:31.618+0000 7f8f0b990700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~1] 2019-12-07T19:34:31.618+0000 7f8f0b990700 15 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] publish_stats_to_osd 54:156 2019-12-07T19:34:31.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=54) prio 5 cost 1048576 e54) queued 2019-12-07T19:34:31.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.618+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=54) prio 5 cost 1048576 e54) pg 0x558c57e88000 2019-12-07T19:34:31.618+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] snap_trimmer posting 2019-12-07T19:34:31.618+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 1a 2019-12-07T19:34:31.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.618+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.618+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:31.618+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57bcc900) upper_bound after SNA_1_000000000000001A_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.0_0000000000000001.2' 2019-12-07T19:34:31.618+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57bcc900) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' 2019-12-07T19:34:31.618+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000001A_.0_0000000000000001.2) returns 0 SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..,buffer::list(len=70, buffer::ptr(0~70 0x558c552382c0 in raw 0x558c552382c0 len 70 nref 1) ) 2019-12-07T19:34:31.618+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[1a~1] ps=[e~1,13~1,19~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:31.618+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~1]] SnapTrimmer state: adding snap 1a to purged_snaps 2019-12-07T19:34:31.619+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:31.619+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~2]] SnapTrimmer state: purged_snaps now [e~1,13~1,19~2], snap_trimq now [] 2019-12-07T19:34:31.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:31.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c58110900 seq 28 2019-12-07T19:34:31.619+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:31.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:31.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:31.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:31.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:31.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58110900 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58110900 onodes shared_blobs 2019-12-07T19:34:31.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58110900 allocated 0x[] released 0x[] 2019-12-07T19:34:31.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110900 prepare 2019-12-07T19:34:31.619+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58110900 2019-12-07T19:34:31.619+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110900 io_done 2019-12-07T19:34:31.619+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~2]] share_pg_info 2019-12-07T19:34:31.619+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:31.619+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c57378000 con 0x558c54c19800 2019-12-07T19:34:31.619+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:31.619+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:31.619+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 -- 0x558c57378e00 con 0x558c54c19c00 2019-12-07T19:34:31.619+0000 7f8f07988700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:31.619+0000 7f8f07988700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,13~1,19~2]] exit Trimming 2019-12-07T19:34:31.619+0000 7f8f07988700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~2] 2019-12-07T19:34:31.619+0000 7f8f07988700 15 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] publish_stats_to_osd 54:157 2019-12-07T19:34:31.619+0000 7f8f07988700 20 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] enter NotTrimming 2019-12-07T19:34:31.619+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:31.619+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:31.619+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] snap_trimmer complete 2019-12-07T19:34:31.619+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x301ace 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001580605s (0.000000099s flush + 0.001580506s kv commit) 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57bd2000> 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2000 kv_submitted 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd2000 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2000 finishing 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd2000 onodes 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2000 done 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd2000 [] 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:31.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x301ace~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x301ace 0x301ace~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d1ace 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xace 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf63 and padding block with 0x9d 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x301f63 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000104068s (0.000000082s flush + 0.000103986s kv commit) 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58110900> 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110900 kv_submitted 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58110900 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110900 finishing 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58110900 onodes 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58110900 done 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58110900 [] 2019-12-07T19:34:31.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 259 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:31.622+0000 7f8f23c56700 15 osd.7 54 enqueue_peering_evt 1.1s1 epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:31.622+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 260 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:31.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e54) pg 0x558c5736a000 2019-12-07T19:34:31.622+0000 7f8f23c56700 15 osd.7 54 enqueue_peering_evt 1.6s2 epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:31.622+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.622+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:31.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e54) pg 0x558c54d5c000 2019-12-07T19:34:31.622+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.622+0000 7f8f05984700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:31.622+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:31.622+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:31.622+0000 7f8f05984700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.622+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c58113500 seq 22 2019-12-07T19:34:31.622+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:31.622+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:31.622+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:31.622+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:31.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:31.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c575c4c00 seq 105 2019-12-07T19:34:31.622+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:31.622+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:31.622+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58113500 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:31.622+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58113500 onodes shared_blobs 2019-12-07T19:34:31.622+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58113500 allocated 0x[] released 0x[] 2019-12-07T19:34:31.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:31.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:31.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:31.622+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113500 prepare 2019-12-07T19:34:31.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:31.622+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58113500 2019-12-07T19:34:31.622+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113500 io_done 2019-12-07T19:34:31.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c4c00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c4c00 onodes shared_blobs 2019-12-07T19:34:31.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c4c00 allocated 0x[] released 0x[] 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4c00 prepare 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.622+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.622+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c4c00 2019-12-07T19:34:31.622+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4c00 io_done 2019-12-07T19:34:31.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3857645568 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997966) bluestore 86 GiB free (0.959884), bluefs_ratio 0.0399269 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x301f63~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x301f63 0x301f63~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d1f63 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf63 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3f8 and padding block with 0xc08 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3023f8 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000147126s (0.000000078s flush + 0.000147048s kv commit) 2019-12-07T19:34:31.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58113500> 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113500 kv_submitted 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58113500 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113500 finishing 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58113500 onodes 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58113500 done 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58113500 [] 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3023f8~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3023f8 0x3023f8~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d23f8 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3f8 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x88d and padding block with 0x773 2019-12-07T19:34:31.623+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x30288d 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000096692s (0.000000084s flush + 0.000096608s kv commit) 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c575c4c00> 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4c00 kv_submitted 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c4c00 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4c00 finishing 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c4c00 onodes 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c4c00 done 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c4c00 [] 2019-12-07T19:34:31.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:31.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 261 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:31.623+0000 7f8f23c56700 15 osd.7 54 enqueue_peering_evt 1.fs5 epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:31.623+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e54) pg 0x558c57383000 2019-12-07T19:34:31.623+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:31.623+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.623+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:31.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:31.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c582a9500 seq 24 2019-12-07T19:34:31.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:31.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:31.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:31.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:31.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:31.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a9500 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a9500 onodes shared_blobs 2019-12-07T19:34:31.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a9500 allocated 0x[] released 0x[] 2019-12-07T19:34:31.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9500 prepare 2019-12-07T19:34:31.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a9500 2019-12-07T19:34:31.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9500 io_done 2019-12-07T19:34:31.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x30288d~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x30288d 0x30288d~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d288d 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x88d 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd22 and padding block with 0x2de 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x302d22 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000104511s (0.000000049s flush + 0.000104462s kv commit) 2019-12-07T19:34:31.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582a9500> 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9500 kv_submitted 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a9500 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9500 finishing 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a9500 onodes 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a9500 done 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a9500 [] 2019-12-07T19:34:31.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 180 ==== MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=68, reqid=osd.2.0:68, at_version=54'24, trim_to=0'0, roll_forward_to=53'22)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c58532000 con 0x558c54c19800 2019-12-07T19:34:31.623+0000 7f8f24c58700 15 osd.7 54 enqueue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000037 epoch 54 MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=68, reqid=osd.2.0:68, at_version=54'24, trim_to=0'0, roll_forward_to=53'22)) v2 2019-12-07T19:34:31.623+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=68, reqid=osd.2.0:68, at_version=54'24, trim_to=0'0, roll_forward_to=53'22)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=68, reqid=osd.2.0:68, at_version=54'24, trim_to=0'0, roll_forward_to=53'22)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.623+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 179 ==== osd_map(54..54 src has 1..54) v4 ==== 661+0+0 (crc 0 0 0) 0x558c58526b40 con 0x558c54c19800 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.623+0000 7f8f1199c700 20 osd.7 54 OSD::ms_dispatch: osd_map(54..54 src has 1..54) v4 2019-12-07T19:34:31.623+0000 7f8f1199c700 10 osd.7 54 do_waiters -- start 2019-12-07T19:34:31.623+0000 7f8f1199c700 10 osd.7 54 do_waiters -- finish 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=68, reqid=osd.2.0:68, at_version=54'24, trim_to=0'0, roll_forward_to=53'22)) v2) prio 127 cost 0 e54) pg 0x558c55141000 2019-12-07T19:34:31.623+0000 7f8f1199c700 20 osd.7 54 _dispatch 0x558c58526b40 osd_map(54..54 src has 1..54) v4 2019-12-07T19:34:31.623+0000 7f8f1199c700 3 osd.7 54 handle_osd_map epochs [54,54], i have 54, src has [1,54] 2019-12-07T19:34:31.623+0000 7f8f1199c700 10 osd.7 54 no new maps here, dropping 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 54 dequeue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000080 MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=68, reqid=osd.2.0:68, at_version=54'24, trim_to=0'0, roll_forward_to=53'22)) v2 pg pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 54 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 53 -> 54 (as per caller) 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=68, reqid=osd.2.0:68, at_version=54'24, trim_to=0'0, roll_forward_to=53'22)) v2 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 snap_mapper.update_snaps 1:bca18876:::smithi13140796-30:1b 15,1b was 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 snap_mapper.get_snaps 1:bca18876:::smithi13140796-30:1b 15,1a,1b 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 snap_mapper.set_snaps 1:bca18876:::smithi13140796-30:1b 15,1b 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 snap_mapper.update_snaps rm SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 53'22 (0'0,53'22] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] append_log log((0'0,53'22], crt=53'22) [54'23 (53'19) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:31.624121+0000 0 snaps [15,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,54'24 (53'20) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:31.624121+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'23 (0'0,54'23] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'22 crt=53'22 lcod 53'22 active mbc={} ps=[e~1,13~1,19~1]] add_log_entry 54'23 (53'19) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:31.624121+0000 0 snaps [15,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'22 crt=53'22 lcod 53'22 active mbc={} ps=[e~1,13~1,19~1]] add_log_entry 54'24 (53'20) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:31.624121+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'22 crt=53'22 lcod 53'22 active mbc={} ps=[e~1,13~1,19~1]] append_log approx pg log length = 24 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'22 crt=53'22 lcod 53'22 active mbc={} ps=[e~1,13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.623+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'23, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58540000 seq 56 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_000000000000001A_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000054.00000000000000000023' <- 0000000054.00000000000000000023 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000054.00000000000000000024' <- 0000000054.00000000000000000024 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58540000 cost 8375 (1 ios * 4000 + 4375 bytes) 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58540000 onodes 0x558c54d96780,0x558c54e13900 shared_blobs 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96888) update 2#1:bca18876:::smithi13140796-30:1b# 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:1b# is 826 (412 bytes onode + 2 bytes spanning blobs + 412 bytes inline extents) 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 1451 (549 bytes onode + 2 bytes spanning blobs + 900 bytes inline extents) 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58540000 allocated 0x[] released 0x[] 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540000 prepare 2019-12-07T19:34:31.623+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58540000 2019-12-07T19:34:31.623+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540000 io_done 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=53'22 crt=53'22 lcod 53'22 active mbc={} ps=[e~1,13~1,19~1]] op_applied version 54'24 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5128 < min_flush_size 524288 2019-12-07T19:34:31.624+0000 7f8f0217d700 10 osd.7 54 dequeue_op 0x558c5740cc60 finish 2019-12-07T19:34:31.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96780 had 1 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13900 had 1 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5151 < min_flush_size 524288 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5151 < min_flush_size 524288 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x302d22~141f to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x302d22 0x302d22~141f to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d2d22 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd22 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x141 and padding block with 0xebf 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x304141 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000139752s (0.000000065s flush + 0.000139687s kv commit) 2019-12-07T19:34:31.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58540000> 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540000 kv_submitted 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58540000 2019-12-07T19:34:31.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540000 finishing 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58540000 onodes 0x558c54d96780,0x558c54e13900 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540000 done 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:31.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58540000 [] 2019-12-07T19:34:31.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.624+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 54/41 ECSubWriteReply(tid=68, last_complete=54'24, committed=1, applied=1)) v2 -- 0x558c58512c00 con 0x558c54c19800 2019-12-07T19:34:31.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 283 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c5737aa00 con 0x558c54c5b400 2019-12-07T19:34:31.625+0000 7f8f23c56700 15 osd.7 54 enqueue_peering_evt 1.bs1 epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:31.625+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e54) pg 0x558c575ef000 2019-12-07T19:34:31.625+0000 7f8f07187700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:31.625+0000 7f8f07187700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.625+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58523200 seq 46 2019-12-07T19:34:31.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:31.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 284 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c5737aa00 con 0x558c54c5b400 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58523200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58523200 onodes shared_blobs 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58523200 allocated 0x[] released 0x[] 2019-12-07T19:34:31.625+0000 7f8f23c56700 15 osd.7 54 enqueue_peering_evt 1.3s4 epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:31.625+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523200 prepare 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58523200 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523200 io_done 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e54) pg 0x558c5513e000 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f07187700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 4(0) info: 1.3s0( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:31.625+0000 7f8f07187700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x304141~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c58523500 seq 64 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x304141 0x304141~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d4141 2019-12-07T19:34:31.625+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x141 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5d6 and padding block with 0xa2a 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58523500 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58523500 onodes shared_blobs 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58523500 allocated 0x[] released 0x[] 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523500 prepare 2019-12-07T19:34:31.625+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58523500 2019-12-07T19:34:31.625+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523500 io_done 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 181 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c19800 2019-12-07T19:34:31.625+0000 7f8f24c58700 15 osd.7 54 enqueue_peering_evt 1.5s4 epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:31.625+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3045d6 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000158127s (0.000000117s flush + 0.000158010s kv commit) 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e54) pg 0x558c57522000 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3045d6~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3045d6 0x3045d6~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d45d6 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5d6 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa6b and padding block with 0x595 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c58540300 seq 21 2019-12-07T19:34:31.625+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58523200> 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523200 kv_submitted 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58523200 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523200 finishing 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58523200 onodes 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58540300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58523200 done 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58540300 onodes shared_blobs 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58540300 allocated 0x[] released 0x[] 2019-12-07T19:34:31.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58523200 [] 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540300 prepare 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58540300 2019-12-07T19:34:31.625+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540300 io_done 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x304a6b 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119577s (0.000000093s flush + 0.000119484s kv commit) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x304a6b~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x304a6b 0x304a6b~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d4a6b 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa6b 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf00 and padding block with 0x100 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58523500> 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523500 kv_submitted 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58523500 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523500 finishing 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58523500 onodes 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58523500 done 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58523500 [] 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x304f00 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122720s (0.000000057s flush + 0.000122663s kv commit) 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58540300> 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540300 kv_submitted 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58540300 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540300 finishing 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58540300 onodes 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540300 done 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58540300 [] 2019-12-07T19:34:31.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.625+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 438 ==== pg_info2(1.as5 1.as5 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:31.626+0000 7f8f24457700 15 osd.7 54 enqueue_peering_evt 1.as5 epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:31.626+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e54) pg 0x558c57e8b000 2019-12-07T19:34:31.626+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:31.626+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.626+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:31.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:31.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c5829a300 seq 19 2019-12-07T19:34:31.626+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:31.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:31.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:31.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:31.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:31.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5829a300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5829a300 onodes shared_blobs 2019-12-07T19:34:31.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5829a300 allocated 0x[] released 0x[] 2019-12-07T19:34:31.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a300 prepare 2019-12-07T19:34:31.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5829a300 2019-12-07T19:34:31.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a300 io_done 2019-12-07T19:34:31.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:31.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x304f00~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x304f00 0x304f00~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d4f00 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf00 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x395 and padding block with 0xc6b 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x305395 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000002 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000152326s (0.000000062s flush + 0.000152264s kv commit) 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5829a300> 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a300 kv_submitted 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5829a300 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a300 finishing 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5829a300 onodes 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829a300 done 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5829a300 [] 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.627+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.627+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 439 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:31.627+0000 7f8f24457700 15 osd.7 54 enqueue_peering_evt 1.9s4 epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:31.627+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.627+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.627+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.627+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.627+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e54) pg 0x558c5500a000 2019-12-07T19:34:31.627+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.9s0( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:31.627+0000 7f8f04181700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.627+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:31.627+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:31.627+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57956300 seq 77 2019-12-07T19:34:31.627+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:31.627+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:31.627+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:31.627+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:31.627+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:31.627+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57956300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.627+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57956300 onodes shared_blobs 2019-12-07T19:34:31.627+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57956300 allocated 0x[] released 0x[] 2019-12-07T19:34:31.627+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956300 prepare 2019-12-07T19:34:31.627+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57956300 2019-12-07T19:34:31.627+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956300 io_done 2019-12-07T19:34:31.627+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x305395~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x305395 0x305395~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d5395 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x395 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x82a and padding block with 0x7d6 2019-12-07T19:34:31.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 263 ==== MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=146, reqid=client.4540.0:396, at_version=54'14, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2944+0+0 (crc 0 0 0) 0x558c579e2800 con 0x558c54c5bc00 2019-12-07T19:34:31.627+0000 7f8f23c56700 15 osd.7 54 enqueue_op 0x558c57effb80 prio 127 cost 0 latency 0.000037 epoch 54 MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=146, reqid=client.4540.0:396, at_version=54'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:31.627+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=146, reqid=client.4540.0:396, at_version=54'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.627+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 262 ==== osd_map(54..54 src has 1..54) v4 ==== 661+0+0 (crc 0 0 0) 0x558c576c18c0 con 0x558c54c5bc00 2019-12-07T19:34:31.627+0000 7f8f1199c700 20 osd.7 54 OSD::ms_dispatch: osd_map(54..54 src has 1..54) v4 2019-12-07T19:34:31.627+0000 7f8f1199c700 10 osd.7 54 do_waiters -- start 2019-12-07T19:34:31.627+0000 7f8f1199c700 10 osd.7 54 do_waiters -- finish 2019-12-07T19:34:31.627+0000 7f8f1199c700 20 osd.7 54 _dispatch 0x558c576c18c0 osd_map(54..54 src has 1..54) v4 2019-12-07T19:34:31.627+0000 7f8f1199c700 3 osd.7 54 handle_osd_map epochs [54,54], i have 54, src has [1,54] 2019-12-07T19:34:31.627+0000 7f8f1199c700 10 osd.7 54 no new maps here, dropping 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x30582a 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000274985s (0.000000092s flush + 0.000274893s kv commit) 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57956300> 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956300 kv_submitted 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57956300 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956300 finishing 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57956300 onodes 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57956300 done 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57956300 [] 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 440 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=146, reqid=client.4540.0:396, at_version=54'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.627+0000 7f8f24457700 15 osd.7 54 enqueue_peering_evt 1.0s3 epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.627+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=146, reqid=client.4540.0:396, at_version=54'14, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) pg 0x558c57383000 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e54) pg 0x558c5500d000 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 54 dequeue_op 0x558c57effb80 prio 127 cost 0 latency 0.000246 MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=146, reqid=client.4540.0:396, at_version=54'14, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 54 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 53 -> 54 (as per caller) 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=146, reqid=client.4540.0:396, at_version=54'14, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 snap_mapper.add_oid 1:fc3a86a2:::smithi13140796-33:1b 15,1b 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:31.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c578a6600 seq 47 2019-12-07T19:34:31.627+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a6600 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:fc3a86a2:::smithi13140796-33:1b got.empty() 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a6600 onodes shared_blobs 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a6600 allocated 0x[] released 0x[] 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 snap_mapper.set_snaps 1:fc3a86a2:::smithi13140796-33:1b 15,1b 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6600 prepare 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a6600 2019-12-07T19:34:31.627+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6600 io_done 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x30582a~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x30582a 0x30582a~495 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d582a 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x82a 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcbf and padding block with 0x341 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 29'12 (0'0,29'12] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,29'12], crt=29'12) [54'13 (21'6) clone 1:fc3a86a2:::smithi13140796-33:1b by unknown.0.0:0 2019-12-07T19:33:58.707684+0000 0 snaps [1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,54'14 (21'6) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:396 2019-12-07T19:34:31.626553+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 0, new_object: 0] 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'13 (0'0,54'13] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=29'12 crt=29'12 lcod 29'12 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 54'13 (21'6) clone 1:fc3a86a2:::smithi13140796-33:1b by unknown.0.0:0 2019-12-07T19:33:58.707684+0000 0 snaps [1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=29'12 crt=29'12 lcod 29'12 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 54'14 (21'6) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:396 2019-12-07T19:34:31.626553+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 0, new_object: 0 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=29'12 crt=29'12 lcod 29'12 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 14 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=29'12 crt=29'12 lcod 29'12 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.627+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x305cbf 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000103153s (0.000000064s flush + 0.000103089s kv commit) 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:31.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57958900 seq 25 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c578a6600> 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6600 kv_submitted 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a6600 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6600 finishing 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a6600 onodes 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a6600 done 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a6600 [] 2019-12-07T19:34:31.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:fc3a86a2:::smithi13140796-33:1b# key 0x858000000000000001fc3a86a221736d'ithi13140796-33!='0x000000000000001bffffffffffffffff'o' 2019-12-07T19:34:31.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:31.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# -> 5#1:fc3a86a2:::smithi13140796-33:1b# 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1255 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.onode(0x558c57f35180).flush flush done 2019-12-07T19:34:31.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# 0x0 2019-12-07T19:34:31.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# -> 5#1:fc3a86a2:::smithi13140796-33:1b# 0x0~21000 -> 0x0~21000 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x0~2000: 0x0~2000 Blob(0x558c57c42c40 blob([0x859000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c429a0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10656 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c57c42c40 blob([0x859000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c429a0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c57c42c40 blob([0x859000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=1)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c55060000 blob([0x859000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c55060000) get_ref 0x0~2000 Blob(0x558c55060000 blob([0x859000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c55060000) get_ref init 0x2000, 1000 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x0~2000: 0x0~2000 Blob(0x558c55060000 blob([0x859000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57c42b60 blob([0x85b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43ea0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10657 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c57c42b60 blob([0x85b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43ea0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c57c42b60 blob([0x85b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=1)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c575676c0 blob([0x85b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c575676c0) get_ref 0x0~2000 Blob(0x558c575676c0 blob([0x85b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c575676c0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c575676c0 blob([0x85b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57897f10 blob([0x85d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897f80 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10658 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c57897f10 blob([0x85d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897f80 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c57897f10 blob([0x85d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=1)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c5829caf0 blob([0x85d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c5829caf0) get_ref 0x0~2000 Blob(0x558c5829caf0 blob([0x85d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c5829caf0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c5829caf0 blob([0x85d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57b97730 blob([0x85f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b977a0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10659 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c57b97730 blob([0x85f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b977a0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c57b97730 blob([0x85f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=1)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c57567730 blob([0x85f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c57567730) get_ref 0x0~2000 Blob(0x558c57567730 blob([0x85f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c57567730) get_ref init 0x2000, 1000 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57567730 blob([0x85f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57b97880 blob([0x861000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b978f0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10660 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c57b97880 blob([0x861000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b978f0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c57b97880 blob([0x861000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=1)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c57e63dc0 blob([0x861000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c57e63dc0) get_ref 0x0~2000 Blob(0x558c57e63dc0 blob([0x861000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.blob(0x558c57e63dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c57e63dc0 blob([0x861000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=2)))) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57b97960 blob([0x863000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b979d0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10661 2019-12-07T19:34:31.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c57b97960 blob([0x863000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b979d0 sbid 0x0)) 2019-12-07T19:34:31.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c57b97960 blob([0x863000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c580ecd20 blob([0x863000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c580ecd20) get_ref 0x0~2000 Blob(0x558c580ecd20 blob([0x863000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c580ecd20) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c580ecd20 blob([0x863000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0xc000~2000: 0x0~2000 Blob(0x558c5750ce00 blob([0x865000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ce70 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10662 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750ce00 blob([0x865000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ce70 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750ce00 blob([0x865000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c57566fc0 blob([0x865000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c57566fc0) get_ref 0x0~2000 Blob(0x558c57566fc0 blob([0x865000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c57566fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57566fc0 blob([0x865000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0xe000~2000: 0x0~2000 Blob(0x558c5750cee0 blob([0x867000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cf50 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10663 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750cee0 blob([0x867000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cf50 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750cee0 blob([0x867000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c54d74d20 blob([0x867000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74d20) get_ref 0x0~2000 Blob(0x558c54d74d20 blob([0x867000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74d20) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c54d74d20 blob([0x867000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x10000~2000: 0x0~2000 Blob(0x558c5750cfc0 blob([0x869000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d030 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10664 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750cfc0 blob([0x869000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d030 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750cfc0 blob([0x869000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c5792d0a0 blob([0x869000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5792d0a0) get_ref 0x0~2000 Blob(0x558c5792d0a0 blob([0x869000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5792d0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5792d0a0 blob([0x869000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x12000~2000: 0x0~2000 Blob(0x558c5750d0a0 blob([0x86b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d110 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10665 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d0a0 blob([0x86b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d110 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d0a0 blob([0x86b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c57ed1f80 blob([0x86b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c57ed1f80) get_ref 0x0~2000 Blob(0x558c57ed1f80 blob([0x86b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c57ed1f80) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57ed1f80 blob([0x86b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x14000~2000: 0x0~2000 Blob(0x558c5750d180 blob([0x86d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d1f0 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10666 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d180 blob([0x86d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d1f0 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d180 blob([0x86d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c5792df10 blob([0x86d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5792df10) get_ref 0x0~2000 Blob(0x558c5792df10 blob([0x86d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5792df10) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c5792df10 blob([0x86d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x16000~2000: 0x0~2000 Blob(0x558c5750d260 blob([0x86f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d2d0 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10667 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d260 blob([0x86f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d2d0 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d260 blob([0x86f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c5829dd50 blob([0x86f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5829dd50) get_ref 0x0~2000 Blob(0x558c5829dd50 blob([0x86f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5829dd50) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c5829dd50 blob([0x86f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x18000~2000: 0x0~2000 Blob(0x558c5750d340 blob([0x871000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d3b0 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10668 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d340 blob([0x871000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d3b0 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d340 blob([0x871000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c574d3f80 blob([0x871000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c574d3f80) get_ref 0x0~2000 Blob(0x558c574d3f80 blob([0x871000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c574d3f80) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c574d3f80 blob([0x871000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c5750d420 blob([0x873000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d490 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10669 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d420 blob([0x873000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d490 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d420 blob([0x873000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c580d7ce0 blob([0x873000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c580d7ce0) get_ref 0x0~2000 Blob(0x558c580d7ce0 blob([0x873000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c580d7ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c580d7ce0 blob([0x873000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c5750d500 blob([0x875000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d570 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10670 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d500 blob([0x875000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d570 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d500 blob([0x875000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c574d23f0 blob([0x875000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c574d23f0) get_ref 0x0~2000 Blob(0x558c574d23f0 blob([0x875000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c574d23f0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c574d23f0 blob([0x875000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c5750d5e0 blob([0x877000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d650 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10671 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d5e0 blob([0x877000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d650 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d5e0 blob([0x877000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c5840f650 blob([0x877000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5840f650) get_ref 0x0~2000 Blob(0x558c5840f650 blob([0x877000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5840f650) get_ref init 0x2000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c5840f650 blob([0x877000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup src 0x20000~1000: 0x0~1000 Blob(0x558c5750d6c0 blob([0x879000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5750d730 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10672 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared Blob(0x558c5750d6c0 blob([0x879000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5750d730 sbid 0x0)) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) make_blob_shared now Blob(0x558c5750d6c0 blob([0x879000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=1)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup new Blob(0x558c5792df80 blob([0x879000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5792df80) get_ref 0x0~1000 Blob(0x558c5792df80 blob([0x879000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.blob(0x558c5792df80) get_ref init 0x1000, 1000 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dup dst 0x20000~1000: 0x0~1000 Blob(0x558c5792df80 blob([0x879000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=2)))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795a388) dirty_range mark inline shard dirty 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# -> 5#1:fc3a86a2:::smithi13140796-33:1b# = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# snapset 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# snapset = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# 1 keys 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# 1 keys = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# hinfo_key (42 bytes) 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) get_onode oid 5#1:fc3a86a2:::smithi13140796-33:head#e key 0x858000000000000001fc3a86a221736d'ithi13140796-33!='0xfffffffffffffffe000000000000000e'o' 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.fs5_head 0x558c57f105a0) r -2 v.len 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# -> 5#1:fc3a86a2:::smithi13140796-33:head#e 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# -> 5#1:fc3a86a2:::smithi13140796-33:head#e = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1256 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2 keys 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2 keys = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.F3C51654.1b.smithi13140796-33..' <- OBJ_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.F3C51654.1b.smithi13140796-33..' <- SNA_1_0000000000000015_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.5_0000000000000001.F3C51654.1b.smithi13140796-33..' <- SNA_1_000000000000001B_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000054.00000000000000000013' <- 0000000054.00000000000000000013 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000054.00000000000000000014' <- 0000000054.00000000000000000014 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57958900 cost 8133 (1 ios * 4000 + 4133 bytes) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57958900 onodes 0x558c5795a280,0x558c5795a500,0x558c57f35180 shared_blobs 0x558c5750ce70,0x558c5750cf50,0x558c5750d030,0x558c5750d110,0x558c5750d1f0,0x558c5750d2d0,0x558c5750d3b0,0x558c5750d490,0x558c5750d570,0x558c5750d650,0x558c5750d730,0x558c57897f80,0x558c57b977a0,0x558c57b978f0,0x558c57b979d0,0x558c57c429a0,0x558c57c43ea0 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795a388) update 5#1:fc3a86a2:::smithi13140796-33:1b# 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795a388) update inline shard 459 bytes from 17 extents 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:1b# is 877 (412 bytes onode + 2 bytes spanning blobs + 463 bytes inline extents) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795a608) update 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795a608) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:head# is 483 (475 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) update 5#1:fc3a86a2:::smithi13140796-33:head#e 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore.extentmap(0x558c57f35288) update inline shard 459 bytes from 17 extents 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:head#e is 950 (485 bytes onode + 2 bytes spanning blobs + 463 bytes inline extents) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a6 is 11 SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a7 is 11 SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a8 is 11 SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a9 is 11 SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29aa is 11 SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ab is 11 SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ac is 11 SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ad is 11 SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ae is 11 SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29af is 11 SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b0 is 11 SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a2 is 11 SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a3 is 11 SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a4 is 11 SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a5 is 11 SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a0 is 11 SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a1 is 11 SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=2))) 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57958900 allocated 0x[] released 0x[] 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958900 prepare 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57958900 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958900 io_done 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=29'12 crt=29'12 lcod 29'12 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 54'14 2019-12-07T19:34:31.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.628+0000 7f8f0117b700 10 osd.7 54 dequeue_op 0x558c57effb80 finish 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5126 < min_flush_size 524288 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795a280 had 1 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795a500 had 1 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35180 had 1 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5149 < min_flush_size 524288 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5149 < min_flush_size 524288 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x305cbf~141d to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x305cbf 0x305cbf~141d to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d5cbf 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcbf 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdc and padding block with 0xf24 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3070dc 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000153891s (0.000000113s flush + 0.000153778s kv commit) 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57958900> 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958900 kv_submitted 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57958900 2019-12-07T19:34:31.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958900 finishing 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57958900 onodes 0x558c5795a280,0x558c5795a500,0x558c57f35180 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57958900 done 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:31.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57958900 [] 2019-12-07T19:34:31.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.628+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.628+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 54/41 ECSubWriteReply(tid=146, last_complete=54'14, committed=1, applied=1)) v2 -- 0x558c58513c00 con 0x558c54c5bc00 2019-12-07T19:34:31.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.628+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 441 ==== osd_map(54..54 src has 1..54) v4 ==== 661+0+0 (crc 0 0 0) 0x558c5853d200 con 0x558c54c64000 2019-12-07T19:34:31.628+0000 7f8f1199c700 20 osd.7 54 OSD::ms_dispatch: osd_map(54..54 src has 1..54) v4 2019-12-07T19:34:31.628+0000 7f8f1199c700 10 osd.7 54 do_waiters -- start 2019-12-07T19:34:31.628+0000 7f8f1199c700 10 osd.7 54 do_waiters -- finish 2019-12-07T19:34:31.628+0000 7f8f1199c700 20 osd.7 54 _dispatch 0x558c5853d200 osd_map(54..54 src has 1..54) v4 2019-12-07T19:34:31.628+0000 7f8f1199c700 3 osd.7 54 handle_osd_map epochs [54,54], i have 54, src has [1,54] 2019-12-07T19:34:31.628+0000 7f8f1199c700 10 osd.7 54 no new maps here, dropping 2019-12-07T19:34:31.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 442 ==== MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=257, reqid=osd.1.0:257, at_version=54'51, trim_to=0'0, roll_forward_to=53'49)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c58157000 con 0x558c54c64000 2019-12-07T19:34:31.628+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c57334840 prio 127 cost 0 latency 0.000025 epoch 54 MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=257, reqid=osd.1.0:257, at_version=54'51, trim_to=0'0, roll_forward_to=53'49)) v2 2019-12-07T19:34:31.628+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=257, reqid=osd.1.0:257, at_version=54'51, trim_to=0'0, roll_forward_to=53'49)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=257, reqid=osd.1.0:257, at_version=54'51, trim_to=0'0, roll_forward_to=53'49)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=257, reqid=osd.1.0:257, at_version=54'51, trim_to=0'0, roll_forward_to=53'49)) v2) prio 127 cost 0 e54) pg 0x558c57380000 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 54 dequeue_op 0x558c57334840 prio 127 cost 0 latency 0.000069 MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=257, reqid=osd.1.0:257, at_version=54'51, trim_to=0'0, roll_forward_to=53'49)) v2 pg pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 54 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 53 -> 54 (as per caller) 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=257, reqid=osd.1.0:257, at_version=54'51, trim_to=0'0, roll_forward_to=53'49)) v2 2019-12-07T19:34:31.629+0000 7f8f04982700 20 snap_mapper.update_snaps 1:10779340:::smithi13140796-32:1b 15,1b was 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:1b 15,1a,1b 2019-12-07T19:34:31.629+0000 7f8f04982700 20 snap_mapper.set_snaps 1:10779340:::smithi13140796-32:1b 15,1b 2019-12-07T19:34:31.629+0000 7f8f04982700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:31.629+0000 7f8f04982700 20 snap_mapper.update_snaps rm SNA_1_000000000000001A_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 53'49 (0'0,53'49] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] append_log log((0'0,53'49], crt=53'49) [54'50 (53'46) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:31.627903+0000 0 snaps [15,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,54'51 (53'47) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:31.627903+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'50 (0'0,54'50] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'49 crt=53'49 lcod 53'49 active mbc={} ps=[e~1,13~1,19~1]] add_log_entry 54'50 (53'46) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:31.627903+0000 0 snaps [15,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'49 crt=53'49 lcod 53'49 active mbc={} ps=[e~1,13~1,19~1]] add_log_entry 54'51 (53'47) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:31.627903+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'49 crt=53'49 lcod 53'49 active mbc={} ps=[e~1,13~1,19~1]] append_log approx pg log length = 51 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'49 crt=53'49 lcod 53'49 active mbc={} ps=[e~1,13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:31.629+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.629+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'50, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58c65b00 seq 46 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_000000000000001A_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000054.00000000000000000050' <- 0000000054.00000000000000000050 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000054.00000000000000000051' <- 0000000054.00000000000000000051 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c65b00 cost 8375 (1 ios * 4000 + 4375 bytes) 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c65b00 onodes 0x558c5840c500,0x558c5840c780 shared_blobs 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c608) update 5#1:10779340:::smithi13140796-32:1b# 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:1b# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 483 (475 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c65b00 allocated 0x[] released 0x[] 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 prepare 2019-12-07T19:34:31.629+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c65b00 2019-12-07T19:34:31.629+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 io_done 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=53'49 crt=53'49 lcod 53'49 active mbc={} ps=[e~1,13~1,19~1]] op_applied version 54'51 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3761 < min_flush_size 524288 2019-12-07T19:34:31.629+0000 7f8f04982700 10 osd.7 54 dequeue_op 0x558c57334840 finish 2019-12-07T19:34:31.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.629+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c500 had 1 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3784 < min_flush_size 524288 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3784 < min_flush_size 524288 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3070dc~ec8 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3070dc 0x3070dc~ec8 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2e0000 x_off 0x2d70dc 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdc 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfa4 and padding block with 0x5c 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x307fa4 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000142402s (0.000000082s flush + 0.000142320s kv commit) 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58c65b00> 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 kv_submitted 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c65b00 2019-12-07T19:34:31.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 finishing 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c65b00 onodes 0x558c5840c500,0x558c5840c780 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c65b00 done 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:31.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c65b00 [] 2019-12-07T19:34:31.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.629+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 443 ==== MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=258, reqid=osd.1.0:258, at_version=54'35, trim_to=0'0, roll_forward_to=53'33)) v2 ==== 3506+0+0 (crc 0 0 0) 0x558c574a4000 con 0x558c54c64000 2019-12-07T19:34:31.629+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c57334c60 prio 127 cost 0 latency 0.000032 epoch 54 MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=258, reqid=osd.1.0:258, at_version=54'35, trim_to=0'0, roll_forward_to=53'33)) v2 2019-12-07T19:34:31.629+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=258, reqid=osd.1.0:258, at_version=54'35, trim_to=0'0, roll_forward_to=53'33)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=258, reqid=osd.1.0:258, at_version=54'35, trim_to=0'0, roll_forward_to=53'33)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=258, reqid=osd.1.0:258, at_version=54'35, trim_to=0'0, roll_forward_to=53'33)) v2) prio 127 cost 0 e54) pg 0x558c54d5f000 2019-12-07T19:34:31.629+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 54/41 ECSubWriteReply(tid=257, last_complete=54'51, committed=1, applied=1)) v2 -- 0x558c55250800 con 0x558c54c64000 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 54 dequeue_op 0x558c57334c60 prio 127 cost 0 latency 0.000119 MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=258, reqid=osd.1.0:258, at_version=54'35, trim_to=0'0, roll_forward_to=53'33)) v2 pg pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] 2019-12-07T19:34:31.629+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=258, reqid=osd.1.0:258, at_version=54'35, trim_to=0'0, roll_forward_to=53'33)) v2 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 snap_mapper.update_snaps 1:ecc72bdb:::smithi13140796-10:1b 1b was 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:1b 1a,1b 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:1b 1b 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 snap_mapper.update_snaps rm SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 53'33 (0'0,53'33] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] append_log log((0'0,53'33], crt=53'33) [54'34 (51'26) modify 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:31.630165+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,54'35 (53'31) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:31.630165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'34 (0'0,54'34] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'33 crt=53'33 lcod 53'33 active mbc={} ps=[e~1,13~1,19~1]] add_log_entry 54'34 (51'26) modify 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:31.630165+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'33 crt=53'33 lcod 53'33 active mbc={} ps=[e~1,13~1,19~1]] add_log_entry 54'35 (53'31) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:31.630165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'33 crt=53'33 lcod 53'33 active mbc={} ps=[e~1,13~1,19~1]] append_log approx pg log length = 35 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'33 crt=53'33 lcod 53'33 active mbc={} ps=[e~1,13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.629+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'34, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57958c00 seq 79 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# 1 keys 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# 1 keys = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# hinfo_key (42 bytes) 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10..' <- SNA_1_000000000000001A_.4_0000000000000001.733E4DBD.1b.smithi13140796-10.. 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000054.00000000000000000034' <- 0000000054.00000000000000000034 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000054.00000000000000000035' <- 0000000054.00000000000000000035 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57958c00 cost 8479 (1 ios * 4000 + 4479 bytes) 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57958c00 onodes 0x558c575a2780,0x558c575a2a00 shared_blobs 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2888) update 4#1:ecc72bdb:::smithi13140796-10:1b# 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:1b# is 57903 (57759 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 1113 (613 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57958c00 allocated 0x[] released 0x[] 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958c00 prepare 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57958c00 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958c00 io_done 2019-12-07T19:34:31.629+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=53'33 crt=53'33 lcod 53'33 active mbc={} ps=[e~1,13~1,19~1]] op_applied version 54'35 2019-12-07T19:34:31.629+0000 7f8f0117b700 10 osd.7 54 dequeue_op 0x558c57334c60 finish 2019-12-07T19:34:31.629+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 92 < min_flush_size 524288 2019-12-07T19:34:31.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 32860 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61930 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2780 had 1 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:31.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 182 ==== MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=69, reqid=osd.2.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=54'24)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57a44800 con 0x558c54c19800 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61953 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f24c58700 15 osd.7 54 enqueue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000019 epoch 54 MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=69, reqid=osd.2.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61953 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=69, reqid=osd.2.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x307fa4~f201 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x307fa4 0x307fa4~f201 to file(ino 16 size 0x310000 mtime 2019-12-07T19:34:31.618802+0000 allocated 310000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2e0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _allocate len 0x71a5 from 0 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _allocate len 0x71a5 from 1 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd440000 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd440000~10000/10000,10000,acd440000 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=69, reqid=osd.2.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x320000 to include allocated 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 64 (was clean) 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=69, reqid=osd.2.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e54) pg 0x558c55141000 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2d7fa4 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfa4 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1a5 and padding block with 0xe5b 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x320000 2019-12-07T19:34:31.630+0000 7f8f06185700 10 osd.7 54 dequeue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=69, reqid=osd.2.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=54'24)) v2 pg pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] 2019-12-07T19:34:31.630+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.ds2 54/41 ECSubWrite(tid=69, reqid=osd.2.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:31.630+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=53'22 active mbc={} ps=[e~1,13~1,19~1]] append_log log((0'0,54'24], crt=53'22) [] 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=54'23 (53'19) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:31.624121+0000 0 snaps [15,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=54'24 (53'20) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:31.624121+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.630+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~1]] append_log approx pg log length = 24 2019-12-07T19:34:31.630+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:31.630+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.630+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.630+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:31.630+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5853a300 seq 57 2019-12-07T19:34:31.630+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:31.630+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:31.630+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.630+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.630+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:31.630+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853a300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:31.630+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853a300 onodes shared_blobs 2019-12-07T19:34:31.630+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853a300 allocated 0x[] released 0x[] 2019-12-07T19:34:31.630+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a300 prepare 2019-12-07T19:34:31.630+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853a300 2019-12-07T19:34:31.630+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a300 io_done 2019-12-07T19:34:31.630+0000 7f8f06185700 10 osd.7 54 dequeue_op 0x558c5740c9a0 finish 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3171a5 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (64) on file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]), flushing log 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 64 len 0x4a crc 0x26d9ed70) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x3f000~1000 to file(ino 1 size 0x3f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x3f000 0x3f000~1000 to file(ino 1 size 0x3f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x40000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3f000 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x40000 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000021 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 183 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c19800 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 64 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f24c58700 15 osd.7 54 enqueue_peering_evt 1.ds2 epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.ds0( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:31.630+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.ds0( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000762880s (0.000000109s flush + 0.000762771s kv commit) 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57958c00> 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958c00 kv_submitted 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57958c00 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958c00 finishing 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57958c00 onodes 0x558c575a2780,0x558c575a2a00 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57958c00 done 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57958c00 [] 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3171a5~471 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3171a5 0x3171a5~471 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2e71a5 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1a5 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x616 and padding block with 0x9ea 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.ds0( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.ds0( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e54) pg 0x558c55141000 2019-12-07T19:34:31.630+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 54/41 ECSubWriteReply(tid=258, last_complete=54'35, committed=1, applied=1)) v2 -- 0x558c54c1f600 con 0x558c54c64000 2019-12-07T19:34:31.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 2(0) info: 1.ds0( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58540600 seq 58 2019-12-07T19:34:31.630+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58540600 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58540600 onodes shared_blobs 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58540600 allocated 0x[] released 0x[] 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540600 prepare 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58540600 2019-12-07T19:34:31.630+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540600 io_done 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x317616 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000163698s (0.000000109s flush + 0.000163589s kv commit) 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853a300> 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a300 kv_submitted 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853a300 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a300 finishing 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853a300 onodes 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853a300 done 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540600 kv_queued 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853a300 [] 2019-12-07T19:34:31.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:31.630+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 54/41 ECSubWriteReply(tid=69, last_complete=54'24, committed=1, applied=1)) v2 -- 0x558c580b9c00 con 0x558c54c19800 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x317616~47d to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x317616 0x317616~47d to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2e7616 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x616 2019-12-07T19:34:31.630+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa93 and padding block with 0x56d 2019-12-07T19:34:31.631+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 444 ==== MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=259, reqid=osd.1.0:257, at_version=0'0, trim_to=0'0, roll_forward_to=54'51)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54d87800 con 0x558c54c64000 2019-12-07T19:34:31.631+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c57efe160 prio 127 cost 0 latency 0.000017 epoch 54 MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=259, reqid=osd.1.0:257, at_version=0'0, trim_to=0'0, roll_forward_to=54'51)) v2 2019-12-07T19:34:31.631+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=259, reqid=osd.1.0:257, at_version=0'0, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=259, reqid=osd.1.0:257, at_version=0'0, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=259, reqid=osd.1.0:257, at_version=0'0, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e54) pg 0x558c57380000 2019-12-07T19:34:31.631+0000 7f8f04982700 10 osd.7 54 dequeue_op 0x558c57efe160 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=259, reqid=osd.1.0:257, at_version=0'0, trim_to=0'0, roll_forward_to=54'51)) v2 pg pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] 2019-12-07T19:34:31.631+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.8s5 54/41 ECSubWrite(tid=259, reqid=osd.1.0:257, at_version=0'0, trim_to=0'0, roll_forward_to=54'51)) v2 2019-12-07T19:34:31.631+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=53'49 active mbc={} ps=[e~1,13~1,19~1]] append_log log((0'0,54'51], crt=53'49) [] 2019-12-07T19:34:31.631+0000 7f8f04982700 20 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=54'50 (53'46) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:31.627903+0000 0 snaps [15,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.631+0000 7f8f04982700 20 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=54'51 (53'47) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:31.627903+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.631+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~1]] append_log approx pg log length = 51 2019-12-07T19:34:31.631+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:31.631+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.631+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:31.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58541b00 seq 47 2019-12-07T19:34:31.631+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:31.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:31.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:31.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58541b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:31.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58541b00 onodes shared_blobs 2019-12-07T19:34:31.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58541b00 allocated 0x[] released 0x[] 2019-12-07T19:34:31.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541b00 prepare 2019-12-07T19:34:31.631+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58541b00 2019-12-07T19:34:31.631+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541b00 io_done 2019-12-07T19:34:31.631+0000 7f8f04982700 10 osd.7 54 dequeue_op 0x558c57efe160 finish 2019-12-07T19:34:31.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 445 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:31.632+0000 7f8f24457700 15 osd.7 54 enqueue_peering_evt 1.8s5 epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:31.632+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.632+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 264 ==== MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=147, reqid=client.4540.0:396, at_version=0'0, trim_to=0'0, roll_forward_to=54'14)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c574a5800 con 0x558c54c5bc00 2019-12-07T19:34:31.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 446 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:31.632+0000 7f8f23c56700 15 osd.7 54 enqueue_op 0x558c581cd760 prio 127 cost 0 latency 0.000020 epoch 54 MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=147, reqid=client.4540.0:396, at_version=0'0, trim_to=0'0, roll_forward_to=54'14)) v2 2019-12-07T19:34:31.632+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=147, reqid=client.4540.0:396, at_version=0'0, trim_to=0'0, roll_forward_to=54'14)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.632+0000 7f8f24457700 15 osd.7 54 enqueue_peering_evt 1.es4 epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:31.632+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=147, reqid=client.4540.0:396, at_version=0'0, trim_to=0'0, roll_forward_to=54'14)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=147, reqid=client.4540.0:396, at_version=0'0, trim_to=0'0, roll_forward_to=54'14)) v2) prio 127 cost 0 e54) pg 0x558c57383000 2019-12-07T19:34:31.632+0000 7f8f05183700 10 osd.7 54 dequeue_op 0x558c581cd760 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=147, reqid=client.4540.0:396, at_version=0'0, trim_to=0'0, roll_forward_to=54'14)) v2 pg pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.632+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.fs5 54/41 ECSubWrite(tid=147, reqid=client.4540.0:396, at_version=0'0, trim_to=0'0, roll_forward_to=54'14)) v2 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e54) pg 0x558c54d98000 2019-12-07T19:34:31.632+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=29'12 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,54'14], crt=29'12) [] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=54'13 (21'6) clone 1:fc3a86a2:::smithi13140796-33:1b by unknown.0.0:0 2019-12-07T19:33:58.707684+0000 0 snaps [1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=54'14 (21'6) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:396 2019-12-07T19:34:31.626553+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 0, new_object: 0 2019-12-07T19:34:31.632+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 14 2019-12-07T19:34:31.632+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.632+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.632+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5853e300 seq 20 2019-12-07T19:34:31.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c582a9800 seq 26 2019-12-07T19:34:31.632+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:31.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head#e onode 0x558c57f35180 txc 0x558c582a9800 2019-12-07T19:34:31.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head#e 0x0 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c57c42c40) put_ref 0x0~2000 Blob(0x558c57c42c40 blob([0x859000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c57c42b60) put_ref 0x0~2000 Blob(0x558c57c42b60 blob([0x85b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c57897f10) put_ref 0x0~2000 Blob(0x558c57897f10 blob([0x85d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853e300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c57b97730) put_ref 0x0~2000 Blob(0x558c57b97730 blob([0x85f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853e300 onodes shared_blobs 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c57b97880) put_ref 0x0~2000 Blob(0x558c57b97880 blob([0x861000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853e300 allocated 0x[] released 0x[] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c57b97960) put_ref 0x0~2000 Blob(0x558c57b97960 blob([0x863000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750ce00) put_ref 0x0~2000 Blob(0x558c5750ce00 blob([0x865000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750cee0) put_ref 0x0~2000 Blob(0x558c5750cee0 blob([0x867000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750cfc0) put_ref 0x0~2000 Blob(0x558c5750cfc0 blob([0x869000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e300 prepare 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d0a0) put_ref 0x0~2000 Blob(0x558c5750d0a0 blob([0x86b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853e300 2019-12-07T19:34:31.632+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e300 io_done 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d180) put_ref 0x0~2000 Blob(0x558c5750d180 blob([0x86d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d260) put_ref 0x0~2000 Blob(0x558c5750d260 blob([0x86f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d340) put_ref 0x0~2000 Blob(0x558c5750d340 blob([0x871000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d420) put_ref 0x0~2000 Blob(0x558c5750d420 blob([0x873000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d500) put_ref 0x0~2000 Blob(0x558c5750d500 blob([0x875000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d5e0) put_ref 0x0~2000 Blob(0x558c5750d5e0 blob([0x877000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.blob(0x558c5750d6c0) put_ref 0x0~1000 Blob(0x558c5750d6c0 blob([0x879000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore.extentmap(0x558c57f35288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57c42c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x859000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57c42b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x85b000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57897f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x85d000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57b97730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x85f000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57b97880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x861000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57b97960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x863000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c5750ce00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x865000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c5750cee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x867000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5750cfc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x869000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c5750d0a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x86b000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c5750d180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x86d000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c5750d260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x86f000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c5750d340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x871000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c5750d420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x873000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c5750d500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x875000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c5750d5e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x877000~2000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~1000: 0x0~1000 Blob(0x558c5750d6c0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=2)))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x879000~1000] 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c5750ce70,0x558c5750cf50,0x558c5750d030,0x558c5750d110,0x558c5750d1f0,0x558c5750d2d0,0x558c5750d3b0,0x558c5750d490,0x558c5750d570,0x558c5750d650,0x558c5750d730,0x558c57897f80,0x558c57b977a0,0x558c57b978f0,0x558c57b979d0,0x558c57c429a0,0x558c57c43ea0 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c5795a500 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:31.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head#e = 0 2019-12-07T19:34:31.632+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:31.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582a9800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582a9800 onodes shared_blobs 0x558c5750ce70,0x558c5750cf50,0x558c5750d030,0x558c5750d110,0x558c5750d1f0,0x558c5750d2d0,0x558c5750d3b0,0x558c5750d490,0x558c5750d570,0x558c5750d650,0x558c5750d730,0x558c57897f80,0x558c57b977a0,0x558c57b978f0,0x558c57b979d0,0x558c57c429a0,0x558c57c43ea0 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a6 is 11 SharedBlob(0x558c5750ce70 loaded (sbid 0x29a6 ref_map(0x865000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a7 is 11 SharedBlob(0x558c5750cf50 loaded (sbid 0x29a7 ref_map(0x867000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a8 is 11 SharedBlob(0x558c5750d030 loaded (sbid 0x29a8 ref_map(0x869000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a9 is 11 SharedBlob(0x558c5750d110 loaded (sbid 0x29a9 ref_map(0x86b000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29aa is 11 SharedBlob(0x558c5750d1f0 loaded (sbid 0x29aa ref_map(0x86d000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ab is 11 SharedBlob(0x558c5750d2d0 loaded (sbid 0x29ab ref_map(0x86f000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ac is 11 SharedBlob(0x558c5750d3b0 loaded (sbid 0x29ac ref_map(0x871000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ad is 11 SharedBlob(0x558c5750d490 loaded (sbid 0x29ad ref_map(0x873000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ae is 11 SharedBlob(0x558c5750d570 loaded (sbid 0x29ae ref_map(0x875000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29af is 11 SharedBlob(0x558c5750d650 loaded (sbid 0x29af ref_map(0x877000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b0 is 11 SharedBlob(0x558c5750d730 loaded (sbid 0x29b0 ref_map(0x879000~1000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a2 is 11 SharedBlob(0x558c57897f80 loaded (sbid 0x29a2 ref_map(0x85d000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a3 is 11 SharedBlob(0x558c57b977a0 loaded (sbid 0x29a3 ref_map(0x85f000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a4 is 11 SharedBlob(0x558c57b978f0 loaded (sbid 0x29a4 ref_map(0x861000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a5 is 11 SharedBlob(0x558c57b979d0 loaded (sbid 0x29a5 ref_map(0x863000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a0 is 11 SharedBlob(0x558c57c429a0 loaded (sbid 0x29a0 ref_map(0x859000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29a1 is 11 SharedBlob(0x558c57c43ea0 loaded (sbid 0x29a1 ref_map(0x85b000~2000=1))) 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582a9800 allocated 0x[] released 0x[] 2019-12-07T19:34:31.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9800 prepare 2019-12-07T19:34:31.632+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582a9800 2019-12-07T19:34:31.632+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9800 io_done 2019-12-07T19:34:31.632+0000 7f8f05183700 10 osd.7 54 dequeue_op 0x558c581cd760 finish 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e54) pg 0x558c57380000 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c578a6900 seq 48 2019-12-07T19:34:31.632+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a6900 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a6900 onodes shared_blobs 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a6900 allocated 0x[] released 0x[] 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6900 prepare 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a6900 2019-12-07T19:34:31.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6900 io_done 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x317a93 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001849338s (0.000000097s flush + 0.001849241s kv commit) 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58540600> 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540600 kv_submitted 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58540600 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540600 finishing 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58540600 onodes 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 2 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540600 done 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58540600 [] 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.632+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2275 < min_flush_size 524288 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3904 < min_flush_size 524288 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35180 had 1 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5030 < min_flush_size 524288 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5053 < min_flush_size 524288 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5053 < min_flush_size 524288 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x317a93~13bd to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x317a93 0x317a93~13bd to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2e7a93 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa93 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe50 and padding block with 0x1b0 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x318e50 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000113524s (0.000000068s flush + 0.000113456s kv commit) 2019-12-07T19:34:31.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58541b00,0x558c5853e300,0x558c582a9800,0x558c578a6900> 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541b00 kv_submitted 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58541b00 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541b00 finishing 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58541b00 onodes 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58541b00 done 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a6900 kv_submitted 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58541b00 [] 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.632+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e300 kv_submitted 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853e300 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e300 finishing 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853e300 onodes 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853e300 done 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853e300 [] 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9800 kv_submitted 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582a9800 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582a9800 finishing 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582a9800 onodes 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582a9800 done 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582a9800 [] 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6900 kv_submitted 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a6900 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6900 finishing 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a6900 onodes 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a6900 done 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a6900 [] 2019-12-07T19:34:31.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.632+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 54/41 ECSubWriteReply(tid=147, last_complete=54'14, committed=1, applied=1)) v2 -- 0x558c58513200 con 0x558c54c5bc00 2019-12-07T19:34:31.632+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.632+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 54/41 ECSubWriteReply(tid=259, last_complete=54'51, committed=1, applied=1)) v2 -- 0x558c55250000 con 0x558c54c64000 2019-12-07T19:34:31.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:31.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 447 ==== MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=260, reqid=osd.1.0:258, at_version=0'0, trim_to=0'0, roll_forward_to=54'35)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c574a3000 con 0x558c54c64000 2019-12-07T19:34:31.633+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c57effce0 prio 127 cost 0 latency 0.000014 epoch 54 MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=260, reqid=osd.1.0:258, at_version=0'0, trim_to=0'0, roll_forward_to=54'35)) v2 2019-12-07T19:34:31.633+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=260, reqid=osd.1.0:258, at_version=0'0, trim_to=0'0, roll_forward_to=54'35)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=260, reqid=osd.1.0:258, at_version=0'0, trim_to=0'0, roll_forward_to=54'35)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=260, reqid=osd.1.0:258, at_version=0'0, trim_to=0'0, roll_forward_to=54'35)) v2) prio 127 cost 0 e54) pg 0x558c54d5f000 2019-12-07T19:34:31.633+0000 7f8f0117b700 10 osd.7 54 dequeue_op 0x558c57effce0 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=260, reqid=osd.1.0:258, at_version=0'0, trim_to=0'0, roll_forward_to=54'35)) v2 pg pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] 2019-12-07T19:34:31.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] _handle_message: MOSDECSubOpWrite(1.7s4 54/41 ECSubWrite(tid=260, reqid=osd.1.0:258, at_version=0'0, trim_to=0'0, roll_forward_to=54'35)) v2 2019-12-07T19:34:31.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=53'33 active mbc={} ps=[e~1,13~1,19~1]] append_log log((0'0,54'35], crt=53'33) [] 2019-12-07T19:34:31.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=54'34 (51'26) modify 1:ecc72bdb:::smithi13140796-10:1b by unknown.0.0:0 2019-12-07T19:34:31.630165+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~1]] rollforward: entry=54'35 (53'31) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:31.630165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~1]] append_log approx pg log length = 35 2019-12-07T19:34:31.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~1]] append_log transaction_applied = 1 2019-12-07T19:34:31.633+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.633+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57958f00 seq 80 2019-12-07T19:34:31.634+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57958f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57958f00 onodes shared_blobs 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57958f00 allocated 0x[] released 0x[] 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958f00 prepare 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57958f00 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958f00 io_done 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 osd.7 54 dequeue_op 0x558c57effce0 finish 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x318e50~471 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x318e50 0x318e50~471 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2e8e50 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe50 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2c1 and padding block with 0xd3f 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3192c1 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000099053s (0.000000094s flush + 0.000098959s kv commit) 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57958f00> 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958f00 kv_submitted 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57958f00 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57958f00 finishing 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57958f00 onodes 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57958f00 done 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57958f00 [] 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.634+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 54/41 ECSubWriteReply(tid=260, last_complete=54'35, committed=1, applied=1)) v2 -- 0x558c58512600 con 0x558c54c64000 2019-12-07T19:34:31.634+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 448 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e54/54) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:31.634+0000 7f8f24457700 15 osd.7 54 enqueue_peering_evt 1.7s4 epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.7s0( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:31.634+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.7s0( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e54) 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.7s0( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e54) queued 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.7s0( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e54) pg 0x558c54d5f000 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~1]] do_peering_event: epoch_sent: 54 epoch_requested: 54 MInfoRec from 1(0) info: 1.7s0( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~1]] proc_primary_info updating purged_snaps to [e~1,13~1,19~2] 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57959200 seq 81 2019-12-07T19:34:31.634+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57959200 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57959200 onodes shared_blobs 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57959200 allocated 0x[] released 0x[] 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959200 prepare 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57959200 2019-12-07T19:34:31.634+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959200 io_done 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3192c1~47d to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3192c1 0x3192c1~47d to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2e92c1 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2c1 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x73e and padding block with 0x8c2 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31973e 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112958s (0.000000085s flush + 0.000112873s kv commit) 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57959200> 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959200 kv_submitted 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57959200 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959200 finishing 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57959200 onodes 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57959200 done 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57959200 [] 2019-12-07T19:34:31.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.634+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.634+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:31.634+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1244942 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.668+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 449 ==== MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=261, reqid=client.4540.0:397, at_version=54'4, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 163058+0+0 (crc 0 0 0) 0x558c575be000 con 0x558c54c64000 2019-12-07T19:34:31.668+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c578d2580 prio 127 cost 0 latency 0.000031 epoch 54 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=261, reqid=client.4540.0:397, at_version=54'4, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:31.668+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=261, reqid=client.4540.0:397, at_version=54'4, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=261, reqid=client.4540.0:397, at_version=54'4, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.668+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=261, reqid=client.4540.0:397, at_version=54'4, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) pg 0x558c57e8b000 2019-12-07T19:34:31.668+0000 7f8f07988700 10 osd.7 54 dequeue_op 0x558c578d2580 prio 127 cost 0 latency 0.000071 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=261, reqid=client.4540.0:397, at_version=54'4, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:31.668+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=261, reqid=client.4540.0:397, at_version=54'4, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:31.668+0000 7f8f07988700 20 snap_mapper.add_oid 1:51ccf92f:::smithi13140796-15:1b 15,1b 2019-12-07T19:34:31.668+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:31.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.668+0000 7f8f07988700 20 snap_mapper.get_snaps 1:51ccf92f:::smithi13140796-15:1b got.empty() 2019-12-07T19:34:31.668+0000 7f8f07988700 20 snap_mapper.set_snaps 1:51ccf92f:::smithi13140796-15:1b 15,1b 2019-12-07T19:34:31.668+0000 7f8f07988700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:31.668+0000 7f8f07988700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:31.668+0000 7f8f07988700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:31.668+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 21'2 (0'0,21'2] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,21'2], crt=21'2) [54'3 (21'2) clone 1:51ccf92f:::smithi13140796-15:1b by unknown.0.0:0 2019-12-07T19:33:58.641780+0000 0 snaps [1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,54'4 (21'2) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:397 2019-12-07T19:34:31.666025+0000 0 ObjectCleanRegions clean_offsets: [0~245760,884736~18446744073708666879], clean_omap: 1, new_object: 0] 2019-12-07T19:34:31.668+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'3 (0'0,54'3] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 54'3 (21'2) clone 1:51ccf92f:::smithi13140796-15:1b by unknown.0.0:0 2019-12-07T19:33:58.641780+0000 0 snaps [1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.668+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'4 (0'0,54'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 54'4 (21'2) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:397 2019-12-07T19:34:31.666025+0000 0 ObjectCleanRegions clean_offsets: [0~245760,884736~18446744073708666879], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.668+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'4 (0'0,54'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 4 2019-12-07T19:34:31.668+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'4 (0'0,54'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:31.668+0000 7f8f07988700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.668+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58110c00 seq 20 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) get_onode oid 5#1:51ccf92f:::smithi13140796-15:1b# key 0x85800000000000000151ccf9'/!smithi13140796-15!='0x000000000000001bffffffffffffffff'o' 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) r -2 v.len 0 2019-12-07T19:34:31.668+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# -> 5#1:51ccf92f:::smithi13140796-15:1b# 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1257 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.onode(0x558c54e13180).flush flush done 2019-12-07T19:34:31.668+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# 0x0 2019-12-07T19:34:31.668+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# -> 5#1:51ccf92f:::smithi13140796-15:1b# 0x0~f000 -> 0x0~f000 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0x0~2000: 0x0~2000 Blob(0x558c578816c0 blob([0x731000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c380 sbid 0x0)) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10673 2019-12-07T19:34:31.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c578816c0 blob([0x731000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c380 sbid 0x0)) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c578816c0 blob([0x731000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c380 loaded (sbid 0x29b1 ref_map(0x731000~2000=1)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c57655880 blob([0x731000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c380 loaded (sbid 0x29b1 ref_map(0x731000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.blob(0x558c57655880) get_ref 0x0~2000 Blob(0x558c57655880 blob([0x731000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c380 loaded (sbid 0x29b1 ref_map(0x731000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.blob(0x558c57655880) get_ref init 0x2000, 1000 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57655880 blob([0x731000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c380 loaded (sbid 0x29b1 ref_map(0x731000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0x2000~2000: 0x0~2000 Blob(0x558c578df490 blob([0x733000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c3f0 sbid 0x0)) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10674 2019-12-07T19:34:31.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c578df490 blob([0x733000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c3f0 sbid 0x0)) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c578df490 blob([0x733000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c3f0 loaded (sbid 0x29b2 ref_map(0x733000~2000=1)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c57655960 blob([0x733000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c3f0 loaded (sbid 0x29b2 ref_map(0x733000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.blob(0x558c57655960) get_ref 0x0~2000 Blob(0x558c57655960 blob([0x733000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c3f0 loaded (sbid 0x29b2 ref_map(0x733000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.blob(0x558c57655960) get_ref init 0x2000, 1000 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57655960 blob([0x733000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c3f0 loaded (sbid 0x29b2 ref_map(0x733000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54e1c460 blob([0x735000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d752d0 sbid 0x0)) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10675 2019-12-07T19:34:31.668+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c54e1c460 blob([0x735000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d752d0 sbid 0x0)) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c54e1c460 blob([0x735000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d752d0 loaded (sbid 0x29b3 ref_map(0x735000~2000=1)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c550603f0 blob([0x735000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d752d0 loaded (sbid 0x29b3 ref_map(0x735000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.blob(0x558c550603f0) get_ref 0x0~2000 Blob(0x558c550603f0 blob([0x735000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d752d0 loaded (sbid 0x29b3 ref_map(0x735000~2000=2)))) 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.blob(0x558c550603f0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.668+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c550603f0 blob([0x735000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d752d0 loaded (sbid 0x29b3 ref_map(0x735000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0x6000~2000: 0x0~2000 Blob(0x558c578817a0 blob([0x737000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10676 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c578817a0 blob([0x737000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c578817a0 blob([0x737000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x29b4 ref_map(0x737000~2000=1)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c575f6150 blob([0x737000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 loaded (sbid 0x29b4 ref_map(0x737000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c575f6150) get_ref 0x0~2000 Blob(0x558c575f6150 blob([0x737000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3dc00 loaded (sbid 0x29b4 ref_map(0x737000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c575f6150) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c575f6150 blob([0x737000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3dc00 loaded (sbid 0x29b4 ref_map(0x737000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0x8000~2000: 0x0~2000 Blob(0x558c5800c000 blob([0x739000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c070 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10677 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c5800c000 blob([0x739000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c070 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c5800c000 blob([0x739000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c070 loaded (sbid 0x29b5 ref_map(0x739000~2000=1)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c576558f0 blob([0x739000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c070 loaded (sbid 0x29b5 ref_map(0x739000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c576558f0) get_ref 0x0~2000 Blob(0x558c576558f0 blob([0x739000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c070 loaded (sbid 0x29b5 ref_map(0x739000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c576558f0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c576558f0 blob([0x739000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c070 loaded (sbid 0x29b5 ref_map(0x739000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0xa000~2000: 0x0~2000 Blob(0x558c5800c0e0 blob([0x73b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c150 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10678 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c5800c0e0 blob([0x73b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c150 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c5800c0e0 blob([0x73b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c150 loaded (sbid 0x29b6 ref_map(0x73b000~2000=1)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c57655a40 blob([0x73b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c150 loaded (sbid 0x29b6 ref_map(0x73b000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655a40) get_ref 0x0~2000 Blob(0x558c57655a40 blob([0x73b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c150 loaded (sbid 0x29b6 ref_map(0x73b000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655a40) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57655a40 blob([0x73b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c150 loaded (sbid 0x29b6 ref_map(0x73b000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0xc000~2000: 0x0~2000 Blob(0x558c5800c1c0 blob([0x73d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c230 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10679 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c5800c1c0 blob([0x73d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c230 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c5800c1c0 blob([0x73d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c230 loaded (sbid 0x29b7 ref_map(0x73d000~2000=1)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c57655ab0 blob([0x73d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c230 loaded (sbid 0x29b7 ref_map(0x73d000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655ab0) get_ref 0x0~2000 Blob(0x558c57655ab0 blob([0x73d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c230 loaded (sbid 0x29b7 ref_map(0x73d000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57655ab0 blob([0x73d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800c230 loaded (sbid 0x29b7 ref_map(0x73d000~2000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup src 0xe000~1000: 0x0~1000 Blob(0x558c5800c2a0 blob([0x73f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800c310 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10680 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared Blob(0x558c5800c2a0 blob([0x73f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800c310 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.as5_head 0x558c54d9f2c0) make_blob_shared now Blob(0x558c5800c2a0 blob([0x73f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800c310 loaded (sbid 0x29b8 ref_map(0x73f000~1000=1)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup new Blob(0x558c57655b20 blob([0x73f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c310 loaded (sbid 0x29b8 ref_map(0x73f000~1000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655b20) get_ref 0x0~1000 Blob(0x558c57655b20 blob([0x73f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800c310 loaded (sbid 0x29b8 ref_map(0x73f000~1000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655b20) get_ref init 0x1000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dup dst 0xe000~1000: 0x0~1000 Blob(0x558c57655b20 blob([0x73f000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800c310 loaded (sbid 0x29b8 ref_map(0x73f000~1000=2)))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c575f9288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# -> 5#1:51ccf92f:::smithi13140796-15:1b# = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# snapset 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# snapset = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# 1 keys 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# 1 keys = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# hinfo_key (42 bytes) 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0xf000~27000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:51ccf92f:::smithi13140796-15:head# 0xf000~27000 - have 0xf000 (61440) bytes fadvise_flags 0x20 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0xf000~27000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58110c00 20 blobs 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.669+0000 7f8f07988700 10 fbmap_alloc 0x558c53a72e00 allocate 0x27000/1000,27000,0 2019-12-07T19:34:31.669+0000 7f8f07988700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x9a000~13000/1000,27000,0 2019-12-07T19:34:31.669+0000 7f8f07988700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xe1000~14000/1000,27000,0 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x9a000~13000,0xe1000~14000] 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655b90 blob([0x9a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655c00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655b90) get_ref 0x0~2000 Blob(0x558c57655b90 blob([0x9a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655c00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655b90) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf000~2000: 0x0~2000 Blob(0x558c57655b90 blob([0x9a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57655c00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57655c18 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655c70 blob([0x9c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655ce0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655c70) get_ref 0x0~2000 Blob(0x558c57655c70 blob([0x9c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655ce0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655c70) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x11000~2000: 0x0~2000 Blob(0x558c57655c70 blob([0x9c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57655ce0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57655cf8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655d50 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655dc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655d50) get_ref 0x0~2000 Blob(0x558c57655d50 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655dc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655d50) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x13000~2000: 0x0~2000 Blob(0x558c57655d50 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57655dc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57655dd8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655e30 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655ea0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655e30) get_ref 0x0~2000 Blob(0x558c57655e30 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655ea0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655e30) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x15000~2000: 0x0~2000 Blob(0x558c57655e30 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57655ea0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57655eb8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57655f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57655f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57655f10 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655f80 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655f10) get_ref 0x0~2000 Blob(0x558c57655f10 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57655f80 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57655f10) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x17000~2000: 0x0~2000 Blob(0x558c57655f10 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57655f80 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57655f98 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b37a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b37a0 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d180 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c550b37a0) get_ref 0x0~2000 Blob(0x558c550b37a0 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d180 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c550b37a0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x19000~2000: 0x0~2000 Blob(0x558c550b37a0 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d180 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5750d198 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eb570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750d0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eb570 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d0a0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c583eb570) get_ref 0x0~2000 Blob(0x558c583eb570 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750d0a0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c583eb570) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558c583eb570 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750d0a0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5750d0b8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750cfc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7960 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cfc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580d7960) get_ref 0x0~2000 Blob(0x558c580d7960 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cfc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580d7960) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558c580d7960 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cfc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5750cfd8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e63810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750cee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e63810 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cee0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57e63810) get_ref 0x0~2000 Blob(0x558c57e63810 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750cee0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57e63810) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558c57e63810 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750cee0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5750cef8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3810 blob([0xac000~1000,0xe1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c550b3810) get_ref 0x0~2000 Blob(0x558c550b3810 blob([0xac000~1000,0xe1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5750ce00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c550b3810) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558c550b3810 blob([0xac000~1000,0xe1000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5750ce00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c5750ce18 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db4f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db4f50 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97960 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57db4f50) get_ref 0x0~2000 Blob(0x558c57db4f50 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97960 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57db4f50) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558c57db4f50 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97960 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57b97978 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a4700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a4700 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97880 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580a4700) get_ref 0x0~2000 Blob(0x558c580a4700 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97880 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580a4700) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558c580a4700 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97880 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57b97898 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ed1ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b97730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ed1ea0 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97730 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57ed1ea0) get_ref 0x0~2000 Blob(0x558c57ed1ea0 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97730 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c57ed1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558c57ed1ea0 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97730 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57b97748 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74bd0 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897f10 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c54d74bd0) get_ref 0x0~2000 Blob(0x558c54d74bd0 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897f10 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c54d74bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558c54d74bd0 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897f10 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57897f28 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829dce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829dce0 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42b60 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c5829dce0) get_ref 0x0~2000 Blob(0x558c5829dce0 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42b60 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c5829dce0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558c5829dce0 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42b60 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57c42b78 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d74a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d74a80 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42c40 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c54d74a80) get_ref 0x0~2000 Blob(0x558c54d74a80 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42c40 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c54d74a80) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c54d74a80 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42c40 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c57c42c58 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55106070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bbc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55106070 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbc00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c55106070) get_ref 0x0~2000 Blob(0x558c55106070 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbc00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c55106070) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c55106070 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bbc00 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c580bbc18 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580bbc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bbce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580bbc70 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbce0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580bbc70) get_ref 0x0~2000 Blob(0x558c580bbc70 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbce0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580bbc70) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c580bbc70 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bbce0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c580bbcf8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580bbd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bbdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580bbd50 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbdc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580bbd50) get_ref 0x0~2000 Blob(0x558c580bbd50 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbdc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580bbd50) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c580bbd50 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bbdc0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c580bbdd8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580bbe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bbea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580bbe30 blob([!~1000,0xf4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbea0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580bbe30) get_ref 0x1000~1000 Blob(0x558c580bbe30 blob([!~1000,0xf4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbea0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.blob(0x558c580bbe30) get_ref init 0x2000, 1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~1000: 0x1000~1000 Blob(0x558c580bbe30 blob([!~1000,0xf4000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580bbea0 sbid 0x0)) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.BufferSpace(0x558c580bbeb8 in 0x558c53a59b20) _discard 0x1000~1000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x36000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0xf000~27000 = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.A833F94F.1b.smithi13140796-15..' <- OBJ_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.A833F94F.1b.smithi13140796-15..' <- SNA_1_0000000000000015_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.5_0000000000000001.A833F94F.1b.smithi13140796-15..' <- SNA_1_000000000000001B_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000054.00000000000000000003' <- 0000000054.00000000000000000003 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000054.00000000000000000004' <- 0000000054.00000000000000000004 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58110c00 cost 252247 (22 ios * 4000 + 164247 bytes) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58110c00 onodes 0x558c54e13180,0x558c575f9180 shared_blobs 0x558c54d3dc00,0x558c54d752d0,0x558c54e1c380,0x558c54e1c3f0,0x558c5800c070,0x558c5800c150,0x558c5800c230,0x558c5800c310 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) update 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) update inline shard 615 bytes from 28 extents 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:head# is 1186 (565 bytes onode + 2 bytes spanning blobs + 619 bytes inline extents) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c575f9288) update 5#1:51ccf92f:::smithi13140796-15:1b# 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore.extentmap(0x558c575f9288) update inline shard 216 bytes from 8 extents 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:1b# is 634 (412 bytes onode + 2 bytes spanning blobs + 220 bytes inline extents) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b4 is 11 SharedBlob(0x558c54d3dc00 loaded (sbid 0x29b4 ref_map(0x737000~2000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b3 is 11 SharedBlob(0x558c54d752d0 loaded (sbid 0x29b3 ref_map(0x735000~2000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b1 is 11 SharedBlob(0x558c54e1c380 loaded (sbid 0x29b1 ref_map(0x731000~2000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b2 is 11 SharedBlob(0x558c54e1c3f0 loaded (sbid 0x29b2 ref_map(0x733000~2000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b5 is 11 SharedBlob(0x558c5800c070 loaded (sbid 0x29b5 ref_map(0x739000~2000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b6 is 11 SharedBlob(0x558c5800c150 loaded (sbid 0x29b6 ref_map(0x73b000~2000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b7 is 11 SharedBlob(0x558c5800c230 loaded (sbid 0x29b7 ref_map(0x73d000~2000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b8 is 11 SharedBlob(0x558c5800c310 loaded (sbid 0x29b8 ref_map(0x73f000~1000=2))) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58110c00 allocated 0x[9a000~13000,e1000~14000] released 0x[] 2019-12-07T19:34:31.669+0000 7f8f07988700 10 freelist allocate 0x9a000~13000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:34:31.669+0000 7f8f07988700 10 freelist allocate 0xe1000~14000 2019-12-07T19:34:31.669+0000 7f8f07988700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110c00 prepare 2019-12-07T19:34:31.669+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58110c00 2019-12-07T19:34:31.669+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.669+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'4 (0'0,54'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 54'4 2019-12-07T19:34:31.669+0000 7f8f07988700 10 osd.7 54 dequeue_op 0x558c578d2580 finish 2019-12-07T19:34:31.669+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.669+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110c00 aio_wait 2019-12-07T19:34:31.669+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58110c00 2019-12-07T19:34:31.669+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110c00 io_done 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4788 < min_flush_size 524288 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13180 had 1 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9180 had 1 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4811 < min_flush_size 524288 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4811 < min_flush_size 524288 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31973e~12cb to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31973e 0x31973e~12cb to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2e973e 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x73e 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa09 and padding block with 0x5f7 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31aa09 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000095418s (0.000003333s flush + 0.000092085s kv commit) 2019-12-07T19:34:31.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58110c00> 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110c00 kv_submitted 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58110c00 2019-12-07T19:34:31.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110c00 finishing 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58110c00 onodes 0x558c54e13180,0x558c575f9180 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750ce18 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f46900 space 0x558c5750ce18 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750cef8 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f76bd0 space 0x558c5750cef8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750cfd8 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4fd40 space 0x558c5750cfd8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d0b8 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e480 space 0x558c5750d0b8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5750d198 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e3f0 space 0x558c5750d198 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655c18 in 0x558c53a59b20) _finish_write discard buffer(0x558c576699e0 space 0x558c57655c18 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655cf8 in 0x558c53a59b20) _finish_write discard buffer(0x558c5813b290 space 0x558c57655cf8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655dd8 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e510 space 0x558c57655dd8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655eb8 in 0x558c53a59b20) _finish_write discard buffer(0x558c579fa990 space 0x558c57655eb8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 54/41 ECSubWriteReply(tid=261, last_complete=54'4, committed=1, applied=1)) v2 -- 0x558c580b8800 con 0x558c54c64000 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57655f98 in 0x558c53a59b20) _finish_write discard buffer(0x558c54dbab40 space 0x558c57655f98 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897f28 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16090 space 0x558c57897f28 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97748 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16000 space 0x558c57b97748 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97898 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c8ec60 space 0x558c57b97898 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b97978 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f77950 space 0x558c57b97978 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42b78 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16120 space 0x558c57c42b78 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42c58 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c161b0 space 0x558c57c42c58 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bbc18 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16240 space 0x558c580bbc18 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bbcf8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c162d0 space 0x558c580bbcf8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bbdd8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16360 space 0x558c580bbdd8 0x0~2000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bbeb8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c163f0 space 0x558c580bbeb8 0x1000~1000 writing nocache) 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58110c00 done 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:31.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58110c00 [] 2019-12-07T19:34:31.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.671+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 450 ==== MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=262, reqid=client.4540.0:398, at_version=54'5, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 158136+0+0 (crc 0 0 0) 0x558c57e7d800 con 0x558c54c64000 2019-12-07T19:34:31.671+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c581cc9a0 prio 127 cost 0 latency 0.000027 epoch 54 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=262, reqid=client.4540.0:398, at_version=54'5, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:31.671+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=262, reqid=client.4540.0:398, at_version=54'5, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.671+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=262, reqid=client.4540.0:398, at_version=54'5, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.671+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.671+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=262, reqid=client.4540.0:398, at_version=54'5, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e54) pg 0x558c57e8b000 2019-12-07T19:34:31.671+0000 7f8f03980700 10 osd.7 54 dequeue_op 0x558c581cc9a0 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=262, reqid=client.4540.0:398, at_version=54'5, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.as5( v 54'4 (0'0,54'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:31.671+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'4 (0'0,54'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=262, reqid=client.4540.0:398, at_version=54'5, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:31.671+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'4 (0'0,54'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,54'4], crt=21'2) [54'5 (54'4) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:398 2019-12-07T19:34:31.670188+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1507328~18446744073708044287], clean_omap: 1, new_object: 0] 2019-12-07T19:34:31.671+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'5 (0'0,54'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'4 crt=21'2 lcod 54'4 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 54'5 (54'4) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:398 2019-12-07T19:34:31.670188+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1507328~18446744073708044287], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.671+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'5 (0'0,54'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'4 crt=21'2 lcod 54'4 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 5 2019-12-07T19:34:31.671+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'5 (0'0,54'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'4 crt=21'2 lcod 54'4 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:31.671+0000 7f8f03980700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.671+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.671+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c5829a600 seq 21 2019-12-07T19:34:31.671+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys 2019-12-07T19:34:31.671+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys = 0 2019-12-07T19:34:31.671+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:31.671+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:31.671+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0x36000~26000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:51ccf92f:::smithi13140796-15:head# 0x36000~26000 - have 0x36000 (221184) bytes fadvise_flags 0x20 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:31.671+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x36000~26000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5829a600 19 blobs 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.671+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate 0x26000/1000,26000,0 2019-12-07T19:34:31.671+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xf5000~9000/1000,26000,0 2019-12-07T19:34:31.671+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x181000~1d000/1000,26000,0 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xf5000~9000,0x181000~1d000] 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0850 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a80 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c578f0850) get_ref 0x0~2000 Blob(0x558c578f0850 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a80 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c578f0850) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c578f0850 blob([0xf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0a80 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0a98 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f62a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6460 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f62a0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6460) get_ref 0x0~2000 Blob(0x558c575f6460 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f62a0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6460) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c575f6460 blob([0xf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f62a0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f62b8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f13b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f13b0 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0fc0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c578f13b0) get_ref 0x0~2000 Blob(0x558c578f13b0 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0fc0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c578f13b0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c578f13b0 blob([0xf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0fc0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f0fd8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f61c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f61c0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1420 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f61c0) get_ref 0x0~2000 Blob(0x558c575f61c0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1420 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f61c0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c575f61c0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1420 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578f1438 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f65b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f6620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f65b0 blob([0xfd000~1000,0x181000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6620 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f65b0) get_ref 0x0~2000 Blob(0x558c575f65b0 blob([0xfd000~1000,0x181000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6620 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f65b0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c575f65b0 blob([0xfd000~1000,0x181000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f6620 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f6638 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f6700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6690 blob([0x182000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6700 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6690) get_ref 0x0~2000 Blob(0x558c575f6690 blob([0x182000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6700 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6690) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c575f6690 blob([0x182000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f6700 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f6718 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f67e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6770 blob([0x184000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f67e0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6770) get_ref 0x0~2000 Blob(0x558c575f6770 blob([0x184000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f67e0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6770) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c575f6770 blob([0x184000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f67e0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f67f8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f68c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6850 blob([0x186000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f68c0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6850) get_ref 0x0~2000 Blob(0x558c575f6850 blob([0x186000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f68c0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6850) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c575f6850 blob([0x186000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f68c0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f68d8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f69a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6930 blob([0x188000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f69a0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6930) get_ref 0x0~2000 Blob(0x558c575f6930 blob([0x188000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f69a0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.blob(0x558c575f6930) get_ref init 0x2000, 1000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c575f6930 blob([0x188000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f69a0 sbid 0x0)) 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f69b8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.671+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f6a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6a10 blob([0x18a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6a80 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6a10) get_ref 0x0~2000 Blob(0x558c575f6a10 blob([0x18a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6a80 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6a10) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c575f6a10 blob([0x18a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f6a80 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f6a98 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f6b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6af0 blob([0x18c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6b60 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6af0) get_ref 0x0~2000 Blob(0x558c575f6af0 blob([0x18c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6b60 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6af0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c575f6af0 blob([0x18c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f6b60 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f6b78 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f6c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6bd0 blob([0x18e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6c40 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6bd0) get_ref 0x0~2000 Blob(0x558c575f6bd0 blob([0x18e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6c40 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c575f6bd0 blob([0x18e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f6c40 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f6c58 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f6d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6cb0 blob([0x190000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6d20 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6cb0) get_ref 0x0~2000 Blob(0x558c575f6cb0 blob([0x190000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f6d20 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c575f6cb0 blob([0x190000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f6d20 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f6d38 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f6d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f1f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f6d90 blob([0x192000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1f10 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6d90) get_ref 0x0~2000 Blob(0x558c575f6d90 blob([0x192000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f1f10 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c575f6d90) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c575f6d90 blob([0x192000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f1f10 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574f1f28 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f00e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f00e0 blob([0x194000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0150 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f00e0) get_ref 0x0~2000 Blob(0x558c574f00e0 blob([0x194000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0150 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f00e0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c574f00e0 blob([0x194000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0150 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574f0168 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f01c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f01c0 blob([0x196000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0230 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f01c0) get_ref 0x0~2000 Blob(0x558c574f01c0 blob([0x196000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0230 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f01c0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c574f01c0 blob([0x196000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0230 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574f0248 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f02a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f0310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f02a0 blob([0x198000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0310 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f02a0) get_ref 0x0~2000 Blob(0x558c574f02a0 blob([0x198000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f0310 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f02a0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c574f02a0 blob([0x198000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0310 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574f0328 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f0380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574f03f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f0380 blob([0x19a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f03f0 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f0380) get_ref 0x0~2000 Blob(0x558c574f0380 blob([0x19a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574f03f0 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c574f0380) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c574f0380 blob([0x19a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f03f0 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574f0408 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7b90 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c580d7b90) get_ref 0x0~2000 Blob(0x558c580d7b90 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.blob(0x558c580d7b90) get_ref init 0x2000, 1000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c580d7b90 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2850 sbid 0x0)) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c574d2868 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5c000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0x36000~26000 = 0 2019-12-07T19:34:31.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.672+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000054.00000000000000000005' <- 0000000054.00000000000000000005 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._fastinfo' <- _fastinfo 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:31.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5829a600 cost 241786 (21 ios * 4000 + 157786 bytes) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5829a600 onodes 0x558c54e13180 shared_blobs 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) update 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) update inline shard 982 bytes from 47 extents 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:head# is 1553 (565 bytes onode + 2 bytes spanning blobs + 986 bytes inline extents) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5829a600 allocated 0x[f5000~9000,181000~1d000] released 0x[] 2019-12-07T19:34:31.672+0000 7f8f03980700 10 freelist allocate 0xf5000~9000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:34:31.672+0000 7f8f03980700 10 freelist allocate 0x181000~1d000 2019-12-07T19:34:31.672+0000 7f8f03980700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:34:31.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a600 prepare 2019-12-07T19:34:31.672+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5829a600 2019-12-07T19:34:31.672+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'5 (0'0,54'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'4 crt=21'2 lcod 54'4 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 54'5 2019-12-07T19:34:31.672+0000 7f8f03980700 10 osd.7 54 dequeue_op 0x558c581cc9a0 finish 2019-12-07T19:34:31.672+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.672+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a600 aio_wait 2019-12-07T19:34:31.672+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5829a600 2019-12-07T19:34:31.672+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a600 io_done 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2802 < min_flush_size 524288 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13180 had 1 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2825 < min_flush_size 524288 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2825 < min_flush_size 524288 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31aa09~b09 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31aa09 0x31aa09~b09 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2eaa09 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa09 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x512 and padding block with 0xaee 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31b512 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.672+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000085266s (0.000003301s flush + 0.000081965s kv commit) 2019-12-07T19:34:31.672+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5829a600> 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.672+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a600 kv_submitted 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5829a600 2019-12-07T19:34:31.672+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5829a600 finishing 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5829a600 onodes 0x558c54e13180 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d2868 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c170e0 space 0x558c574d2868 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0168 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16ea0 space 0x558c574f0168 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0248 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16f30 space 0x558c574f0248 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0328 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16fc0 space 0x558c574f0328 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f0408 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c17050 space 0x558c574f0408 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574f1f28 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16e10 space 0x558c574f1f28 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 54/41 ECSubWriteReply(tid=262, last_complete=54'5, committed=1, applied=1)) v2 -- 0x558c580b8a00 con 0x558c54c64000 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f62b8 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e750 space 0x558c575f62b8 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f6638 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e900 space 0x558c575f6638 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f6718 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16990 space 0x558c575f6718 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f67f8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16a20 space 0x558c575f67f8 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f68d8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16ab0 space 0x558c575f68d8 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f69b8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16b40 space 0x558c575f69b8 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f6a98 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16bd0 space 0x558c575f6a98 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f6b78 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16c60 space 0x558c575f6b78 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f6c58 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16cf0 space 0x558c575f6c58 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f6d38 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16d80 space 0x558c575f6d38 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0a98 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e6c0 space 0x558c578f0a98 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0fd8 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e7e0 space 0x558c578f0fd8 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1438 in 0x558c53a59b20) _finish_write discard buffer(0x558c57f4e870 space 0x558c578f1438 0x0~2000 writing nocache) 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5829a600 done 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:31.672+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5829a600 [] 2019-12-07T19:34:31.672+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.672+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.673+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 451 ==== MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=263, reqid=client.4540.0:399, at_version=54'6, trim_to=0'0, roll_forward_to=54'4)) v2 ==== 39352+0+0 (crc 0 0 0) 0x558c57e7c800 con 0x558c54c64000 2019-12-07T19:34:31.673+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c581ccc60 prio 127 cost 0 latency 0.000022 epoch 54 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=263, reqid=client.4540.0:399, at_version=54'6, trim_to=0'0, roll_forward_to=54'4)) v2 2019-12-07T19:34:31.673+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=263, reqid=client.4540.0:399, at_version=54'6, trim_to=0'0, roll_forward_to=54'4)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=263, reqid=client.4540.0:399, at_version=54'6, trim_to=0'0, roll_forward_to=54'4)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=263, reqid=client.4540.0:399, at_version=54'6, trim_to=0'0, roll_forward_to=54'4)) v2) prio 127 cost 0 e54) pg 0x558c57e8b000 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 54 dequeue_op 0x558c581ccc60 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=263, reqid=client.4540.0:399, at_version=54'6, trim_to=0'0, roll_forward_to=54'4)) v2 pg pg[1.as5( v 54'5 (0'0,54'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'5 (0'0,54'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=263, reqid=client.4540.0:399, at_version=54'6, trim_to=0'0, roll_forward_to=54'4)) v2 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'5 (0'0,54'5] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=21'2 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,54'5], crt=21'2) [54'6 (54'5) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:399 2019-12-07T19:34:31.671156+0000 0 ObjectCleanRegions clean_offsets: [0~1507328,1654784~18446744073707896831], clean_omap: 1, new_object: 0] 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'5 crt=21'2 lcod 54'5 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 54'6 (54'5) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:399 2019-12-07T19:34:31.671156+0000 0 ObjectCleanRegions clean_offsets: [0~1507328,1654784~18446744073707896831], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'5 crt=54'4 lcod 54'5 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=54'3 (21'2) clone 1:51ccf92f:::smithi13140796-15:1b by unknown.0.0:0 2019-12-07T19:33:58.641780+0000 0 snaps [1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'5 crt=54'4 lcod 54'5 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=54'4 (21'2) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:397 2019-12-07T19:34:31.666025+0000 0 ObjectCleanRegions clean_offsets: [0~245760,884736~18446744073708666879], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'5 crt=54'4 lcod 54'5 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 6 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'5 crt=54'4 lcod 54'5 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:31.673+0000 7f8f03980700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.673+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c580d2000 seq 22 2019-12-07T19:34:31.673+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys = 0 2019-12-07T19:34:31.673+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:31.673+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0x5c000~9000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:51ccf92f:::smithi13140796-15:head# 0x5c000~9000 - have 0x5c000 (376832) bytes fadvise_flags 0x20 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x5c000~9000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d2000 5 blobs 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:31.673+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate 0x9000/1000,9000,0 2019-12-07T19:34:31.673+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x19e000~8000/1000,9000,0 2019-12-07T19:34:31.673+0000 7f8f03980700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x339000~1000/1000,9000,0 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x19e000~8000,0x339000~1000] 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55060380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f64d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55060380 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f64d0 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c55060380) get_ref 0x0~2000 Blob(0x558c55060380 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f64d0 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c55060380) get_ref init 0x2000, 1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~2000: 0x0~2000 Blob(0x558c55060380 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f64d0 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c575f64e8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792c230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792c230 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c5792c230) get_ref 0x0~2000 Blob(0x558c5792c230 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c5792c230) get_ref init 0x2000, 1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5e000~2000: 0x0~2000 Blob(0x558c5792c230 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c310 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5792c328 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e563f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e563f0 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c54e563f0) get_ref 0x0~2000 Blob(0x558c54e563f0 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c54e563f0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x60000~2000: 0x0~2000 Blob(0x558c54e563f0 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dfea0 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578dfeb8 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578dfce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578dfb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578dfce0 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfb20 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c578dfce0) get_ref 0x0~2000 Blob(0x558c578dfce0 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dfb20 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c578dfce0) get_ref init 0x2000, 1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x62000~2000: 0x0~2000 Blob(0x558c578dfce0 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dfb20 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c578dfb38 in 0x558c53a59b20) _discard 0x0~2000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794dc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794dc00 blob([0x339000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c5794dc00) get_ref 0x0~1000 Blob(0x558c5794dc00 blob([0x339000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.blob(0x558c5794dc00) get_ref init 0x1000, 1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x64000~1000: 0x0~1000 Blob(0x558c5794dc00 blob([0x339000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5794db20 sbid 0x0)) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.BufferSpace(0x558c5794db38 in 0x558c53a59b20) _discard 0x0~1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x65000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) dirty_range mark inline shard dirty 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 0x5c000~9000 = 0 2019-12-07T19:34:31.673+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.673+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000054.00000000000000000006' <- 0000000054.00000000000000000006 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._fastinfo' <- _fastinfo 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d2000 cost 63002 (6 ios * 4000 + 39002 bytes) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d2000 onodes 0x558c54e13180 shared_blobs 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) update 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) update inline shard 1074 bytes from 52 extents 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:head# is 1645 (565 bytes onode + 2 bytes spanning blobs + 1078 bytes inline extents) 2019-12-07T19:34:31.673+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d2000 allocated 0x[19e000~8000,339000~1000] released 0x[] 2019-12-07T19:34:31.673+0000 7f8f03980700 10 freelist allocate 0x19e000~8000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:34:31.673+0000 7f8f03980700 10 freelist allocate 0x339000~1000 2019-12-07T19:34:31.673+0000 7f8f03980700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2000 prepare 2019-12-07T19:34:31.673+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c580d2000 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'5 crt=54'4 lcod 54'5 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 54'6 2019-12-07T19:34:31.673+0000 7f8f03980700 10 osd.7 54 dequeue_op 0x558c581ccc60 finish 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.673+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2000 aio_wait 2019-12-07T19:34:31.673+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d2000 2019-12-07T19:34:31.673+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2000 io_done 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2894 < min_flush_size 524288 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13180 had 1 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2917 < min_flush_size 524288 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2917 < min_flush_size 524288 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31b512~b65 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31b512 0x31b512~b65 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2eb512 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x512 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x77 and padding block with 0xf89 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31c077 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.673+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000114502s (0.000005183s flush + 0.000109319s kv commit) 2019-12-07T19:34:31.673+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d2000> 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.673+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2000 kv_submitted 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d2000 2019-12-07T19:34:31.673+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2000 finishing 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d2000 onodes 0x558c54e13180 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f64e8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c16480 space 0x558c575f64e8 0x0~2000 writing nocache) 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dfb38 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c17320 space 0x558c578dfb38 0x0~2000 writing nocache) 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578dfeb8 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c17290 space 0x558c578dfeb8 0x0~2000 writing nocache) 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792c328 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c17200 space 0x558c5792c328 0x0~2000 writing nocache) 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794db38 in 0x558c53a59b20) _finish_write discard buffer(0x558c58c173b0 space 0x558c5794db38 0x0~1000 writing nocache) 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2000 done 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:31.673+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d2000 [] 2019-12-07T19:34:31.673+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.673+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.673+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 54/41 ECSubWriteReply(tid=263, last_complete=54'6, committed=1, applied=1)) v2 -- 0x558c580b8800 con 0x558c54c64000 2019-12-07T19:34:31.673+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.673+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.673+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 452 ==== MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=264, reqid=client.4540.0:400, at_version=54'7, trim_to=0'0, roll_forward_to=54'4)) v2 ==== 2455+0+0 (crc 0 0 0) 0x558c57e7d000 con 0x558c54c64000 2019-12-07T19:34:31.673+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000020 epoch 54 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=264, reqid=client.4540.0:400, at_version=54'7, trim_to=0'0, roll_forward_to=54'4)) v2 2019-12-07T19:34:31.673+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=264, reqid=client.4540.0:400, at_version=54'7, trim_to=0'0, roll_forward_to=54'4)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.674+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.674+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=264, reqid=client.4540.0:400, at_version=54'7, trim_to=0'0, roll_forward_to=54'4)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.674+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.674+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=264, reqid=client.4540.0:400, at_version=54'7, trim_to=0'0, roll_forward_to=54'4)) v2) prio 127 cost 0 e54) pg 0x558c57e8b000 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 54 dequeue_op 0x558c581cdb80 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=264, reqid=client.4540.0:400, at_version=54'7, trim_to=0'0, roll_forward_to=54'4)) v2 pg pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'4 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'4 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=264, reqid=client.4540.0:400, at_version=54'7, trim_to=0'0, roll_forward_to=54'4)) v2 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'6 (0'0,54'6] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'4 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,54'6], crt=54'4) [54'7 (54'6) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:400 2019-12-07T19:34:31.671176+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'6 crt=54'4 lcod 54'6 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 54'7 (54'6) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:400 2019-12-07T19:34:31.671176+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'6 crt=54'4 lcod 54'6 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 7 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'6 crt=54'4 lcod 54'6 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:31.674+0000 7f8f07988700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.674+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 54'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.674+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58110f00 seq 23 2019-12-07T19:34:31.674+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 3 keys 2019-12-07T19:34:31.674+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 3 keys = 0 2019-12-07T19:34:31.674+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:34:31.674+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:31.674+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000054.00000000000000000007' <- 0000000054.00000000000000000007 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._fastinfo' <- _fastinfo 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.674+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:31.674+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58110f00 cost 6105 (1 ios * 4000 + 2105 bytes) 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58110f00 onodes 0x558c54e13180 shared_blobs 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore.extentmap(0x558c54e13288) update 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:head# is 1646 (566 bytes onode + 2 bytes spanning blobs + 1078 bytes inline extents) 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58110f00 allocated 0x[] released 0x[] 2019-12-07T19:34:31.674+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110f00 prepare 2019-12-07T19:34:31.674+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58110f00 2019-12-07T19:34:31.674+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110f00 io_done 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'6 crt=54'4 lcod 54'6 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 54'7 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.674+0000 7f8f07988700 10 osd.7 54 dequeue_op 0x558c581cdb80 finish 2019-12-07T19:34:31.674+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2826 < min_flush_size 524288 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13180 had 1 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2849 < min_flush_size 524288 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2849 < min_flush_size 524288 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31c077~b21 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31c077 0x31c077~b21 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2ec077 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x77 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb98 and padding block with 0x468 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31cb98 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.674+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000099940s (0.000000111s flush + 0.000099829s kv commit) 2019-12-07T19:34:31.674+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58110f00> 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110f00 kv_submitted 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58110f00 2019-12-07T19:34:31.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58110f00 finishing 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58110f00 onodes 0x558c54e13180 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58110f00 done 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:31.674+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58110f00 [] 2019-12-07T19:34:31.674+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.674+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.674+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.674+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 54/41 ECSubWriteReply(tid=264, last_complete=54'7, committed=1, applied=1)) v2 -- 0x558c580b8a00 con 0x558c54c64000 2019-12-07T19:34:31.674+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.675+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 453 ==== MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=265, reqid=client.4540.0:400, at_version=0'0, trim_to=0'0, roll_forward_to=54'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57e7a800 con 0x558c54c64000 2019-12-07T19:34:31.675+0000 7f8f24457700 15 osd.7 54 enqueue_op 0x558c581cd1e0 prio 127 cost 0 latency 0.000016 epoch 54 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=265, reqid=client.4540.0:400, at_version=0'0, trim_to=0'0, roll_forward_to=54'7)) v2 2019-12-07T19:34:31.675+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=265, reqid=client.4540.0:400, at_version=0'0, trim_to=0'0, roll_forward_to=54'7)) v2) prio 127 cost 0 e54) 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=265, reqid=client.4540.0:400, at_version=0'0, trim_to=0'0, roll_forward_to=54'7)) v2) prio 127 cost 0 e54) queued 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=265, reqid=client.4540.0:400, at_version=0'0, trim_to=0'0, roll_forward_to=54'7)) v2) prio 127 cost 0 e54) pg 0x558c57e8b000 2019-12-07T19:34:31.675+0000 7f8f03980700 10 osd.7 54 dequeue_op 0x558c581cd1e0 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=265, reqid=client.4540.0:400, at_version=0'0, trim_to=0'0, roll_forward_to=54'7)) v2 pg pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'4 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:31.675+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'4 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.as5 54/41 ECSubWrite(tid=265, reqid=client.4540.0:400, at_version=0'0, trim_to=0'0, roll_forward_to=54'7)) v2 2019-12-07T19:34:31.675+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'4 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,54'7], crt=54'4) [] 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=54'5 (54'4) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:398 2019-12-07T19:34:31.670188+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1507328~18446744073708044287], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=54'6 (54'5) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:399 2019-12-07T19:34:31.671156+0000 0 ObjectCleanRegions clean_offsets: [0~1507328,1654784~18446744073707896831], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=54'7 (54'6) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:400 2019-12-07T19:34:31.671176+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:31.675+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 7 2019-12-07T19:34:31.675+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:31.675+0000 7f8f03980700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:31.675+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:31.675+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:31.675+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c580d2600 seq 24 2019-12-07T19:34:31.675+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:31.675+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:31.675+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:31.675+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:31.675+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:31.675+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d2600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:31.675+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d2600 onodes shared_blobs 2019-12-07T19:34:31.675+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d2600 allocated 0x[] released 0x[] 2019-12-07T19:34:31.675+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2600 prepare 2019-12-07T19:34:31.675+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d2600 2019-12-07T19:34:31.675+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2600 io_done 2019-12-07T19:34:31.675+0000 7f8f03980700 10 osd.7 54 dequeue_op 0x558c581cd1e0 finish 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31cb98~471 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31cb98 0x31cb98~471 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2ecb98 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb98 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9 and padding block with 0xff7 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31d009 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:31.675+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000091005s (0.000000118s flush + 0.000090887s kv commit) 2019-12-07T19:34:31.675+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d2600> 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:31.675+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2600 kv_submitted 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d2600 2019-12-07T19:34:31.675+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2600 finishing 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d2600 onodes 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2600 done 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:31.675+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d2600 [] 2019-12-07T19:34:31.675+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:31.675+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:31.675+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.675+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 54/41 ECSubWriteReply(tid=265, last_complete=54'7, committed=1, applied=1)) v2 -- 0x558c54c1f600 con 0x558c54c64000 2019-12-07T19:34:31.675+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:31.685+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0259228 2019-12-07T19:34:31.685+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.735+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 10 osd.7 54 tick_without_osd_lock 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f50f000/0x40000000/0x1680000000, data 0xb2599e/0xaef000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 25 osd.7 54 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875883+0000 last_rx_front 2019-12-07T19:34:27.875916+0000 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 25 osd.7 54 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875931+0000 last_rx_front 2019-12-07T19:34:27.875870+0000 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 25 osd.7 54 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875893+0000 last_rx_front 2019-12-07T19:34:27.875903+0000 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 25 osd.7 54 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875766+0000 last_rx_front 2019-12-07T19:34:27.875813+0000 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 25 osd.7 54 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:27.875562+0000 last_rx_back 2019-12-07T19:34:27.875796+0000 last_rx_front 2019-12-07T19:34:27.875837+0000 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 20 osd.7 54 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 20 osd.7 54 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 20 osd.7 54 scrub_load_below_threshold loadavg per cpu 0.0375 < max 5 = yes 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 20 osd.7 54 sched_scrub load_is_low=1 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 10 osd.7 54 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:08.239446+0000 > 2019-12-07T19:34:31.744409+0000 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 20 osd.7 54 sched_scrub done 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 10 osd.7 54 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 20 osd.7 54 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:31.743+0000 7f8f1fbbe700 10 osd.7 54 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:31.785+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.835+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.854+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 17 ==== osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:31.855358+0000/55.312870539s send_stamp 55.312870539s delta_ub 0.022346444s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c552d1400 2019-12-07T19:34:31.854+0000 7f8f24457700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021930327s,0.022346444s]) 2019-12-07T19:34:31.854+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:31.855358+0000/55.312870539s send_stamp 55.290940212s delta_ub 18446744073.687621289s) v5 -- 0x558c580b8a00 con 0x558c552d1400 2019-12-07T19:34:31.854+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 17 ==== osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:31.855358+0000/55.312870539s send_stamp 55.312870539s delta_ub 0.022346444s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c54c18000 2019-12-07T19:34:31.854+0000 7f8f23c56700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021882366s,0.022346444s]) 2019-12-07T19:34:31.854+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:31.855358+0000/55.312870539s send_stamp 55.290988173s delta_ub 18446744073.687669250s) v5 -- 0x558c58513c00 con 0x558c54c18000 2019-12-07T19:34:31.885+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.936+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:31.985+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.035+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.085+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.135+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.186+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.236+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.251+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:32.286+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.287+0000 7f8f21451700 10 osd.7 54 tick 2019-12-07T19:34:32.287+0000 7f8f21451700 10 osd.7 54 do_waiters -- start 2019-12-07T19:34:32.287+0000 7f8f21451700 10 osd.7 54 do_waiters -- finish 2019-12-07T19:34:32.287+0000 7f8f21451700 20 osd.7 54 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:32.336+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.345+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 13 ==== osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.346242+0000/55.812565798s send_stamp 55.812565798s delta_ub 0.031040502s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c55251e00 con 0x558c54c5ac00 2019-12-07T19:34:32.345+0000 7f8f23c56700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030726852s,0.031040502s]) 2019-12-07T19:34:32.345+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 13 ==== osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.346242+0000/55.812565798s send_stamp 55.812565798s delta_ub 0.031040502s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c54c5a400 2019-12-07T19:34:32.345+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.346242+0000/55.812565798s send_stamp 55.781838946s delta_ub 18446744073.678824764s) v5 -- 0x558c58513c00 con 0x558c54c5ac00 2019-12-07T19:34:32.345+0000 7f8f23c56700 10 osd.7 54 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 53 -> 54 (as per caller) 2019-12-07T19:34:32.345+0000 7f8f24c58700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030702289s,0.031040502s]) 2019-12-07T19:34:32.345+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.346242+0000/55.812565798s send_stamp 55.781863509s delta_ub 18446744073.678849327s) v5 -- 0x558c58512c00 con 0x558c54c5a400 2019-12-07T19:34:32.386+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.436+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.445+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 17 ==== osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.446017+0000/55.917736413s send_stamp 55.917736413s delta_ub 0.036495787s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58512600 con 0x558c552d1c00 2019-12-07T19:34:32.445+0000 7f8f24457700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036031402s,0.036495787s]) 2019-12-07T19:34:32.445+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.446017+0000/55.917736413s send_stamp 55.881705011s delta_ub 18446744073.673520214s) v5 -- 0x558c580b8a00 con 0x558c552d1c00 2019-12-07T19:34:32.445+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 17 ==== osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.446017+0000/55.917736413s send_stamp 55.917736413s delta_ub 0.036495787s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c552d1800 2019-12-07T19:34:32.445+0000 7f8f24c58700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036005845s,0.036495787s]) 2019-12-07T19:34:32.445+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.446017+0000/55.917736413s send_stamp 55.881730568s delta_ub 18446744073.673545771s) v5 -- 0x558c58512c00 con 0x558c552d1800 2019-12-07T19:34:32.486+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.536+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.575+0000 7f8f0097a700 5 osd.7 54 heartbeat osd_stat(store_statfs(0x163f50f000/0x40000000/0x1680000000, data 0xb2599e/0xaef000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5] op hist []) 2019-12-07T19:34:32.575+0000 7f8f0097a700 20 osd.7 54 check_full_status cur ratio 0.000118722, physical ratio 0.000118722, new state none 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.688345878s) v5 -- 0x558c57519400 con 0x558c54c9e000 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.688345878s) v5 -- 0x558c57519000 con 0x558c54c9e400 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.673545771s) v5 -- 0x558c57519600 con 0x558c551b7800 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.673545771s) v5 -- 0x558c548cc200 con 0x558c551b7c00 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.687669250s) v5 -- 0x558c55271c00 con 0x558c552c2000 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.687669250s) v5 -- 0x558c551afc00 con 0x558c552c2400 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.696048255s) v5 -- 0x558c58332400 con 0x558c552c2800 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.696048255s) v5 -- 0x558c58332a00 con 0x558c552c2c00 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.678849327s) v5 -- 0x558c54c1e000 con 0x558c54c65800 2019-12-07T19:34:32.575+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.011278219s delta_ub 18446744073.678849327s) v5 -- 0x558c54c1f200 con 0x558c54c65c00 2019-12-07T19:34:32.575+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 12 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.042317327s delta_ub 0.031039108s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c54c65800 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:27.875796+0000 -> 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:27.875837+0000 2019-12-07T19:34:32.575+0000 7f8f23c56700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030818956s,0.031039108s]) 2019-12-07T19:34:32.575+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 12 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.042335905s delta_ub 0.031057686s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c54c65c00 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:27.875837+0000 -> 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:32.575839+0000) and older pending ping(s) 2019-12-07T19:34:32.575+0000 7f8f23c56700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030811872s,0.031057686s]) 2019-12-07T19:34:32.575+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 16 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.025227327s delta_ub 0.013949108s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c552c2c00 2019-12-07T19:34:32.575+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 12 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.033064745s delta_ub 0.021786526s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c54c9e000 2019-12-07T19:34:32.575+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 16 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.025281054s delta_ub 0.014002835s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c552c2800 2019-12-07T19:34:32.575+0000 7f8f24457700 25 osd.7 54 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:27.875766+0000 last_rx_front 2019-12-07T19:34:27.875813+0000 -> 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:32.575+0000 7f8f24457700 10 osd.7 54 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 53 -> 54 (as per caller) 2019-12-07T19:34:32.575+0000 7f8f24457700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013679614s,0.013949108s]) 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:27.875883+0000 -> 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:27.875916+0000 2019-12-07T19:34:32.575+0000 7f8f23c56700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021485653s,0.021786526s]) 2019-12-07T19:34:32.575+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 16 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.033687786s delta_ub 0.022409567s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c1f600 con 0x558c552c2000 2019-12-07T19:34:32.575+0000 7f8f24c58700 25 osd.7 54 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:27.875766+0000 -> 2019-12-07T19:34:32.576154+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:32.575+0000 7f8f24c58700 25 osd.7 54 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:32.575839+0000) and older pending ping(s) 2019-12-07T19:34:32.575+0000 7f8f24c58700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013686897s,0.014002835s]) 2019-12-07T19:34:32.575+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 16 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.033662320s delta_ub 0.022384101s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c552c2400 2019-12-07T19:34:32.575+0000 7f8f24457700 25 osd.7 54 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:27.875893+0000 -> 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:27.875903+0000 2019-12-07T19:34:32.575+0000 7f8f24457700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022074759s,0.022409567s]) 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:27.875903+0000 -> 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:32.575+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 16 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.047896487s delta_ub 0.036618268s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c551b7c00 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:32.575839+0000) and older pending ping(s) 2019-12-07T19:34:32.575+0000 7f8f23c56700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022038874s,0.022384101s]) 2019-12-07T19:34:32.575+0000 7f8f24c58700 25 osd.7 54 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:27.875931+0000 last_rx_front 2019-12-07T19:34:27.875870+0000 -> 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:32.575+0000 7f8f24c58700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036249596s,0.036618268s]) 2019-12-07T19:34:32.575+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 16 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.047870332s delta_ub 0.036592113s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c551b7800 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:27.875931+0000 -> 2019-12-07T19:34:32.576218+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:32.575+0000 7f8f23c56700 25 osd.7 54 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:32.575839+0000) and older pending ping(s) 2019-12-07T19:34:32.575+0000 7f8f23c56700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036212722s,0.036592113s]) 2019-12-07T19:34:32.575+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 12 ==== osd_ping(ping_reply e54 up_from 13 ping_stamp 2019-12-07T19:34:32.575839+0000/56.011278219s send_stamp 56.033120607s delta_ub 0.021842388s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c54c9e400 2019-12-07T19:34:32.575+0000 7f8f24c58700 25 osd.7 54 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:27.875916+0000 -> 2019-12-07T19:34:32.576231+0000 2019-12-07T19:34:32.575+0000 7f8f24c58700 25 osd.7 54 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:32.575839+0000) and older pending ping(s) 2019-12-07T19:34:32.575+0000 7f8f24c58700 20 osd.7 54 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021449957s,0.021842388s]) 2019-12-07T19:34:32.587+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1265507 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 454 ==== osd_map(55..55 src has 1..55) v4 ==== 633+0+0 (crc 0 0 0) 0x558c5853d8c0 con 0x558c54c64000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 osd.7 54 OSD::ms_dispatch: osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 osd.7 54 do_waiters -- start 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 osd.7 54 do_waiters -- finish 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 osd.7 54 _dispatch 0x558c5853d8c0 osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.633+0000 7f8f1199c700 3 osd.7 54 handle_osd_map epochs [55,55], i have 54, src has [1,55] 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 osd.7 54 handle_osd_map got inc map for epoch 55 2019-12-07T19:34:32.633+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3874e826:::osdmap.54:0# 0x0~0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5809aaf0 blob([0x98000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580a2770 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d7880 blob([!~1000,0x99000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c580ec230 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3874e826:::osdmap.54:0# 0x0~1464 = 5220 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 osd.7 54 add_map_bl 54 5220 bytes 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 osd.7 54 got_full_map 55, nothing requested 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 osd.7 54 handle_osd_map pg_num_history pg_num_history(e55 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {55={}} 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 osd.7 54 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e55 [1,55] lci=[13,55]) 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c580d3200 seq 46 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cb67972f:::inc_osdmap.55:0# key 0x7f7fffffffffffffffcb6797'/!inc_osdmap.55!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cb67972f:::inc_osdmap.55:0# 0x0~24d 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1258 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cb67972f:::inc_osdmap.55:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d3200 1 blobs 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x33a000~1000/1000,1000,0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x33a000~1000] 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e63f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809b730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e63f80 blob([0x33a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b730 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c57e63f80) get_ref 0x0~24d Blob(0x558c57e63f80 blob([0x33a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b730 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c57e63f80) get_ref init 0x1000, 1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c57e63f80 blob([0x33a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5809b730 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5809b748 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e888) dirty_range mark inline shard dirty 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cb67972f:::inc_osdmap.55:0# 0x0~24d = 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:35f4e826:::osdmap.55:0# key 0x7f7fffffffffffffff35f4e8'&!osdmap.55!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:35f4e826:::osdmap.55:0# 0x0~1448 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1259 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:35f4e826:::osdmap.55:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d3200 2 blobs 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x33b000~2000/1000,2000,0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x33b000~2000] 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578805b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794c1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578805b0 blob([0x33b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c1c0 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c578805b0) get_ref 0x0~1000 Blob(0x558c578805b0 blob([0x33b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794c1c0 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c578805b0) get_ref init 0x1000, 1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c578805b0 blob([0x33b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5794c1c0 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5794c1d8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809ac40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396460 blob([!~1000,0x33c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809ac40 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c57396460) get_ref 0x1000~448 Blob(0x558c57396460 blob([!~1000,0x33c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c5809ac40 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c57396460) get_ref init 0x2000, 1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c57396460 blob([!~1000,0x33c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c5809ac40 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5809ac58 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851eb08) dirty_range mark inline shard dirty 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:35f4e826:::osdmap.55:0# 0x0~1448 = 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d3200 0 blobs 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c580d3200 0 blobs 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d3200 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d3200 onodes 0x558c53a91900,0x558c55276c80,0x558c5851e780,0x558c5851ea00 shared_blobs 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e888) update #-1:cb67972f:::inc_osdmap.55:0# 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851e888) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cb67972f:::inc_osdmap.55:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851eb08) update #-1:35f4e826:::osdmap.55:0# 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851eb08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:35f4e826:::osdmap.55:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d3200 allocated 0x[33a000~3000] released 0x[] 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 freelist allocate 0x33a000~3000 2019-12-07T19:34:32.633+0000 7f8f1199c700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3200 prepare 2019-12-07T19:34:32.633+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c580d3200 2019-12-07T19:34:32.633+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3200 aio_wait 2019-12-07T19:34:32.634+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d3200 2019-12-07T19:34:32.634+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3200 io_done 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851e780 had 1 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851ea00 had 1 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3857580032 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997949) bluestore 86 GiB free (0.959881), bluefs_ratio 0.0399264 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31d009~226c to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31d009 0x31d009~226c to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~2f0000 x_off 0x2ed009 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x275 and padding block with 0xd8b 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x31f275 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000163389s (0.000004816s flush + 0.000158573s kv commit) 2019-12-07T19:34:32.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c580d3200> 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3200 kv_submitted 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d3200 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c580d3200 osr 0x558c54ea4b40 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 43 0x1b000~1000 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 44 0x1b000~1000 crc 618bf9df 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 43 0x2000~1000 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 44 0x2000~1000 crc e2299622 2019-12-07T19:34:32.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7 54 _committed_osd_maps 55..55 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7 54 advance to epoch 55 (<= last 55 <= newest_map 55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 7 osd.7 55 consume_map version 55 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.8s5 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.0s3 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.1s1 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.9s4 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.as5 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.2s0 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.bs1 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.3s4 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.cs0 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.4s0 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.5s4 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.ds2 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.es4 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.6s2 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.fs5 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 identify_splits_and_merges 1.7s4 e54 to e55 pg_nums {18=16} 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 55 prune_pg_created 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:0.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:1.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:2.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:3.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:4.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:5.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:6.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7:7.consume_map 55 (was 54) 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:32.634+0000 7f8f141a1700 10 osd.7 55 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.8s5 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.0s3 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.1s1 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.9s4 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.as5 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.2s0 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.bs1 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c5736a000 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.3s4 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.cs0 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.4s0 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.5s4 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.ds2 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.es4 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.6s2 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.fs5 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 15 osd.7 55 enqueue_peering_evt 1.7s4 epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) 2019-12-07T19:34:32.634+0000 7f8f141a1700 7 osd.7 55 activate_map version 55 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c57e8b000 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c5500a000 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 54 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c575ef000 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c57e88000 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c54d98000 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 54 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:32.634+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] state: Active advmap 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 54 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] state: Active: handling ActMap 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] Active: kicking snap trim 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f03980700 7 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] state: handle ActMap primary 2019-12-07T19:34:32.634+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~2] 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f03980700 15 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] publish_stats_to_osd 54:158 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c54d5c000 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f0918b700 20 osd.7 55 agent_entry empty queue 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 54 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c5513e000 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 54 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:32.634+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 455 ==== MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=267, reqid=client.4540.0:403, at_version=55'37, trim_to=0'0, roll_forward_to=54'35)) v2 ==== 2960+0+0 (crc 0 0 0) 0x558c57e7c000 con 0x558c54c64000 2019-12-07T19:34:32.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:32.634+0000 7f8f24457700 15 osd.7 55 enqueue_op 0x558c54e79e40 prio 127 cost 0 latency 0.001471 epoch 55 MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=267, reqid=client.4540.0:403, at_version=55'37, trim_to=0'0, roll_forward_to=54'35)) v2 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=267, reqid=client.4540.0:403, at_version=55'37, trim_to=0'0, roll_forward_to=54'35)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c57522000 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c57676000 2019-12-07T19:34:32.634+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] state: Active advmap 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] state: Active: handling ActMap 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] Active: kicking snap trim 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.634+0000 7f8f06986700 7 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] state: handle ActMap primary 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~2] 2019-12-07T19:34:32.634+0000 7f8f06986700 15 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] publish_stats_to_osd 54:182 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c548ab000 2019-12-07T19:34:32.634+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c57380000 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.634+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.634+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.634+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 54 -> 55 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.634+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c55141000 2019-12-07T19:34:32.634+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.634+0000 7f8f06986700 10 osd.7 pg_epoch: 54 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 54 -> 55 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] state: Active advmap 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.635+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 54 -> 55 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 54 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.635+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:32.635+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.635+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.635+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:32.635+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.635+0000 7f8f06986700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.635+0000 7f8f06185700 20 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.635+0000 7f8f06185700 20 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.635+0000 7f8f06185700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.635+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] state: Active: handling ActMap 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.635+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] Active: kicking snap trim 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c5500d000 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.635+0000 7f8f06986700 7 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] state: handle ActMap primary 2019-12-07T19:34:32.635+0000 7f8f06185700 20 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.635+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.635+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] prepare_stats_for_publish reporting purged_snaps [e~1,13~1,19~2] 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 54 -> 55 2019-12-07T19:34:32.635+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.635+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 54 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:32.635+0000 7f8f06986700 15 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] publish_stats_to_osd 54:146 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.635+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.635+0000 7f8f06986700 20 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.635+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:32.635+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.635+0000 7f8f06986700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.635+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.635+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f04982700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.635+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c57383000 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 54 -> 55 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.635+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.635+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.635+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) queued 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 55 epoch_requested: 55 NullEvt) prio 255 cost 10 e55) pg 0x558c54d5f000 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 55 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 54 -> 55 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 54 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:32.635+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 55 advance_pg new pool opts old pool opts 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 55 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 55 2019-12-07T19:34:32.635+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 55 epoch_requested: 55 NullEvt 2019-12-07T19:34:32.635+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=267, reqid=client.4540.0:403, at_version=55'37, trim_to=0'0, roll_forward_to=54'35)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.635+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=267, reqid=client.4540.0:403, at_version=55'37, trim_to=0'0, roll_forward_to=54'35)) v2) prio 127 cost 0 e55) pg 0x558c54d5f000 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 55 dequeue_op 0x558c54e79e40 prio 127 cost 0 latency 0.001920 MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=267, reqid=client.4540.0:403, at_version=55'37, trim_to=0'0, roll_forward_to=54'35)) v2 pg pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 55 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 54 -> 55 (as per caller) 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=267, reqid=client.4540.0:403, at_version=55'37, trim_to=0'0, roll_forward_to=54'35)) v2 2019-12-07T19:34:32.635+0000 7f8f05183700 20 snap_mapper.add_oid 1:ec1285ab:::smithi13140796-19:20 15,1b,20 2019-12-07T19:34:32.635+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.635+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ec1285ab:::smithi13140796-19:20 got.empty() 2019-12-07T19:34:32.635+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ec1285ab:::smithi13140796-19:20 15,1b,20 2019-12-07T19:34:32.635+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.635+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.635+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.635+0000 7f8f05183700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 54'35 (0'0,54'35] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,54'35], crt=54'35) [55'36 (53'33) clone 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [20,1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,55'37 (53'33) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:403 2019-12-07T19:34:32.633512+0000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 0, new_object: 0] 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'36 (0'0,55'36] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=54'35 crt=54'35 lcod 54'35 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'36 (53'33) clone 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [20,1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=54'35 crt=54'35 lcod 54'35 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'37 (53'33) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:403 2019-12-07T19:34:32.633512+0000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 0, new_object: 0 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=54'35 crt=54'35 lcod 54'35 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 37 2019-12-07T19:34:32.635+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=54'35 crt=54'35 lcod 54'35 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.635+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.635+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'36, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c583b6000 seq 82 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ec1285ab:::smithi13140796-19:20# key 0x848000000000000001ec1285ab21736d'ithi13140796-19!='0x0000000000000020ffffffffffffffff'o' 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:32.635+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:20# 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1260 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.onode(0x558c54e44000).flush flush done 2019-12-07T19:34:32.635+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# 0x0 2019-12-07T19:34:32.635+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:20# 0x0~5e000 -> 0x0~5e000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x0~2000: 0x0~2000 Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c580bbab0 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c580bbab0) get_ref 0x0~2000 Blob(0x558c580bbab0 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c580bbab0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x0~2000: 0x0~2000 Blob(0x558c580bbab0 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d3d030 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5750d500 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d500) get_ref 0x0~2000 Blob(0x558c5750d500 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d500) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c5750d500 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x4000~2000: 0x0~2000 Blob(0x558c54d3ccb0 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c57e62f50 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c57e62f50) get_ref 0x0~2000 Blob(0x558c57e62f50 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c57e62f50) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57e62f50 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x6000~2000: 0x0~2000 Blob(0x558c55265a40 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5750d5e0 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d5e0) get_ref 0x0~2000 Blob(0x558c5750d5e0 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5750d5e0 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x8000~2000: 0x0~2000 Blob(0x558c54e43030 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5750d420 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d420) get_ref 0x0~2000 Blob(0x558c5750d420 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d420) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5750d420 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0xa000~2000: 0x0~2000 Blob(0x558c54e43110 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5829ddc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5829ddc0) get_ref 0x0~2000 Blob(0x558c5829ddc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5829ddc0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5829ddc0 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0xc000~2000: 0x0~2000 Blob(0x558c54e431f0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5829dab0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5829dab0) get_ref 0x0~2000 Blob(0x558c5829dab0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5829dab0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c5829dab0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0xe000~2000: 0x0~2000 Blob(0x558c54e432d0 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5750d260 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d260) get_ref 0x0~2000 Blob(0x558c5750d260 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5750d260) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c5750d260 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x10000~2000: 0x0~2000 Blob(0x558c54e433b0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794ee00 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794ee00) get_ref 0x0~2000 Blob(0x558c5794ee00 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794ee00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5794ee00 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x12000~1000: 0x0~1000 Blob(0x558c54e43490 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794ee70 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794ee70) get_ref 0x0~1000 Blob(0x558c5794ee70 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794ee70) get_ref init 0x1000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x12000~1000: 0x0~1000 Blob(0x558c5794ee70 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x13000~2000: 0x0~2000 Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10681 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57566d90 blob([0x5f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57566d90 blob([0x5f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794eee0 blob([0x5f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794eee0) get_ref 0x0~2000 Blob(0x558c5794eee0 blob([0x5f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794eee0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x13000~2000: 0x0~2000 Blob(0x558c5794eee0 blob([0x5f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x15000~2000: 0x0~2000 Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10682 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57567b90 blob([0x5f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57567b90 blob([0x5f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794ef50 blob([0x5f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794ef50) get_ref 0x0~2000 Blob(0x558c5794ef50 blob([0x5f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794ef50) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x15000~2000: 0x0~2000 Blob(0x558c5794ef50 blob([0x5f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x17000~2000: 0x0~2000 Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10683 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c5742b810 blob([0x5f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c5742b810 blob([0x5f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794efc0 blob([0x5f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794efc0) get_ref 0x0~2000 Blob(0x558c5794efc0 blob([0x5f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794efc0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x17000~2000: 0x0~2000 Blob(0x558c5794efc0 blob([0x5f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x19000~2000: 0x0~2000 Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10684 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c5742b960 blob([0x5fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c5742b960 blob([0x5fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f030 blob([0x5fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f030) get_ref 0x0~2000 Blob(0x558c5794f030 blob([0x5fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f030) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x19000~2000: 0x0~2000 Blob(0x558c5794f030 blob([0x5fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x1b000~2000: 0x0~2000 Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10685 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c574fa930 blob([0x5fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c574fa930 blob([0x5fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f0a0 blob([0x5fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f0a0) get_ref 0x0~2000 Blob(0x558c5794f0a0 blob([0x5fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x1b000~2000: 0x0~2000 Blob(0x558c5794f0a0 blob([0x5fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x1d000~2000: 0x0~2000 Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10686 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57897730 blob([0x5ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57897730 blob([0x5ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f110 blob([0x5ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f110) get_ref 0x0~2000 Blob(0x558c5794f110 blob([0x5ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f110) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x1d000~2000: 0x0~2000 Blob(0x558c5794f110 blob([0x5ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x1f000~2000: 0x0~2000 Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10687 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57897650 blob([0x601000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57897650 blob([0x601000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f180 blob([0x601000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f180) get_ref 0x0~2000 Blob(0x558c5794f180 blob([0x601000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f180) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x1f000~2000: 0x0~2000 Blob(0x558c5794f180 blob([0x601000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x21000~2000: 0x0~2000 Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10688 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57897570 blob([0x603000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57897570 blob([0x603000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f1f0 blob([0x603000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f1f0) get_ref 0x0~2000 Blob(0x558c5794f1f0 blob([0x603000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x21000~2000: 0x0~2000 Blob(0x558c5794f1f0 blob([0x603000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x23000~2000: 0x0~2000 Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10689 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57897490 blob([0x605000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57897490 blob([0x605000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f260 blob([0x605000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f260) get_ref 0x0~2000 Blob(0x558c5794f260 blob([0x605000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f260) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c5794f260 blob([0x605000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x25000~2000: 0x0~2000 Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10690 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580a5570 blob([0x607000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580a5570 blob([0x607000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f2d0 blob([0x607000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f2d0) get_ref 0x0~2000 Blob(0x558c5794f2d0 blob([0x607000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c5794f2d0 blob([0x607000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x27000~2000: 0x0~2000 Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10691 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580a5650 blob([0x609000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580a5650 blob([0x609000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f340 blob([0x609000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f340) get_ref 0x0~2000 Blob(0x558c5794f340 blob([0x609000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f340) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c5794f340 blob([0x609000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x29000~2000: 0x0~2000 Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10692 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580a5490 blob([0x60b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580a5490 blob([0x60b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f3b0 blob([0x60b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f3b0) get_ref 0x0~2000 Blob(0x558c5794f3b0 blob([0x60b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c5794f3b0 blob([0x60b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10693 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580a5ce0 blob([0x60d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580a5ce0 blob([0x60d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f420 blob([0x60d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f420) get_ref 0x0~2000 Blob(0x558c5794f420 blob([0x60d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f420) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c5794f420 blob([0x60d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10694 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580a5c00 blob([0x60f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580a5c00 blob([0x60f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f490 blob([0x60f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f490) get_ref 0x0~2000 Blob(0x558c5794f490 blob([0x60f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f490) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c5794f490 blob([0x60f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x2f000~2000: 0x0~2000 Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10695 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c54e56460 blob([0x623000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c54e56460 blob([0x623000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f500 blob([0x623000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f500) get_ref 0x0~2000 Blob(0x558c5794f500 blob([0x623000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f500) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x2f000~2000: 0x0~2000 Blob(0x558c5794f500 blob([0x623000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x31000~2000: 0x0~2000 Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10696 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c436c0 blob([0x625000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c436c0 blob([0x625000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=1)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f570 blob([0x625000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f570) get_ref 0x0~2000 Blob(0x558c5794f570 blob([0x625000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.blob(0x558c5794f570) get_ref init 0x2000, 1000 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x31000~2000: 0x0~2000 Blob(0x558c5794f570 blob([0x625000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=2)))) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x33000~2000: 0x0~2000 Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) 2019-12-07T19:34:32.635+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10697 2019-12-07T19:34:32.635+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c43570 blob([0x627000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c43570 blob([0x627000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f5e0 blob([0x627000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f5e0) get_ref 0x0~2000 Blob(0x558c5794f5e0 blob([0x627000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x33000~2000: 0x0~2000 Blob(0x558c5794f5e0 blob([0x627000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x35000~2000: 0x0~2000 Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10698 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c43490 blob([0x629000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c43490 blob([0x629000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f650 blob([0x629000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f650) get_ref 0x0~2000 Blob(0x558c5794f650 blob([0x629000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f650) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x35000~2000: 0x0~2000 Blob(0x558c5794f650 blob([0x629000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x37000~2000: 0x0~2000 Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10699 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c433b0 blob([0x62b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c433b0 blob([0x62b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f6c0 blob([0x62b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f6c0) get_ref 0x0~2000 Blob(0x558c5794f6c0 blob([0x62b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x37000~2000: 0x0~2000 Blob(0x558c5794f6c0 blob([0x62b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x39000~2000: 0x0~2000 Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10700 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c432d0 blob([0x62d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c432d0 blob([0x62d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f730 blob([0x62d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f730) get_ref 0x0~2000 Blob(0x558c5794f730 blob([0x62d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f730) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x39000~2000: 0x0~2000 Blob(0x558c5794f730 blob([0x62d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x3b000~2000: 0x0~2000 Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10701 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c431f0 blob([0x62f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c431f0 blob([0x62f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f7a0 blob([0x62f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f7a0) get_ref 0x0~2000 Blob(0x558c5794f7a0 blob([0x62f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x3b000~2000: 0x0~2000 Blob(0x558c5794f7a0 blob([0x62f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x3d000~2000: 0x0~2000 Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10702 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c43110 blob([0x631000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c43110 blob([0x631000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f810 blob([0x631000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f810) get_ref 0x0~2000 Blob(0x558c5794f810 blob([0x631000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f810) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x3d000~2000: 0x0~2000 Blob(0x558c5794f810 blob([0x631000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x3f000~2000: 0x0~2000 Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10703 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c42770 blob([0x633000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c42770 blob([0x633000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f880 blob([0x633000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f880) get_ref 0x0~2000 Blob(0x558c5794f880 blob([0x633000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f880) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x3f000~2000: 0x0~2000 Blob(0x558c5794f880 blob([0x633000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x41000~2000: 0x0~2000 Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10704 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c42690 blob([0x635000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c42690 blob([0x635000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f8f0 blob([0x635000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f8f0) get_ref 0x0~2000 Blob(0x558c5794f8f0 blob([0x635000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x41000~2000: 0x0~2000 Blob(0x558c5794f8f0 blob([0x635000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x43000~2000: 0x0~2000 Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10705 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c425b0 blob([0x637000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c425b0 blob([0x637000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c54e56e00 blob([0x637000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56e00) get_ref 0x0~2000 Blob(0x558c54e56e00 blob([0x637000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56e00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x43000~2000: 0x0~2000 Blob(0x558c54e56e00 blob([0x637000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x45000~2000: 0x0~2000 Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10706 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c424d0 blob([0x639000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c424d0 blob([0x639000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c54e56d20 blob([0x639000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56d20) get_ref 0x0~2000 Blob(0x558c54e56d20 blob([0x639000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56d20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x45000~2000: 0x0~2000 Blob(0x558c54e56d20 blob([0x639000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x47000~2000: 0x0~2000 Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10707 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c43030 blob([0x63b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c43030 blob([0x63b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c54e56c40 blob([0x63b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56c40) get_ref 0x0~2000 Blob(0x558c54e56c40 blob([0x63b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56c40) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x47000~2000: 0x0~2000 Blob(0x558c54e56c40 blob([0x63b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x49000~2000: 0x0~2000 Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10708 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c42f50 blob([0x63d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c42f50 blob([0x63d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c54e56b60 blob([0x63d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56b60) get_ref 0x0~2000 Blob(0x558c54e56b60 blob([0x63d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56b60) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x49000~2000: 0x0~2000 Blob(0x558c54e56b60 blob([0x63d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x4b000~1000: 0x1000~1000 Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10709 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c54e56af0 blob([!~1000,0x63f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56af0) get_ref 0x1000~1000 Blob(0x558c54e56af0 blob([!~1000,0x63f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56af0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x4b000~1000: 0x1000~1000 Blob(0x558c54e56af0 blob([!~1000,0x63f000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10710 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57897d50 blob([0x640000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57897d50 blob([0x640000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c54e56a10 blob([0x640000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56a10) get_ref 0x0~2000 Blob(0x558c54e56a10 blob([0x640000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c54e56a10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c54e56a10 blob([0x640000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10711 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c43dc0 blob([0x642000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c43dc0 blob([0x642000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f960 blob([0x642000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f960) get_ref 0x0~2000 Blob(0x558c5794f960 blob([0x642000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f960) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c5794f960 blob([0x642000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x50000~2000: 0x0~2000 Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10712 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57c43f10 blob([0x644000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57c43f10 blob([0x644000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794f9d0 blob([0x644000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f9d0) get_ref 0x0~2000 Blob(0x558c5794f9d0 blob([0x644000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794f9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c5794f9d0 blob([0x644000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x52000~2000: 0x0~2000 Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10713 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57b96e00 blob([0x646000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57b96e00 blob([0x646000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794fa40 blob([0x646000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fa40) get_ref 0x0~2000 Blob(0x558c5794fa40 blob([0x646000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fa40) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c5794fa40 blob([0x646000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x54000~2000: 0x0~2000 Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10714 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57b96ee0 blob([0x648000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57b96ee0 blob([0x648000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794fab0 blob([0x648000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fab0) get_ref 0x0~2000 Blob(0x558c5794fab0 blob([0x648000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fab0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c5794fab0 blob([0x648000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x56000~2000: 0x0~2000 Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10715 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57b96fc0 blob([0x64a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57b96fc0 blob([0x64a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794fb20 blob([0x64a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fb20) get_ref 0x0~2000 Blob(0x558c5794fb20 blob([0x64a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fb20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c5794fb20 blob([0x64a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x58000~2000: 0x0~2000 Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10716 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57b970a0 blob([0x64c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57b970a0 blob([0x64c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794fb90 blob([0x64c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fb90) get_ref 0x0~2000 Blob(0x558c5794fb90 blob([0x64c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fb90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c5794fb90 blob([0x64c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10717 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57b97180 blob([0x64e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57b97180 blob([0x64e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794fc00 blob([0x64e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fc00) get_ref 0x0~2000 Blob(0x558c5794fc00 blob([0x64e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fc00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c5794fc00 blob([0x64e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10718 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c57b97260 blob([0x650000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 sbid 0x0)) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c57b97260 blob([0x650000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=1)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup new Blob(0x558c5794fc70 blob([0x650000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fc70) get_ref 0x0~2000 Blob(0x558c5794fc70 blob([0x650000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.blob(0x558c5794fc70) get_ref init 0x2000, 1000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c5794fc70 blob([0x650000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=2)))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark inline shard dirty 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) dirty_range mark inline shard dirty 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:20# = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# snapset 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# snapset = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# 1 keys 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# 1 keys = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# hinfo_key (42 bytes) 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ec1285ab:::smithi13140796-19:head#25 key 0x848000000000000001ec1285ab21736d'ithi13140796-19!='0xfffffffffffffffe0000000000000025'o' 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:head#25 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# -> 4#1:ec1285ab:::smithi13140796-19:head#25 = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1261 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- SNA_1_0000000000000015_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- SNA_1_000000000000001B_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- SNA_1_0000000000000020_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000055.00000000000000000036' <- 0000000055.00000000000000000036 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000055.00000000000000000037' <- 0000000055.00000000000000000037 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b6000 cost 8327 (1 ios * 4000 + 4327 bytes) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b6000 onodes 0x558c54e44000,0x558c585ce000,0x558c585ce280 shared_blobs 0x558c54677960,0x558c54cf9650,0x558c54d3d110,0x558c54e42fc0,0x558c54e430a0,0x558c54e43180,0x558c54e43260,0x558c54e43340,0x558c54e43420,0x558c54e43500,0x558c55265960,0x558c5742b730,0x558c5742b7a0,0x558c5742b8f0,0x558c57567ce0,0x558c57897500,0x558c578975e0,0x558c578976c0,0x558c578977a0,0x558c57b96e70,0x558c57b96f50,0x558c57b97030,0x558c57b97110,0x558c57b971f0,0x558c57b972d0,0x558c57c42540,0x558c57c42620,0x558c57c42700,0x558c57c427e0,0x558c57c42e00,0x558c57c42ee0,0x558c57c42fc0,0x558c57c430a0,0x558c57c43180,0x558c57c43260,0x558c57c43340,0x558c57c43420,0x558c57c43500,0x558c57c435e0,0x558c57c43650,0x558c57c43e30,0x558c57c43f80,0x558c580a55e0,0x558c580a56c0,0x558c580a5730,0x558c580a57a0,0x558c580a59d0,0x558c580a5c70 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head#25 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update inline shard 1311 bytes from 48 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore.extentmap(0x558c54e44108) reshard 0x[0,ffffffff) of 0 shards on 4#1:ec1285ab:::smithi13140796-19:head#25 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard new shard 0x0 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard new shard 0x23000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard new shard 0x47000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard old [] 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update 4#1:ec1285ab:::smithi13140796-19:head#25 force 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update shard 0x23000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c54e44108) update shard 0x47000 is 343 bytes (was 0) from 12 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head#25 is 500 (498 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update 4#1:ec1285ab:::smithi13140796-19:20# 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update inline shard 1311 bytes from 48 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore.extentmap(0x558c585ce108) reshard 0x[0,ffffffff) of 0 shards on 4#1:ec1285ab:::smithi13140796-19:20# 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard new shard 0x0 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard new shard 0x23000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard new shard 0x47000 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard old [] 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes)] 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update 4#1:ec1285ab:::smithi13140796-19:20# force 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update shard 0x23000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update shard 0x47000 is 343 bytes (was 0) from 12 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:20# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce388) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce388) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 491 (483 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c6 is 11 SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a0 is 11 SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a1 is 11 SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a3 is 11 SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a4 is 11 SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a5 is 11 SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a6 is 11 SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a7 is 11 SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a8 is 11 SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a9 is 11 SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a2 is 11 SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ba is 11 SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bc is 11 SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bb is 11 SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b9 is 11 SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c0 is 11 SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bf is 11 SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29be is 11 SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bd is 11 SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d9 is 11 SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29da is 11 SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29db is 11 SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29dc is 11 SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29dd is 11 SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29de is 11 SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d1 is 11 SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d0 is 11 SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cf is 11 SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ce is 11 SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d5 is 11 SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d4 is 11 SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d3 is 11 SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d2 is 11 SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cd is 11 SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cc is 11 SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cb is 11 SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ca is 11 SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c9 is 11 SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d6 is 11 SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c8 is 11 SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d7 is 11 SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d8 is 11 SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c3 is 11 SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c2 is 11 SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c7 is 11 SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c4 is 11 SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c1 is 11 SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c5 is 11 SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=2))) 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b6000 allocated 0x[] released 0x[] 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6000 prepare 2019-12-07T19:34:32.636+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b6000 2019-12-07T19:34:32.636+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6000 io_done 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.636+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=54'35 crt=54'35 lcod 54'35 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'37 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3467 < min_flush_size 524288 2019-12-07T19:34:32.636+0000 7f8f05183700 10 osd.7 55 dequeue_op 0x558c54e79e40 finish 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8203 < min_flush_size 524288 2019-12-07T19:34:32.636+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce000 had 1 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce280 had 1 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8226 < min_flush_size 524288 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8226 < min_flush_size 524288 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x31f275~2022 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x31f275 0x31f275~2022 to file(ino 16 size 0x320000 mtime 2019-12-07T19:34:31.631288+0000 allocated 320000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~2f0000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _allocate len 0x1297 from 0 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _allocate len 0x1297 from 1 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd450000 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd450000~10000/10000,10000,acd450000 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x330000 to include allocated 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 65 (was clean) 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~300000 x_off 0x2ef275 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x275 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x297 and padding block with 0xd69 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x330000 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x321297 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (65) on file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]), flushing log 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 65 len 0x4a crc 0x187dcfaa) 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x40000~1000 to file(ino 1 size 0x40000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x40000 0x40000~1000 to file(ino 1 size 0x40000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x41000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x40000 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x41000 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:32.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000020 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 65 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000436574s (0.000000116s flush + 0.000436458s kv commit) 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583b6000> 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6000 kv_submitted 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b6000 2019-12-07T19:34:32.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6000 finishing 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b6000 onodes 0x558c54e44000,0x558c585ce000,0x558c585ce280 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b6000 done 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:32.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b6000 [] 2019-12-07T19:34:32.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.636+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.636+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 55/41 ECSubWriteReply(tid=267, last_complete=55'37, committed=1, applied=1)) v2 -- 0x558c58512800 con 0x558c54c64000 2019-12-07T19:34:32.637+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1278622 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.637+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.637+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 456 ==== MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=268, reqid=client.4540.0:403, at_version=0'0, trim_to=0'0, roll_forward_to=55'37)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748d000 con 0x558c54c64000 2019-12-07T19:34:32.637+0000 7f8f24457700 15 osd.7 55 enqueue_op 0x558c54e79340 prio 127 cost 0 latency 0.000018 epoch 55 MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=268, reqid=client.4540.0:403, at_version=0'0, trim_to=0'0, roll_forward_to=55'37)) v2 2019-12-07T19:34:32.637+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=268, reqid=client.4540.0:403, at_version=0'0, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=268, reqid=client.4540.0:403, at_version=0'0, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=268, reqid=client.4540.0:403, at_version=0'0, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e55) pg 0x558c54d5f000 2019-12-07T19:34:32.637+0000 7f8f0117b700 10 osd.7 55 dequeue_op 0x558c54e79340 prio 127 cost 0 latency 0.000045 MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=268, reqid=client.4540.0:403, at_version=0'0, trim_to=0'0, roll_forward_to=55'37)) v2 pg pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.637+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.7s4 55/41 ECSubWrite(tid=268, reqid=client.4540.0:403, at_version=0'0, trim_to=0'0, roll_forward_to=55'37)) v2 2019-12-07T19:34:32.637+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=54'35 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'37], crt=54'35) [] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'36 (53'33) clone 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [20,1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'37 (53'33) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:403 2019-12-07T19:34:32.633512+0000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 0, new_object: 0 2019-12-07T19:34:32.637+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 37 2019-12-07T19:34:32.637+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.637+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.637+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.637+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57959500 seq 83 2019-12-07T19:34:32.637+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head#25 onode 0x558c54e44000 txc 0x558c57959500 2019-12-07T19:34:32.637+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head#25 0x0 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d180) put_ref 0x0~2000 Blob(0x558c54d3d180 blob([0x84000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d030) put_ref 0x0~2000 Blob(0x558c54d3d030 blob([0x86000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3ccb0) put_ref 0x0~2000 Blob(0x558c54d3ccb0 blob([0x88000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c55265a40) put_ref 0x0~2000 Blob(0x558c55265a40 blob([0x8a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43030) put_ref 0x0~2000 Blob(0x558c54e43030 blob([0x8c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43110) put_ref 0x0~2000 Blob(0x558c54e43110 blob([0x8e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54e431f0) put_ref 0x0~2000 Blob(0x558c54e431f0 blob([0x90000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54e432d0) put_ref 0x0~2000 Blob(0x558c54e432d0 blob([0x92000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54e433b0) put_ref 0x0~2000 Blob(0x558c54e433b0 blob([0x94000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43490) put_ref 0x0~1000 Blob(0x558c54e43490 blob([0x96000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57566d90) put_ref 0x0~2000 Blob(0x558c57566d90 blob([0x5f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57567b90) put_ref 0x0~2000 Blob(0x558c57567b90 blob([0x5f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c5742b810) put_ref 0x0~2000 Blob(0x558c5742b810 blob([0x5f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c5742b960) put_ref 0x0~2000 Blob(0x558c5742b960 blob([0x5fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c574fa930) put_ref 0x0~2000 Blob(0x558c574fa930 blob([0x5fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57897730) put_ref 0x0~2000 Blob(0x558c57897730 blob([0x5ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57897650) put_ref 0x0~2000 Blob(0x558c57897650 blob([0x601000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57897570) put_ref 0x0~2000 Blob(0x558c57897570 blob([0x603000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57897490) put_ref 0x0~2000 Blob(0x558c57897490 blob([0x605000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5570) put_ref 0x0~2000 Blob(0x558c580a5570 blob([0x607000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5650) put_ref 0x0~2000 Blob(0x558c580a5650 blob([0x609000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5490) put_ref 0x0~2000 Blob(0x558c580a5490 blob([0x60b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5ce0) put_ref 0x0~2000 Blob(0x558c580a5ce0 blob([0x60d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5c00) put_ref 0x0~2000 Blob(0x558c580a5c00 blob([0x60f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c54e56460) put_ref 0x0~2000 Blob(0x558c54e56460 blob([0x623000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c436c0) put_ref 0x0~2000 Blob(0x558c57c436c0 blob([0x625000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43570) put_ref 0x0~2000 Blob(0x558c57c43570 blob([0x627000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43490) put_ref 0x0~2000 Blob(0x558c57c43490 blob([0x629000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c433b0) put_ref 0x0~2000 Blob(0x558c57c433b0 blob([0x62b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c432d0) put_ref 0x0~2000 Blob(0x558c57c432d0 blob([0x62d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c431f0) put_ref 0x0~2000 Blob(0x558c57c431f0 blob([0x62f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43110) put_ref 0x0~2000 Blob(0x558c57c43110 blob([0x631000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42770) put_ref 0x0~2000 Blob(0x558c57c42770 blob([0x633000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42690) put_ref 0x0~2000 Blob(0x558c57c42690 blob([0x635000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c425b0) put_ref 0x0~2000 Blob(0x558c57c425b0 blob([0x637000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c424d0) put_ref 0x0~2000 Blob(0x558c57c424d0 blob([0x639000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43030) put_ref 0x0~2000 Blob(0x558c57c43030 blob([0x63b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42f50) put_ref 0x0~2000 Blob(0x558c57c42f50 blob([0x63d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c42e70) put_ref 0x1000~1000 Blob(0x558c57c42e70 blob([!~1000,0x63f000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57897d50) put_ref 0x0~2000 Blob(0x558c57897d50 blob([0x640000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43dc0) put_ref 0x0~2000 Blob(0x558c57c43dc0 blob([0x642000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57c43f10) put_ref 0x0~2000 Blob(0x558c57c43f10 blob([0x644000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96e00) put_ref 0x0~2000 Blob(0x558c57b96e00 blob([0x646000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96ee0) put_ref 0x0~2000 Blob(0x558c57b96ee0 blob([0x648000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96fc0) put_ref 0x0~2000 Blob(0x558c57b96fc0 blob([0x64a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57b970a0) put_ref 0x0~2000 Blob(0x558c57b970a0 blob([0x64c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97180) put_ref 0x0~2000 Blob(0x558c57b97180 blob([0x64e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97260) put_ref 0x0~2000 Blob(0x558c57b97260 blob([0x650000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark shard 0x23000 dirty 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore.extentmap(0x558c54e44108) dirty_range mark shard 0x47000 dirty 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54d3d180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d3d030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x86000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c54d3ccb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x88000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c55265a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x8a000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c54e43030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x8c000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c54e43110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x8e000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c54e431f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x90000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c54e432d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x92000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c54e433b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x94000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~1000: 0x0~1000 Blob(0x558c54e43490 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x96000~1000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x13000~2000: 0x0~2000 Blob(0x558c57566d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5f5000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x15000~2000: 0x0~2000 Blob(0x558c57567b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5f7000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x17000~2000: 0x0~2000 Blob(0x558c5742b810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5f9000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x19000~2000: 0x0~2000 Blob(0x558c5742b960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5fb000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1b000~2000: 0x0~2000 Blob(0x558c574fa930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5fd000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1d000~2000: 0x0~2000 Blob(0x558c57897730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ff000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1f000~2000: 0x0~2000 Blob(0x558c57897650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x601000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x21000~2000: 0x0~2000 Blob(0x558c57897570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x603000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x23000~2000: 0x0~2000 Blob(0x558c57897490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x605000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x25000~2000: 0x0~2000 Blob(0x558c580a5570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x607000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x27000~2000: 0x0~2000 Blob(0x558c580a5650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x609000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x29000~2000: 0x0~2000 Blob(0x558c580a5490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x60b000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2b000~2000: 0x0~2000 Blob(0x558c580a5ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x60d000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2d000~2000: 0x0~2000 Blob(0x558c580a5c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x60f000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2f000~2000: 0x0~2000 Blob(0x558c54e56460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x623000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x31000~2000: 0x0~2000 Blob(0x558c57c436c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x625000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x33000~2000: 0x0~2000 Blob(0x558c57c43570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x627000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x35000~2000: 0x0~2000 Blob(0x558c57c43490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x629000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x37000~2000: 0x0~2000 Blob(0x558c57c433b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x62b000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x39000~2000: 0x0~2000 Blob(0x558c57c432d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x62d000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3b000~2000: 0x0~2000 Blob(0x558c57c431f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x62f000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3d000~2000: 0x0~2000 Blob(0x558c57c43110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x631000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3f000~2000: 0x0~2000 Blob(0x558c57c42770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x633000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x41000~2000: 0x0~2000 Blob(0x558c57c42690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x635000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x43000~2000: 0x0~2000 Blob(0x558c57c425b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x637000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x45000~2000: 0x0~2000 Blob(0x558c57c424d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x639000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x47000~2000: 0x0~2000 Blob(0x558c57c43030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x63b000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x49000~2000: 0x0~2000 Blob(0x558c57c42f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x63d000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4b000~1000: 0x1000~1000 Blob(0x558c57c42e70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x63f000~1000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~2000: 0x0~2000 Blob(0x558c57897d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x640000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4e000~2000: 0x0~2000 Blob(0x558c57c43dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x642000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x50000~2000: 0x0~2000 Blob(0x558c57c43f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=2)))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x644000~2000] 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=1))) 2019-12-07T19:34:32.637+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x52000~2000: 0x0~2000 Blob(0x558c57b96e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=2)))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x646000~2000] 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x54000~2000: 0x0~2000 Blob(0x558c57b96ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=2)))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x648000~2000] 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x56000~2000: 0x0~2000 Blob(0x558c57b96fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=2)))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x64a000~2000] 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x58000~2000: 0x0~2000 Blob(0x558c57b970a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=2)))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x64c000~2000] 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5a000~2000: 0x0~2000 Blob(0x558c57b97180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=2)))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x64e000~2000] 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5c000~2000: 0x0~2000 Blob(0x558c57b97260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=2)))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x650000~2000] 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x23000 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x47000 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54677960,0x558c54cf9650,0x558c54d3d110,0x558c54e42fc0,0x558c54e430a0,0x558c54e43180,0x558c54e43260,0x558c54e43340,0x558c54e43420,0x558c54e43500,0x558c55265960,0x558c5742b730,0x558c5742b7a0,0x558c5742b8f0,0x558c57567ce0,0x558c57897500,0x558c578975e0,0x558c578976c0,0x558c578977a0,0x558c57b96e70,0x558c57b96f50,0x558c57b97030,0x558c57b97110,0x558c57b971f0,0x558c57b972d0,0x558c57c42540,0x558c57c42620,0x558c57c42700,0x558c57c427e0,0x558c57c42e00,0x558c57c42ee0,0x558c57c42fc0,0x558c57c430a0,0x558c57c43180,0x558c57c43260,0x558c57c43340,0x558c57c43420,0x558c57c43500,0x558c57c435e0,0x558c57c43650,0x558c57c43e30,0x558c57c43f80,0x558c580a55e0,0x558c580a56c0,0x558c580a5730,0x558c580a57a0,0x558c580a59d0,0x558c580a5c70 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c585ce280 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head#25 = 0 2019-12-07T19:34:32.638+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57959500 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57959500 onodes shared_blobs 0x558c54677960,0x558c54cf9650,0x558c54d3d110,0x558c54e42fc0,0x558c54e430a0,0x558c54e43180,0x558c54e43260,0x558c54e43340,0x558c54e43420,0x558c54e43500,0x558c55265960,0x558c5742b730,0x558c5742b7a0,0x558c5742b8f0,0x558c57567ce0,0x558c57897500,0x558c578975e0,0x558c578976c0,0x558c578977a0,0x558c57b96e70,0x558c57b96f50,0x558c57b97030,0x558c57b97110,0x558c57b971f0,0x558c57b972d0,0x558c57c42540,0x558c57c42620,0x558c57c42700,0x558c57c427e0,0x558c57c42e00,0x558c57c42ee0,0x558c57c42fc0,0x558c57c430a0,0x558c57c43180,0x558c57c43260,0x558c57c43340,0x558c57c43420,0x558c57c43500,0x558c57c435e0,0x558c57c43650,0x558c57c43e30,0x558c57c43f80,0x558c580a55e0,0x558c580a56c0,0x558c580a5730,0x558c580a57a0,0x558c580a59d0,0x558c580a5c70 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c6 is 11 SharedBlob(0x558c54677960 loaded (sbid 0x29c6 ref_map(0x60f000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a0 is 11 SharedBlob(0x558c54cf9650 loaded (sbid 0x28a0 ref_map(0x84000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a1 is 11 SharedBlob(0x558c54d3d110 loaded (sbid 0x28a1 ref_map(0x86000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a3 is 11 SharedBlob(0x558c54e42fc0 loaded (sbid 0x28a3 ref_map(0x8a000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a4 is 11 SharedBlob(0x558c54e430a0 loaded (sbid 0x28a4 ref_map(0x8c000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a5 is 11 SharedBlob(0x558c54e43180 loaded (sbid 0x28a5 ref_map(0x8e000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a6 is 11 SharedBlob(0x558c54e43260 loaded (sbid 0x28a6 ref_map(0x90000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a7 is 11 SharedBlob(0x558c54e43340 loaded (sbid 0x28a7 ref_map(0x92000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a8 is 11 SharedBlob(0x558c54e43420 loaded (sbid 0x28a8 ref_map(0x94000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a9 is 11 SharedBlob(0x558c54e43500 loaded (sbid 0x28a9 ref_map(0x96000~1000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28a2 is 11 SharedBlob(0x558c55265960 loaded (sbid 0x28a2 ref_map(0x88000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ba is 11 SharedBlob(0x558c5742b730 loaded (sbid 0x29ba ref_map(0x5f7000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bc is 11 SharedBlob(0x558c5742b7a0 loaded (sbid 0x29bc ref_map(0x5fb000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bb is 11 SharedBlob(0x558c5742b8f0 loaded (sbid 0x29bb ref_map(0x5f9000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29b9 is 11 SharedBlob(0x558c57567ce0 loaded (sbid 0x29b9 ref_map(0x5f5000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c0 is 11 SharedBlob(0x558c57897500 loaded (sbid 0x29c0 ref_map(0x603000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bf is 11 SharedBlob(0x558c578975e0 loaded (sbid 0x29bf ref_map(0x601000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29be is 11 SharedBlob(0x558c578976c0 loaded (sbid 0x29be ref_map(0x5ff000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29bd is 11 SharedBlob(0x558c578977a0 loaded (sbid 0x29bd ref_map(0x5fd000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d9 is 11 SharedBlob(0x558c57b96e70 loaded (sbid 0x29d9 ref_map(0x646000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29da is 11 SharedBlob(0x558c57b96f50 loaded (sbid 0x29da ref_map(0x648000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29db is 11 SharedBlob(0x558c57b97030 loaded (sbid 0x29db ref_map(0x64a000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29dc is 11 SharedBlob(0x558c57b97110 loaded (sbid 0x29dc ref_map(0x64c000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29dd is 11 SharedBlob(0x558c57b971f0 loaded (sbid 0x29dd ref_map(0x64e000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29de is 11 SharedBlob(0x558c57b972d0 loaded (sbid 0x29de ref_map(0x650000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d1 is 11 SharedBlob(0x558c57c42540 loaded (sbid 0x29d1 ref_map(0x637000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d0 is 11 SharedBlob(0x558c57c42620 loaded (sbid 0x29d0 ref_map(0x635000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cf is 11 SharedBlob(0x558c57c42700 loaded (sbid 0x29cf ref_map(0x633000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ce is 11 SharedBlob(0x558c57c427e0 loaded (sbid 0x29ce ref_map(0x631000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d5 is 11 SharedBlob(0x558c57c42e00 loaded (sbid 0x29d5 ref_map(0x63f000~1000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d4 is 11 SharedBlob(0x558c57c42ee0 loaded (sbid 0x29d4 ref_map(0x63d000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d3 is 11 SharedBlob(0x558c57c42fc0 loaded (sbid 0x29d3 ref_map(0x63b000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d2 is 11 SharedBlob(0x558c57c430a0 loaded (sbid 0x29d2 ref_map(0x639000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cd is 11 SharedBlob(0x558c57c43180 loaded (sbid 0x29cd ref_map(0x62f000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cc is 11 SharedBlob(0x558c57c43260 loaded (sbid 0x29cc ref_map(0x62d000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29cb is 11 SharedBlob(0x558c57c43340 loaded (sbid 0x29cb ref_map(0x62b000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ca is 11 SharedBlob(0x558c57c43420 loaded (sbid 0x29ca ref_map(0x629000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c9 is 11 SharedBlob(0x558c57c43500 loaded (sbid 0x29c9 ref_map(0x627000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d6 is 11 SharedBlob(0x558c57c435e0 loaded (sbid 0x29d6 ref_map(0x640000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c8 is 11 SharedBlob(0x558c57c43650 loaded (sbid 0x29c8 ref_map(0x625000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d7 is 11 SharedBlob(0x558c57c43e30 loaded (sbid 0x29d7 ref_map(0x642000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29d8 is 11 SharedBlob(0x558c57c43f80 loaded (sbid 0x29d8 ref_map(0x644000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c3 is 11 SharedBlob(0x558c580a55e0 loaded (sbid 0x29c3 ref_map(0x609000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c2 is 11 SharedBlob(0x558c580a56c0 loaded (sbid 0x29c2 ref_map(0x607000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c7 is 11 SharedBlob(0x558c580a5730 loaded (sbid 0x29c7 ref_map(0x623000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c4 is 11 SharedBlob(0x558c580a57a0 loaded (sbid 0x29c4 ref_map(0x60b000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c1 is 11 SharedBlob(0x558c580a59d0 loaded (sbid 0x29c1 ref_map(0x605000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29c5 is 11 SharedBlob(0x558c580a5c70 loaded (sbid 0x29c5 ref_map(0x60d000~2000=1))) 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57959500 allocated 0x[] released 0x[] 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959500 prepare 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57959500 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959500 io_done 2019-12-07T19:34:32.638+0000 7f8f0117b700 10 osd.7 55 dequeue_op 0x558c54e79340 finish 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2550 < min_flush_size 524288 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44000 had 1 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2573 < min_flush_size 524288 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2573 < min_flush_size 524288 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x321297~a0d to file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x321297 0x321297~a0d to file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~300000 x_off 0x2f1297 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x297 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xca4 and padding block with 0x35c 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x321ca4 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57959500> 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000133705s (0.000000089s flush + 0.000133616s kv commit) 2019-12-07T19:34:32.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959500 kv_submitted 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57959500 2019-12-07T19:34:32.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959500 finishing 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57959500 onodes 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57959500 done 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:32.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57959500 [] 2019-12-07T19:34:32.638+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.638+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 55/41 ECSubWriteReply(tid=268, last_complete=55'37, committed=1, applied=1)) v2 -- 0x558c58513800 con 0x558c54c64000 2019-12-07T19:34:32.638+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.638+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 457 ==== MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=269, reqid=client.4540.0:404, at_version=55'40, trim_to=0'0, roll_forward_to=53'38)) v2 ==== 171258+0+0 (crc 0 0 0) 0x558c57e01800 con 0x558c54c64000 2019-12-07T19:34:32.651+0000 7f8f24457700 15 osd.7 55 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000050 epoch 55 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=269, reqid=client.4540.0:404, at_version=55'40, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:32.651+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=269, reqid=client.4540.0:404, at_version=55'40, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=269, reqid=client.4540.0:404, at_version=55'40, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.651+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=269, reqid=client.4540.0:404, at_version=55'40, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) pg 0x558c5500a000 2019-12-07T19:34:32.651+0000 7f8f04181700 10 osd.7 55 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000093 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=269, reqid=client.4540.0:404, at_version=55'40, trim_to=0'0, roll_forward_to=53'38)) v2 pg pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.651+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=269, reqid=client.4540.0:404, at_version=55'40, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:32.651+0000 7f8f04181700 20 snap_mapper.add_oid 1:9b8adabe:::smithi13140796-6:20 15,1b,20 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.651+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:20 got.empty() 2019-12-07T19:34:32.651+0000 7f8f04181700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:20 15,1b,20 2019-12-07T19:34:32.651+0000 7f8f04181700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.651+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_0000000000000015_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.651+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.651+0000 7f8f04181700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.651+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 53'38 (0'0,53'38] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,53'38], crt=53'38) [55'39 (53'38) clone 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:18.620957+0000 0 snaps [20,1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,55'40 (53'38) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:404 2019-12-07T19:34:32.649174+0000 0 ObjectCleanRegions clean_offsets: [0~1261568,1933312~18446744073707618303], clean_omap: 1, new_object: 0] 2019-12-07T19:34:32.651+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'39 (0'0,55'39] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'38 crt=53'38 lcod 53'38 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'39 (53'38) clone 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:18.620957+0000 0 snaps [20,1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.651+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'40 (0'0,55'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'38 crt=53'38 lcod 53'38 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'40 (53'38) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:404 2019-12-07T19:34:32.649174+0000 0 ObjectCleanRegions clean_offsets: [0~1261568,1933312~18446744073707618303], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.651+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'40 (0'0,55'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'38 crt=53'38 lcod 53'38 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 40 2019-12-07T19:34:32.651+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'40 (0'0,55'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'38 crt=53'38 lcod 53'38 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.651+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.651+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'39, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57956600 seq 78 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:20# key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0x0000000000000020ffffffffffffffff'o' 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:20# 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1262 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.onode(0x558c57a35400).flush flush done 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# 0x0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:20# 0x0~4d000 -> 0x0~4d000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x0~2000: 0x0~2000 Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3dc0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3dc0) get_ref 0x0~2000 Blob(0x558c579b3dc0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x0~2000: 0x0~2000 Blob(0x558c579b3dc0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5794e310 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5794e310) get_ref 0x0~2000 Blob(0x558c5794e310 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5794e310) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c5794e310 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4000~2000: 0x0~2000 Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f515e0 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f515e0) get_ref 0x0~2000 Blob(0x558c57f515e0 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f515e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57f515e0 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54e42540 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5794e380 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5794e380) get_ref 0x0~2000 Blob(0x558c5794e380 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5794e380) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5794e380 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x8000~2000: 0x0~2000 Blob(0x558c578813b0 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5794e2a0 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5794e2a0) get_ref 0x0~2000 Blob(0x558c5794e2a0 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5794e2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5794e2a0 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57881490 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c574fdb90 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c574fdb90) get_ref 0x0~2000 Blob(0x558c574fdb90 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c574fdb90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c574fdb90 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57880bd0 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c574fdc00 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c574fdc00) get_ref 0x0~2000 Blob(0x558c574fdc00 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c574fdc00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c574fdc00 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0xe000~2000: 0x0~2000 Blob(0x558c578de310 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57818460 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818460) get_ref 0x0~2000 Blob(0x558c57818460 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818460) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57818460 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x10000~2000: 0x0~2000 Blob(0x558c578de770 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c576adce0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c576adce0) get_ref 0x0~2000 Blob(0x558c576adce0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c576adce0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c576adce0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x12000~2000: 0x0~2000 Blob(0x558c578de5b0 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5898aee0 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5898aee0) get_ref 0x0~2000 Blob(0x558c5898aee0 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5898aee0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c5898aee0 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x14000~2000: 0x0~2000 Blob(0x558c578debd0 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3e30 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3e30) get_ref 0x0~2000 Blob(0x558c579b3e30 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3e30) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c579b3e30 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x16000~2000: 0x0~2000 Blob(0x558c578dea10 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c578f0bd0 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0bd0) get_ref 0x0~2000 Blob(0x558c578f0bd0 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c578f0bd0 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x18000~2000: 0x0~2000 Blob(0x558c578decb0 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b29a0 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b29a0) get_ref 0x0~2000 Blob(0x558c579b29a0 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b29a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c579b29a0 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c578def50 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5898ae00 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5898ae00) get_ref 0x0~2000 Blob(0x558c5898ae00 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c5898ae00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c5898ae00 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c578df030 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2930 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2930) get_ref 0x0~2000 Blob(0x558c579b2930 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2930) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c579b2930 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c578df110 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f51a40 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f51a40) get_ref 0x0~2000 Blob(0x558c57f51a40 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f51a40) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c57f51a40 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x20000~2000: 0x0~2000 Blob(0x558c578df1f0 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c576add50 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c576add50) get_ref 0x0~2000 Blob(0x558c576add50 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c576add50) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c576add50 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x22000~2000: 0x0~2000 Blob(0x558c54e1c070 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c578f0d90 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0d90) get_ref 0x0~2000 Blob(0x558c578f0d90 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0d90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c578f0d90 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x24000~2000: 0x0~2000 Blob(0x558c54e1c150 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3ea0 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3ea0) get_ref 0x0~2000 Blob(0x558c579b3ea0 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c579b3ea0 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x26000~2000: 0x0~2000 Blob(0x558c54e1c8c0 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c578f0b60 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0b60) get_ref 0x0~2000 Blob(0x558c578f0b60 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0b60) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c578f0b60 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x28000~2000: 0x0~2000 Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2a80 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a80) get_ref 0x0~2000 Blob(0x558c579b2a80 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a80) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c579b2a80 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c578f1880 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f516c0 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f516c0) get_ref 0x0~2000 Blob(0x558c57f516c0 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f516c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c57f516c0 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c574e0d90 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2a10 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a10) get_ref 0x0~2000 Blob(0x558c579b2a10 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2a10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c579b2a10 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c574e0930 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57818310 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818310) get_ref 0x0~2000 Blob(0x558c57818310 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818310) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c57818310 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x30000~2000: 0x0~2000 Blob(0x558c578df260 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b21c0 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b21c0) get_ref 0x0~2000 Blob(0x558c579b21c0 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b21c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c579b21c0 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x32000~2000: 0x0~2000 Blob(0x558c578ded90 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c54cf8b60 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8b60) get_ref 0x0~2000 Blob(0x558c54cf8b60 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c54cf8b60) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c54cf8b60 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x34000~2000: 0x0~2000 Blob(0x558c578dee70 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3f10 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f10) get_ref 0x0~2000 Blob(0x558c579b3f10 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c579b3f10 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x36000~2000: 0x0~2000 Blob(0x558c54e1c700 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57819500 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57819500) get_ref 0x0~2000 Blob(0x558c57819500 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57819500) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c57819500 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x38000~2000: 0x0~2000 Blob(0x558c54e1c620 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2b60 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2b60) get_ref 0x0~2000 Blob(0x558c579b2b60 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c579b2b60 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c54e1c540 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c578f0d20 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0d20) get_ref 0x0~2000 Blob(0x558c578f0d20 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c578f0d20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c578f0d20 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2af0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2af0) get_ref 0x0~2000 Blob(0x558c579b2af0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2af0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c579b2af0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f50d90 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f50d90) get_ref 0x0~2000 Blob(0x558c57f50d90 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f50d90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c57f50d90 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x40000~2000: 0x0~2000 Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57818bd0 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818bd0) get_ref 0x0~2000 Blob(0x558c57818bd0 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c57818bd0 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x42000~2000: 0x0~2000 Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57db53b0 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57db53b0) get_ref 0x0~2000 Blob(0x558c57db53b0 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57db53b0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c57db53b0 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x44000~2000: 0x0~2000 Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b3f80 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f80) get_ref 0x0~2000 Blob(0x558c579b3f80 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b3f80) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c579b3f80 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x46000~2000: 0x0~2000 Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57818070 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818070) get_ref 0x0~2000 Blob(0x558c57818070 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57818070) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c57818070 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x48000~2000: 0x0~2000 Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2c40 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2c40) get_ref 0x0~2000 Blob(0x558c579b2c40 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2c40) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c579b2c40 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f50d20 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f50d20) get_ref 0x0~2000 Blob(0x558c57f50d20 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c57f50d20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c57f50d20 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4c000~1000: 0x0~1000 Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=1)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579b2bd0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2bd0) get_ref 0x0~1000 Blob(0x558c579b2bd0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.blob(0x558c579b2bd0) get_ref init 0x1000, 1000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4c000~1000: 0x0~1000 Blob(0x558c579b2bd0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore.extentmap(0x558c57573a08) dirty_range mark inline shard dirty 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:20# = 0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# snapset 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# snapset = 0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# 1 keys 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# 1 keys = 0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# hinfo_key (42 bytes) 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:32.651+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4d000~29000 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9b8adabe:::smithi13140796-6:head# 0x4d000~29000 - have 0x4d000 (315392) bytes fadvise_flags 0x20 2019-12-07T19:34:32.651+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:32.651+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x4d000~29000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c57956600 21 blobs 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.652+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate 0x29000/1000,29000,0 2019-12-07T19:34:32.652+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x33d000~13000/1000,29000,0 2019-12-07T19:34:32.652+0000 7f8f04181700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x377000~16000/1000,29000,0 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x33d000~13000,0x377000~16000] 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818000 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc00 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57818000) get_ref 0x0~2000 Blob(0x558c57818000 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc00 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57818000) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c57818000 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc00 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c576adc18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f50cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f50cb0 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970000 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57f50cb0) get_ref 0x0~2000 Blob(0x558c57f50cb0 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970000 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57f50cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558c57f50cb0 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970000 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57970018 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819490 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57819490) get_ref 0x0~2000 Blob(0x558c57819490 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57819490) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x51000~2000: 0x0~2000 Blob(0x558c57819490 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579b2d38 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5490 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2cb0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5490) get_ref 0x0~2000 Blob(0x558c57db5490 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2cb0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5490) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x53000~2000: 0x0~2000 Blob(0x558c57db5490 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2cb0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579b2cc8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f16c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f16c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c578f16c0) get_ref 0x0~2000 Blob(0x558c578f16c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c578f16c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x55000~2000: 0x0~2000 Blob(0x558c578f16c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c576adc88 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5420 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970070 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5420) get_ref 0x0~2000 Blob(0x558c57db5420 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970070 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5420) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x57000~2000: 0x0~2000 Blob(0x558c57db5420 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970070 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57970088 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578181c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578181c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c578181c0) get_ref 0x0~2000 Blob(0x558c578181c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c578181c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x59000~2000: 0x0~2000 Blob(0x558c578181c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579b2e18 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e000 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d90 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e000) get_ref 0x0~2000 Blob(0x558c5794e000 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d90 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e000) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5b000~2000: 0x0~2000 Blob(0x558c5794e000 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d90 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579b2da8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578180e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578180e0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2150 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c578180e0) get_ref 0x0~2000 Blob(0x558c578180e0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2150 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c578180e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5d000~2000: 0x0~2000 Blob(0x558c578180e0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2150 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579b2168 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579700e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e070 blob([0x34f000~1000,0x377000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579700e0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e070) get_ref 0x0~2000 Blob(0x558c5794e070 blob([0x34f000~1000,0x377000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579700e0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e070) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5f000~2000: 0x0~2000 Blob(0x558c5794e070 blob([0x34f000~1000,0x377000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579700e0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579700f8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819420 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57819420) get_ref 0x0~2000 Blob(0x558c57819420 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57819420) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x61000~2000: 0x0~2000 Blob(0x558c57819420 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579b2ef8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e0e0 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e0e0) get_ref 0x0~2000 Blob(0x558c5794e0e0 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x63000~2000: 0x0~2000 Blob(0x558c5794e0e0 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c579b2e88 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db58f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f67a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db58f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67a40 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db58f0) get_ref 0x0~2000 Blob(0x558c57db58f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67a40 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db58f0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x65000~2000: 0x0~2000 Blob(0x558c57db58f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67a40 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f67a58 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f67960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e150 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67960 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e150) get_ref 0x0~2000 Blob(0x558c5794e150 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67960 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e150) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x67000~2000: 0x0~2000 Blob(0x558c5794e150 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67960 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f67978 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f678f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5880 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f678f0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5880) get_ref 0x0~2000 Blob(0x558c57db5880 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f678f0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5880) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x69000~2000: 0x0~2000 Blob(0x558c57db5880 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f678f0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f67908 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57314000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e1c0 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314000 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e1c0) get_ref 0x0~2000 Blob(0x558c5794e1c0 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314000 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6b000~2000: 0x0~2000 Blob(0x558c5794e1c0 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314000 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57314018 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db5810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809ae70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db5810 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809ae70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5810) get_ref 0x0~2000 Blob(0x558c57db5810 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809ae70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db5810) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6d000~2000: 0x0~2000 Blob(0x558c57db5810 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809ae70 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5809ae88 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f67880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e230 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67880 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e230) get_ref 0x0~2000 Blob(0x558c5794e230 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67880 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e230) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6f000~2000: 0x0~2000 Blob(0x558c5794e230 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67880 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c57f67898 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57db57a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794e3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57db57a0 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e3f0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db57a0) get_ref 0x0~2000 Blob(0x558c57db57a0 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e3f0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c57db57a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x71000~2000: 0x0~2000 Blob(0x558c57db57a0 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e3f0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5794e408 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794e4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e460 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e4d0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e460) get_ref 0x0~2000 Blob(0x558c5794e460 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e4d0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e460) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x73000~2000: 0x0~2000 Blob(0x558c5794e460 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e4d0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5794e4e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794e540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794e5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794e540 blob([!~1000,0x38c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e5b0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e540) get_ref 0x1000~1000 Blob(0x558c5794e540 blob([!~1000,0x38c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e5b0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.blob(0x558c5794e540) get_ref init 0x2000, 1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x75000~1000: 0x1000~1000 Blob(0x558c5794e540 blob([!~1000,0x38c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5794e5b0 sbid 0x0)) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.BufferSpace(0x558c5794e5c8 in 0x558c53a59a40) _discard 0x1000~1000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x76000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) dirty_range mark inline shard dirty 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x4d000~29000 = 0 2019-12-07T19:34:32.652+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.652+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.652+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- SNA_1_0000000000000015_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- SNA_1_000000000000001B_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- SNA_1_0000000000000020_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.652+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000055.00000000000000000039' <- 0000000055.00000000000000000039 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000055.00000000000000000040' <- 0000000055.00000000000000000040 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57956600 cost 264618 (23 ios * 4000 + 172618 bytes) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57956600 onodes 0x558c57573900,0x558c57a35400 shared_blobs 0x558c54e1c000,0x558c54e1c0e0,0x558c54e1c4d0,0x558c54e1c5b0,0x558c54e1c690,0x558c54e1c770,0x558c54e1c850,0x558c54e1c930,0x558c54e1ca10,0x558c54e1caf0,0x558c54e1cbd0,0x558c54e1ccb0,0x558c54e1cd90,0x558c54e1ce70,0x558c54e1cf50,0x558c54e1d030,0x558c54e1d110,0x558c574e0e00,0x558c574e1ce0,0x558c57880f50,0x558c57881180,0x558c57881340,0x558c57881420,0x558c578de3f0,0x558c578de4d0,0x558c578de690,0x558c578de930,0x558c578deaf0,0x558c578ded20,0x558c578dee00,0x558c578deee0,0x558c578defc0,0x558c578df0a0,0x558c578df180,0x558c578f1730,0x558c578f17a0,0x558c57bea070,0x558c57dcac40,0x558c57dcb6c0 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57573a08) update 4#1:9b8adabe:::smithi13140796-6:20# 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57573a08) update inline shard 1053 bytes from 39 extents 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:20# is 124356 (123297 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update inline shard 1471 bytes from 60 extents 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore.extentmap(0x558c57a35508) reshard 0x[0,ffffffff) of 0 shards on 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard extent_avg 24, target 500, slop 100 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard new shard 0x0 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard new shard 0x28000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard new shard 0x4f000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard new [0x0(0x0 bytes),0x28000(0x0 bytes),0x4f000(0x0 bytes)] 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard old [] 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard fin [0x0(0x0 bytes),0x28000(0x0 bytes),0x4f000(0x0 bytes)] 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# force 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update shard 0x0 is 543 bytes (was 0) from 20 extents 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update shard 0x28000 is 535 bytes (was 0) from 20 extents 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update shard 0x4f000 is 403 bytes (was 0) from 20 extents 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 123473 (123471 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ba is 11 SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bb is 11 SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c7 is 11 SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c6 is 11 SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c5 is 11 SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c4 is 11 SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bd is 11 SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bc is 11 SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c8 is 11 SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c9 is 11 SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ca is 11 SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cb is 11 SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cc is 11 SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cd is 11 SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ce is 11 SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cf is 11 SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d0 is 11 SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c1 is 11 SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c0 is 11 SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b0 is 11 SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ad is 11 SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28af is 11 SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ae is 11 SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b1 is 11 SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b3 is 11 SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b2 is 11 SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b5 is 11 SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b4 is 11 SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c2 is 11 SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c3 is 11 SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b6 is 11 SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b7 is 11 SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b8 is 11 SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b9 is 11 SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bf is 11 SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28be is 11 SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ac is 11 SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ab is 11 SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28aa is 11 SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2))) 2019-12-07T19:34:32.652+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57956600 allocated 0x[33d000~13000,377000~16000] released 0x[] 2019-12-07T19:34:32.652+0000 7f8f04181700 10 freelist allocate 0x33d000~13000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:32.652+0000 7f8f04181700 10 freelist allocate 0x377000~16000 2019-12-07T19:34:32.652+0000 7f8f04181700 20 freelist _xor first_key 0x300000 last_key 0x380000 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956600 prepare 2019-12-07T19:34:32.652+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c57956600 2019-12-07T19:34:32.653+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.653+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'40 (0'0,55'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=53'38 crt=53'38 lcod 53'38 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'40 2019-12-07T19:34:32.653+0000 7f8f04181700 10 osd.7 55 dequeue_op 0x558c54e78840 finish 2019-12-07T19:34:32.653+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956600 aio_wait 2019-12-07T19:34:32.653+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57956600 2019-12-07T19:34:32.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956600 io_done 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 25436 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 58204 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 90972 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 123740 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 156508 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 189276 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 222044 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 253503 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573900 had 1 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 253526 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 253526 < min_flush_size 524288 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x321ca4~3de56 to file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x321ca4 0x321ca4~3de56 to file(ino 16 size 0x330000 mtime 2019-12-07T19:34:32.637815+0000 allocated 330000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~300000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _allocate len 0x2fafa from 0 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _allocate len 0x2fafa from 1 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x30000/10000,30000,acd460000 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd460000~30000/10000,30000,acd460000 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x360000 to include allocated 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 66 (was clean) 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x360000 mtime 2019-12-07T19:34:32.654739+0000 allocated 360000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~330000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~330000 x_off 0x2f1ca4 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xca4 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xafa and padding block with 0x506 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x35fafa 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (66) on file(ino 16 size 0x360000 mtime 2019-12-07T19:34:32.654739+0000 allocated 360000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~330000]), flushing log 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x360000 mtime 2019-12-07T19:34:32.654739+0000 allocated 360000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~330000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 66 len 0x4a crc 0xb8a2726c) 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x41000~1000 to file(ino 1 size 0x41000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x41000 0x41000~1000 to file(ino 1 size 0x41000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x42000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x41000 2019-12-07T19:34:32.653+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 184 ==== osd_map(55..55 src has 1..55) v4 ==== 633+0+0 (crc 0 0 0) 0x558c5781cb40 con 0x558c54c19800 2019-12-07T19:34:32.653+0000 7f8f1199c700 20 osd.7 55 OSD::ms_dispatch: osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.653+0000 7f8f1199c700 10 osd.7 55 do_waiters -- start 2019-12-07T19:34:32.653+0000 7f8f1199c700 10 osd.7 55 do_waiters -- finish 2019-12-07T19:34:32.653+0000 7f8f1199c700 20 osd.7 55 _dispatch 0x558c5781cb40 osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.653+0000 7f8f1199c700 3 osd.7 55 handle_osd_map epochs [55,55], i have 55, src has [1,55] 2019-12-07T19:34:32.653+0000 7f8f1199c700 10 osd.7 55 no new maps here, dropping 2019-12-07T19:34:32.653+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 185 ==== MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=70, reqid=client.4540.0:408, at_version=55'26, trim_to=0'0, roll_forward_to=54'24)) v2 ==== 3263+0+0 (crc 0 0 0) 0x558c575bf800 con 0x558c54c19800 2019-12-07T19:34:32.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x42000 2019-12-07T19:34:32.653+0000 7f8f24c58700 15 osd.7 55 enqueue_op 0x558c5740d340 prio 127 cost 0 latency 0.000032 epoch 55 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=70, reqid=client.4540.0:408, at_version=55'26, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:32.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:32.653+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=70, reqid=client.4540.0:408, at_version=55'26, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=70, reqid=client.4540.0:408, at_version=55'26, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=70, reqid=client.4540.0:408, at_version=55'26, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) pg 0x558c55141000 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 55 dequeue_op 0x558c5740d340 prio 127 cost 0 latency 0.000092 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=70, reqid=client.4540.0:408, at_version=55'26, trim_to=0'0, roll_forward_to=54'24)) v2 pg pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 55 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 54 -> 55 (as per caller) 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=70, reqid=client.4540.0:408, at_version=55'26, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:32.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000070 2019-12-07T19:34:32.654+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 snap_mapper.add_oid 1:bca18876:::smithi13140796-30:20 20 2019-12-07T19:34:32.654+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 snap_mapper.get_snaps 1:bca18876:::smithi13140796-30:20 got.empty() 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 snap_mapper.set_snaps 1:bca18876:::smithi13140796-30:20 20 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:32.654+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 66 2019-12-07T19:34:32.654+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x360000 mtime 2019-12-07T19:34:32.654739+0000 allocated 360000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~330000]) 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 54'24 (0'0,54'24] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,54'24], crt=54'24) [55'25 (54'24) clone 1:bca18876:::smithi13140796-30:20 by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,55'26 (54'24) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:408 2019-12-07T19:34:32.653937+0000 0 ObjectCleanRegions clean_offsets: [0~1540096,1540096~18446744073708011519], clean_omap: 1, new_object: 0] 2019-12-07T19:34:32.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000925515s (0.000005057s flush + 0.000920458s kv commit) 2019-12-07T19:34:32.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'25 (0'0,55'25] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=54'24 crt=54'24 lcod 54'24 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'25 (54'24) clone 1:bca18876:::smithi13140796-30:20 by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'26 (0'0,55'26] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=54'24 crt=54'24 lcod 54'24 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'26 (54'24) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:408 2019-12-07T19:34:32.653937+0000 0 ObjectCleanRegions clean_offsets: [0~1540096,1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'26 (0'0,55'26] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=54'24 crt=54'24 lcod 54'24 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 26 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'26 (0'0,55'26] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=54'24 crt=54'24 lcod 54'24 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.654+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'25, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58540900 seq 59 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bca18876:::smithi13140796-30:20# key 0x828000000000000001bca188'v!smithi13140796-30!='0x0000000000000020ffffffffffffffff'o' 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:32.654+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:20# 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1263 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.onode(0x558c54e13900).flush flush done 2019-12-07T19:34:32.654+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# 0x0 2019-12-07T19:34:32.654+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:20# 0x0~5e000 -> 0x0~5e000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x0~2000: 0x0~2000 Blob(0x558c575836c0 blob([0xae6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10719 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575836c0 blob([0xae6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575836c0 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3e30 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3e30) get_ref 0x0~2000 Blob(0x558c574e3e30 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3e30) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c574e3e30 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c575835e0 blob([0xae8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10720 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575835e0 blob([0xae8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575835e0 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a930 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a930) get_ref 0x0~2000 Blob(0x558c5898a930 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a930) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c5898a930 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57583260 blob([0xaea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10721 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57583260 blob([0xaea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57583260 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c585d5d50 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5d50) get_ref 0x0~2000 Blob(0x558c585d5d50 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5d50) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c585d5d50 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57538a80 blob([0xaec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10722 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57538a80 blob([0xaec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57538a80 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3d50 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3d50) get_ref 0x0~2000 Blob(0x558c574e3d50 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3d50) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c574e3d50 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c575389a0 blob([0xaee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10723 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575389a0 blob([0xaee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575389a0 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a850 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a850) get_ref 0x0~2000 Blob(0x558c5898a850 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a850) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5898a850 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c575388c0 blob([0xaf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10724 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575388c0 blob([0xaf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57956600> 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575388c0 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956600 kv_submitted 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57956600 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5809b810 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5809b810) get_ref 0x0~2000 Blob(0x558c5809b810 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5809b810) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956600 finishing 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57956600 onodes 0x558c57573900,0x558c57a35400 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5809b810 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57314018 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8f8c0 space 0x558c57314018 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c575387e0 blob([0xaf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10725 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adc18 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fa70 space 0x558c576adc18 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c575387e0 blob([0xaf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576adc88 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fcb0 space 0x558c576adc88 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c575387e0 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794e408 in 0x558c53a59a40) _finish_write discard buffer(0x558c585ea000 space 0x558c5794e408 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794e4e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585ea090 space 0x558c5794e4e8 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a8c0 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794e5c8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585ea120 space 0x558c5794e5c8 0x1000~1000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a8c0) get_ref 0x0~2000 Blob(0x558c5898a8c0 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970018 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fb00 space 0x558c57970018 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c5898a8c0 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970088 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fd40 space 0x558c57970088 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57538690 blob([0xaf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579700f8 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c17e60 space 0x558c579700f8 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10726 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2168 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fef0 space 0x558c579b2168 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57538690 blob([0xaf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2cc8 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fc20 space 0x558c579b2cc8 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2d38 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fb90 space 0x558c579b2d38 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57538690 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2da8 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fe60 space 0x558c579b2da8 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2e18 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8fdd0 space 0x558c579b2e18 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3c70 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 55/41 ECSubWriteReply(tid=269, last_complete=55'40, committed=1, applied=1)) v2 -- 0x558c57950200 con 0x558c54c64000 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2e88 in 0x558c53a59a40) _finish_write discard buffer(0x558c5813b560 space 0x558c579b2e88 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2ef8 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c17ef0 space 0x558c579b2ef8 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3c70) get_ref 0x0~2000 Blob(0x558c574e3c70 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f67898 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8f7a0 space 0x558c57f67898 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3c70) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f67908 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8f950 space 0x558c57f67908 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f67978 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8f9e0 space 0x558c57f67978 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c574e3c70 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f67a58 in 0x558c53a59a40) _finish_write discard buffer(0x558c57f779e0 space 0x558c57f67a58 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c546772d0 blob([0xaf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10727 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809ae88 in 0x558c53a59a40) _finish_write discard buffer(0x558c58c8f830 space 0x558c5809ae88 0x0~2000 writing nocache) 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c546772d0 blob([0xaf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c546772d0 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57956600 done 2019-12-07T19:34:32.654+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:32.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57956600 [] 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a7e0 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a7e0) get_ref 0x0~2000 Blob(0x558c5898a7e0 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5898a7e0 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c5809b420 blob([0xaf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10728 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c5809b420 blob([0xaf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c5809b420 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3b90 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3b90) get_ref 0x0~2000 Blob(0x558c574e3b90 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3b90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c574e3b90 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57c42460 blob([0xafa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10729 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57c42460 blob([0xafa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57c42460 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a770 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a770) get_ref 0x0~2000 Blob(0x558c5898a770 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a770) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c5898a770 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c5794c690 blob([0xafc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10730 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c5794c690 blob([0xafc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c5794c690 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3ab0 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3ab0) get_ref 0x0~2000 Blob(0x558c574e3ab0 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c574e3ab0 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c580ec380 blob([0xafe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10731 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c580ec380 blob([0xafe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c580ec380 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a700) get_ref 0x0~2000 Blob(0x558c5898a700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a700) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c5898a700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57bea1c0 blob([0xb00000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10732 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57bea1c0 blob([0xb00000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57bea1c0 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e39d0 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e39d0) get_ref 0x0~2000 Blob(0x558c574e39d0 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c574e39d0 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c55060230 blob([0xb02000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10733 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c55060230 blob([0xb02000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c55060230 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a690 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a690) get_ref 0x0~2000 Blob(0x558c5898a690 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a690) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c5898a690 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c574d3ab0 blob([0xb04000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10734 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574d3ab0 blob([0xb04000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574d3ab0 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e38f0 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e38f0) get_ref 0x0~2000 Blob(0x558c574e38f0 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e38f0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c574e38f0 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x20000~2000: 0x0~2000 Blob(0x558c57db55e0 blob([0xb06000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10735 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57db55e0 blob([0xb06000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57db55e0 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a620 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a620) get_ref 0x0~2000 Blob(0x558c5898a620 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a620) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c5898a620 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x22000~2000: 0x0~2000 Blob(0x558c57db5340 blob([0xb08000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10736 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57db5340 blob([0xb08000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57db5340 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3810 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3810) get_ref 0x0~2000 Blob(0x558c574e3810 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3810) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c574e3810 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x24000~2000: 0x0~2000 Blob(0x558c574d2310 blob([0xb0a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10737 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c574d2310 blob([0xb0a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c574d2310 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a5b0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a5b0) get_ref 0x0~2000 Blob(0x558c5898a5b0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c5898a5b0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x26000~2000: 0x0~2000 Blob(0x558c57881260 blob([0xb0c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10738 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57881260 blob([0xb0c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57881260 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3730 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3730) get_ref 0x0~2000 Blob(0x558c574e3730 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3730) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c574e3730 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x28000~2000: 0x0~2000 Blob(0x558c57efc0e0 blob([0xb0e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10739 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57efc0e0 blob([0xb0e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57efc0e0 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a540 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a540) get_ref 0x0~2000 Blob(0x558c5898a540 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a540) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c5898a540 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c57a07960 blob([0xb10000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10740 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57a07960 blob([0xb10000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57a07960 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e33b0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e33b0) get_ref 0x0~2000 Blob(0x558c574e33b0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e33b0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c574e33b0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c57ff4150 blob([0xb12000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10741 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c57ff4150 blob([0xb12000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c57ff4150 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a4d0 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a4d0) get_ref 0x0~2000 Blob(0x558c5898a4d0 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c5898a4d0 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c54df8930 blob([0xb14000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10742 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c54df8930 blob([0xb14000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c54df8930 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3500 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3500) get_ref 0x0~2000 Blob(0x558c574e3500 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3500) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c574e3500 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x30000~2000: 0x0~2000 Blob(0x558c580a5500 blob([0xb16000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10743 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c580a5500 blob([0xb16000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c580a5500 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a460 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a460) get_ref 0x0~2000 Blob(0x558c5898a460 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a460) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c5898a460 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x32000~2000: 0x0~2000 Blob(0x558c58c67c70 blob([0xb18000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10744 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c67c70 blob([0xb18000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c67c70 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e35e0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e35e0) get_ref 0x0~2000 Blob(0x558c574e35e0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e35e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c574e35e0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x34000~2000: 0x0~2000 Blob(0x558c58c67d50 blob([0xb1a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10745 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c67d50 blob([0xb1a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c67d50 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c578de000 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c578de000) get_ref 0x0~2000 Blob(0x558c578de000 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c578de000) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c578de000 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x36000~2000: 0x0~2000 Blob(0x558c58c67e30 blob([0xb1c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10746 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c67e30 blob([0xb1c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c67e30 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e36c0 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e36c0) get_ref 0x0~2000 Blob(0x558c574e36c0 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e36c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c574e36c0 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x38000~2000: 0x0~2000 Blob(0x558c58c67f10 blob([0xb1e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10747 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c67f10 blob([0xb1e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c67f10 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5809acb0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5809acb0) get_ref 0x0~2000 Blob(0x558c5809acb0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5809acb0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c5809acb0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c58c78000 blob([0xb20000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10748 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78000 blob([0xb20000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78000 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c574e3340 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3340) get_ref 0x0~2000 Blob(0x558c574e3340 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3340) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c574e3340 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c58c780e0 blob([0xb22000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10749 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c780e0 blob([0xb22000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c780e0 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c57583b20 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c57583b20) get_ref 0x0~2000 Blob(0x558c57583b20 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c57583b20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c57583b20 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c58c781c0 blob([0xb24000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10750 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c781c0 blob([0xb24000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c781c0 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898a9a0 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a9a0) get_ref 0x0~2000 Blob(0x558c5898a9a0 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c5898a9a0 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x40000~2000: 0x0~2000 Blob(0x558c58c782a0 blob([0xb26000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10751 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c782a0 blob([0xb26000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c782a0 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898aa10 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aa10) get_ref 0x0~2000 Blob(0x558c5898aa10 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aa10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c5898aa10 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x42000~2000: 0x0~2000 Blob(0x558c58c78380 blob([0xb28000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10752 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78380 blob([0xb28000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78380 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898aa80 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aa80) get_ref 0x0~2000 Blob(0x558c5898aa80 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aa80) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c5898aa80 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x44000~2000: 0x0~2000 Blob(0x558c58c78460 blob([0xb2a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10753 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78460 blob([0xb2a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78460 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898aaf0 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aaf0) get_ref 0x0~2000 Blob(0x558c5898aaf0 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c5898aaf0 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x46000~2000: 0x0~2000 Blob(0x558c58c78540 blob([0xb2c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10754 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78540 blob([0xb2c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78540 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898ab60 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ab60) get_ref 0x0~2000 Blob(0x558c5898ab60 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ab60) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c5898ab60 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x48000~2000: 0x0~2000 Blob(0x558c58c78620 blob([0xb2e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10755 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78620 blob([0xb2e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78620 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898abd0 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898abd0) get_ref 0x0~2000 Blob(0x558c5898abd0 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898abd0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c5898abd0 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c58c78700 blob([0xb30000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10756 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78700 blob([0xb30000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78700 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898ac40 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ac40) get_ref 0x0~2000 Blob(0x558c5898ac40 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ac40) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c5898ac40 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c58c787e0 blob([0xb32000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10757 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c787e0 blob([0xb32000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c787e0 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898acb0 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898acb0) get_ref 0x0~2000 Blob(0x558c5898acb0 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898acb0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c5898acb0 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c58c788c0 blob([0xb34000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10758 2019-12-07T19:34:32.654+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c788c0 blob([0xb34000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 sbid 0x0)) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c788c0 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=1)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898ad20 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2)))) 2019-12-07T19:34:32.654+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ad20) get_ref 0x0~2000 Blob(0x558c5898ad20 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ad20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c5898ad20 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x50000~2000: 0x0~2000 Blob(0x558c58c789a0 blob([0xb36000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10759 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c789a0 blob([0xb36000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c789a0 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=1)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c5898ad90 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ad90) get_ref 0x0~2000 Blob(0x558c5898ad90 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ad90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c5898ad90 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x52000~2000: 0x0~2000 Blob(0x558c58c78a80 blob([0xb38000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10760 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78a80 blob([0xb38000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78a80 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=1)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c579d6000 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6000) get_ref 0x0~2000 Blob(0x558c579d6000 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6000) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c579d6000 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x54000~2000: 0x0~2000 Blob(0x558c58c78b60 blob([0xb3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10761 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78b60 blob([0xb3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78b60 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=1)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c579d6070 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6070) get_ref 0x0~2000 Blob(0x558c579d6070 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6070) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c579d6070 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x56000~2000: 0x0~2000 Blob(0x558c58c78c40 blob([0xb3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10762 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78c40 blob([0xb3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78c40 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=1)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c579d60e0 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d60e0) get_ref 0x0~2000 Blob(0x558c579d60e0 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d60e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c579d60e0 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x58000~2000: 0x0~2000 Blob(0x558c58c78d20 blob([0xb3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10763 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78d20 blob([0xb3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78d20 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=1)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c579d6150 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6150) get_ref 0x0~2000 Blob(0x558c579d6150 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6150) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c579d6150 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c58c78e00 blob([0xb40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10764 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78e00 blob([0xb40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78e00 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=1)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c579d61c0 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d61c0) get_ref 0x0~2000 Blob(0x558c579d61c0 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d61c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c579d61c0 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c58c78ee0 blob([0xb42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10765 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared Blob(0x558c58c78ee0 blob([0xb42000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 sbid 0x0)) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) make_blob_shared now Blob(0x558c58c78ee0 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=1)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup new Blob(0x558c579d6230 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6230) get_ref 0x0~2000 Blob(0x558c579d6230 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6230) get_ref init 0x2000, 1000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c579d6230 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2)))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dirty_range mark inline shard dirty 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dirty_range mark inline shard dirty 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:20# = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# snapset 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# snapset = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# 1 keys 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# 1 keys = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# hinfo_key (42 bytes) 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.D35811E6.20.smithi13140796-30..' <- OBJ_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.2_0000000000000001.D35811E6.20.smithi13140796-30..' <- SNA_1_0000000000000020_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000055.00000000000000000025' <- 0000000055.00000000000000000025 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000055.00000000000000000026' <- 0000000055.00000000000000000026 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58540900 cost 8310 (1 ios * 4000 + 4310 bytes) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58540900 onodes 0x558c54e12c80,0x558c54e13900 shared_blobs 0x558c54cf8230,0x558c550604d0,0x558c574d2cb0,0x558c574fd9d0,0x558c578f00e0,0x558c578f0150,0x558c578f1110,0x558c578f1180,0x558c578f1340,0x558c5794d9d0,0x558c57a06bd0,0x558c57a07f80,0x558c57bea540,0x558c57beac40,0x558c57db5500,0x558c57db5570,0x558c57db5960,0x558c57e63d50,0x558c57f512d0,0x558c57f51340,0x558c57f51810,0x558c57ff4310,0x558c5809b180,0x558c5809b3b0,0x558c58c67c00,0x558c58c67ce0,0x558c58c67dc0,0x558c58c67ea0,0x558c58c67f80,0x558c58c78070,0x558c58c78150,0x558c58c78230,0x558c58c78310,0x558c58c783f0,0x558c58c784d0,0x558c58c785b0,0x558c58c78690,0x558c58c78770,0x558c58c78850,0x558c58c78930,0x558c58c78a10,0x558c58c78af0,0x558c58c78bd0,0x558c58c78cb0,0x558c58c78d90,0x558c58c78e70,0x558c58c78f50 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) update 2#1:bca18876:::smithi13140796-30:20# 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) update inline shard 1272 bytes from 47 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore.extentmap(0x558c54e12d88) reshard 0x[0,ffffffff) of 0 shards on 2#1:bca18876:::smithi13140796-30:20# 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard new shard 0x0 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard new shard 0x24000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard new shard 0x48000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard old [] 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) update 2#1:bca18876:::smithi13140796-30:20# force 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) update shard 0x48000 is 302 bytes (was 0) from 11 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:20# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update inline shard 1272 bytes from 47 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore.extentmap(0x558c54e13a08) reshard 0x[0,ffffffff) of 0 shards on 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard new shard 0x0 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard new shard 0x24000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard new shard 0x48000 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard old [] 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update 2#1:bca18876:::smithi13140796-30:head# force 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update shard 0x48000 is 302 bytes (was 0) from 11 extents 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 636 (634 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e8 is 11 SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ea is 11 SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f2 is 11 SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e6 is 11 SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e2 is 11 SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e1 is 11 SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e0 is 11 SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e4 is 11 SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e7 is 11 SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ed is 11 SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ee is 11 SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f3 is 11 SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f0 is 11 SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f5 is 11 SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f1 is 11 SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ef is 11 SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29eb is 11 SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e5 is 11 SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e3 is 11 SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29df is 11 SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e9 is 11 SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ec is 11 SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f6 is 11 SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f4 is 11 SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f7 is 11 SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f8 is 11 SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f9 is 11 SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fa is 11 SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fb is 11 SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fc is 11 SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fd is 11 SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fe is 11 SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ff is 11 SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a00 is 11 SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a01 is 11 SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a02 is 11 SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a03 is 11 SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a04 is 11 SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a05 is 11 SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a06 is 11 SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a07 is 11 SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a08 is 11 SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a09 is 11 SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0a is 11 SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0b is 11 SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0c is 11 SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0d is 11 SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2))) 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58540900 allocated 0x[] released 0x[] 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540900 prepare 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58540900 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540900 io_done 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'26 (0'0,55'26] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=54'24 crt=54'24 lcod 54'24 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'26 2019-12-07T19:34:32.655+0000 7f8f0217d700 10 osd.7 55 dequeue_op 0x558c5740d340 finish 2019-12-07T19:34:32.655+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.655+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.655+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1286 < min_flush_size 524288 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7741 < min_flush_size 524288 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12c80 had 1 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13900 had 1 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7764 < min_flush_size 524288 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7764 < min_flush_size 524288 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x360000 mtime 2019-12-07T19:34:32.654739+0000 allocated 360000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~330000]) 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x35fafa~1e54 to file(ino 16 size 0x360000 mtime 2019-12-07T19:34:32.654739+0000 allocated 360000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~330000]) 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x35fafa 0x35fafa~1e54 to file(ino 16 size 0x360000 mtime 2019-12-07T19:34:32.654739+0000 allocated 360000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~330000]) 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _allocate len 0x194e from 0 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _allocate len 0x194e from 1 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd490000 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd490000~10000/10000,10000,acd490000 2019-12-07T19:34:32.655+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x370000 to include allocated 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 67 (was clean) 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x370000 mtime 2019-12-07T19:34:32.656823+0000 allocated 370000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~340000]) 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~340000 x_off 0x32fafa 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xafa 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x94e and padding block with 0x6b2 2019-12-07T19:34:32.655+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x370000 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x36194e 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (67) on file(ino 16 size 0x370000 mtime 2019-12-07T19:34:32.656823+0000 allocated 370000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~340000]), flushing log 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x370000 mtime 2019-12-07T19:34:32.656823+0000 allocated 370000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~340000]) 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 67 len 0x4a crc 0xfc09afa5) 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x42000~1000 to file(ino 1 size 0x42000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x42000 0x42000~1000 to file(ino 1 size 0x42000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x43000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x42000 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x43000 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:32.656+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000020 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 67 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x370000 mtime 2019-12-07T19:34:32.656823+0000 allocated 370000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~340000]) 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000723391s (0.000000121s flush + 0.000723270s kv commit) 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58540900> 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540900 kv_submitted 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58540900 2019-12-07T19:34:32.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540900 finishing 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58540900 onodes 0x558c54e12c80,0x558c54e13900 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540900 done 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:32.656+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58540900 [] 2019-12-07T19:34:32.656+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.656+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.656+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 55/41 ECSubWriteReply(tid=70, last_complete=55'26, committed=1, applied=1)) v2 -- 0x558c580b9e00 con 0x558c54c19800 2019-12-07T19:34:32.656+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.656+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 186 ==== MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=71, reqid=client.4540.0:409, at_version=55'27, trim_to=0'0, roll_forward_to=54'24)) v2 ==== 2180+0+0 (crc 0 0 0) 0x558c54693000 con 0x558c54c19800 2019-12-07T19:34:32.656+0000 7f8f24c58700 15 osd.7 55 enqueue_op 0x558c57efe2c0 prio 127 cost 0 latency 0.000021 epoch 55 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=71, reqid=client.4540.0:409, at_version=55'27, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:32.656+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=71, reqid=client.4540.0:409, at_version=55'27, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=71, reqid=client.4540.0:409, at_version=55'27, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=71, reqid=client.4540.0:409, at_version=55'27, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) pg 0x558c55141000 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 osd.7 55 dequeue_op 0x558c57efe2c0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=71, reqid=client.4540.0:409, at_version=55'27, trim_to=0'0, roll_forward_to=54'24)) v2 pg pg[1.ds2( v 55'26 (0'0,55'26] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'26 (0'0,55'26] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=71, reqid=client.4540.0:409, at_version=55'27, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'26 (0'0,55'26] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'26], crt=54'24) [55'27 (55'26) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:409 2019-12-07T19:34:32.653968+0000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 0, new_object: 0] 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'27 (0'0,55'27] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'26 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'27 (55'26) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:409 2019-12-07T19:34:32.653968+0000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 0, new_object: 0 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'27 (0'0,55'27] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'26 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 27 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'27 (0'0,55'27] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'26 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.656+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'27, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58540c00 seq 60 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bca18876:::smithi13140796-30:head#1b key 0x828000000000000001bca188'v!smithi13140796-30!='0xfffffffffffffffe000000000000001b'o' 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:32.656+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:head#1b 2019-12-07T19:34:32.656+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# -> 2#1:bca18876:::smithi13140796-30:head#1b = 0 2019-12-07T19:34:32.656+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# -> 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1264 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.onode(0x558c54e12c80).flush flush done 2019-12-07T19:34:32.656+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 0x0 2019-12-07T19:34:32.656+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# -> 2#1:bca18876:::smithi13140796-30:head# 0x0~5e000 -> 0x0~5e000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x0~2000: 0x0~2000 Blob(0x558c574e3e30 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c585d5f10 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5f10) get_ref 0x0~2000 Blob(0x558c585d5f10 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5f10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c585d5f10 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c5898a930 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c585d5c00 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5c00) get_ref 0x0~2000 Blob(0x558c585d5c00 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5c00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c585d5c00 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c585d5d50 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c585d5c70 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5c70) get_ref 0x0~2000 Blob(0x558c585d5c70 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5c70) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c585d5c70 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c574e3d50 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6310 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6310) get_ref 0x0~2000 Blob(0x558c579d6310 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6310) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c579d6310 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c5898a850 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6380 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6380) get_ref 0x0~2000 Blob(0x558c579d6380 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6380) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c579d6380 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c5809b810 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d63f0 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d63f0) get_ref 0x0~2000 Blob(0x558c579d63f0 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d63f0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c579d63f0 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c5898a8c0 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6460 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6460) get_ref 0x0~2000 Blob(0x558c579d6460 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6460) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c579d6460 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c574e3c70 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d64d0 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d64d0) get_ref 0x0~2000 Blob(0x558c579d64d0 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d64d0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c579d64d0 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c5898a7e0 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6540 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6540) get_ref 0x0~2000 Blob(0x558c579d6540 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6540) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c579d6540 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x12000~2000: 0x0~2000 Blob(0x558c574e3b90 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d65b0 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d65b0) get_ref 0x0~2000 Blob(0x558c579d65b0 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d65b0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c579d65b0 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x14000~2000: 0x0~2000 Blob(0x558c5898a770 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6620 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6620) get_ref 0x0~2000 Blob(0x558c579d6620 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6620) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c579d6620 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x16000~2000: 0x0~2000 Blob(0x558c574e3ab0 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6690 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6690) get_ref 0x0~2000 Blob(0x558c579d6690 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6690) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c579d6690 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x18000~2000: 0x0~2000 Blob(0x558c5898a700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6700) get_ref 0x0~2000 Blob(0x558c579d6700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6700) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c579d6700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c574e39d0 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6770 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6770) get_ref 0x0~2000 Blob(0x558c579d6770 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6770) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c579d6770 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c5898a690 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d67e0 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d67e0) get_ref 0x0~2000 Blob(0x558c579d67e0 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d67e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c579d67e0 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c574e38f0 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6850 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6850) get_ref 0x0~2000 Blob(0x558c579d6850 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6850) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c579d6850 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x20000~2000: 0x0~2000 Blob(0x558c5898a620 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d68c0 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d68c0) get_ref 0x0~2000 Blob(0x558c579d68c0 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d68c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c579d68c0 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x22000~2000: 0x0~2000 Blob(0x558c574e3810 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6930 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 265 ==== osd_map(55..55 src has 1..55) v4 ==== 633+0+0 (crc 0 0 0) 0x558c579d8000 con 0x558c54c5bc00 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6930) get_ref 0x0~2000 Blob(0x558c579d6930 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f1199c700 20 osd.7 55 OSD::ms_dispatch: osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6930) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f1199c700 10 osd.7 55 do_waiters -- start 2019-12-07T19:34:32.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 266 ==== MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=148, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c585cfb80 con 0x558c54c5bc00 2019-12-07T19:34:32.656+0000 7f8f1199c700 10 osd.7 55 do_waiters -- finish 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c579d6930 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f1199c700 20 osd.7 55 _dispatch 0x558c579d8000 osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x24000~2000: 0x0~2000 Blob(0x558c5898a5b0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f23c56700 15 osd.7 55 enqueue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000016 epoch 55 MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=148, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d69a0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=148, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) 2019-12-07T19:34:32.656+0000 7f8f1199c700 3 osd.7 55 handle_osd_map epochs [55,55], i have 55, src has [1,55] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d69a0) get_ref 0x0~2000 Blob(0x558c579d69a0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f1199c700 10 osd.7 55 no new maps here, dropping 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d69a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c579d69a0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x26000~2000: 0x0~2000 Blob(0x558c574e3730 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6a10 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6a10) get_ref 0x0~2000 Blob(0x558c579d6a10 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6a10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c579d6a10 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=148, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x28000~2000: 0x0~2000 Blob(0x558c5898a540 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6a80 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=148, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) pg 0x558c54d5c000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6a80) get_ref 0x0~2000 Blob(0x558c579d6a80 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6a80) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c579d6a80 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c574e33b0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6af0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6af0) get_ref 0x0~2000 Blob(0x558c579d6af0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6af0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0197c700 10 osd.7 55 dequeue_op 0x558c578d2dc0 prio 127 cost 0 latency 0.000049 MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=148, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c579d6af0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 10 osd.7 55 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 54 -> 55 (as per caller) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c5898a4d0 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6b60 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6b60) get_ref 0x0~2000 Blob(0x558c579d6b60 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6b60) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c579d6b60 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=148, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c574e3500 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6bd0 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6bd0) get_ref 0x0~2000 Blob(0x558c579d6bd0 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0197c700 25 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c579d6bd0 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x30000~2000: 0x0~2000 Blob(0x558c5898a460 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6c40 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~29000 size 0x29000 (167936) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6c40) get_ref 0x0~2000 Blob(0x558c579d6c40 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6c40) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c579d6c40 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x32000~2000: 0x0~2000 Blob(0x558c574e35e0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6cb0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6cb0) get_ref 0x0~2000 Blob(0x558c579d6cb0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c579d6cb0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x34000~2000: 0x0~2000 Blob(0x558c578de000 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6d20 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e4d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6d20) get_ref 0x0~2000 Blob(0x558c579d6d20 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=3)))) 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6d20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.656+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e5b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.656+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c579d6d20 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x36000~2000: 0x0~2000 Blob(0x558c574e36c0 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d6d90 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6d90) get_ref 0x0~2000 Blob(0x558c579d6d90 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6d90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c579d6d90 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0cb0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x38000~2000: 0x0~2000 Blob(0x558c5809acb0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0bd0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7ab0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0af0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7ab0) get_ref 0x0~2000 Blob(0x558c579d7ab0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0a10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c579d7ab0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c574e3340 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f07e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7b20 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7b20) get_ref 0x0~2000 Blob(0x558c579d7b20 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7b20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c579d7b20 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5742a1c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c57583b20 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7b90 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7b90) get_ref 0x0~2000 Blob(0x558c579d7b90 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 = 167936 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7b90) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c579d7b90 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c5898a9a0 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read read request=167936 r=167936 len=167936 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7c00 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7c00) get_ref 0x0~2000 Blob(0x558c579d7c00 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7c00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: Getting attr on 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c579d7c00 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x40000~2000: 0x0~2000 Blob(0x558c5898aa10 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: not in cache 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.657+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7c70 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7c70) get_ref 0x0~2000 Blob(0x558c579d7c70 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7c70) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: found on disk, size 167936 2019-12-07T19:34:32.657+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c579d7c70 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key = 0 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x42000~2000: 0x0~2000 Blob(0x558c5898aa80 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7ce0 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read: Checking hash of 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7ce0) get_ref 0x0~2000 Blob(0x558c579d7ce0 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c579d7ce0 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 55/28 ECSubReadReply(tid=148, attrs_read=0)) v2 -- 0x558c57c05b80 con 0x558c54c5bc00 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x44000~2000: 0x0~2000 Blob(0x558c5898aaf0 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7d50 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7d50) get_ref 0x0~2000 Blob(0x558c579d7d50 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7d50) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c579d7d50 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 10 osd.7 55 dequeue_op 0x558c578d2dc0 finish 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x46000~2000: 0x0~2000 Blob(0x558c5898ab60 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7dc0 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7dc0) get_ref 0x0~2000 Blob(0x558c579d7dc0 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c579d7dc0 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x48000~2000: 0x0~2000 Blob(0x558c5898abd0 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7e30 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7e30) get_ref 0x0~2000 Blob(0x558c579d7e30 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7e30) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c579d7e30 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c5898ac40 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7ea0 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7ea0) get_ref 0x0~2000 Blob(0x558c579d7ea0 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c579d7ea0 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c5898acb0 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7f10 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7f10) get_ref 0x0~2000 Blob(0x558c579d7f10 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7f10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c579d7f10 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c5898ad20 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c579d7f80 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7f80) get_ref 0x0~2000 Blob(0x558c579d7f80 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c579d7f80) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c579d7f80 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x50000~2000: 0x0~2000 Blob(0x558c5898ad90 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c5895a000 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a000) get_ref 0x0~2000 Blob(0x558c5895a000 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a000) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c5895a000 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x52000~2000: 0x0~2000 Blob(0x558c579d6000 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c5895a070 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a070) get_ref 0x0~2000 Blob(0x558c5895a070 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a070) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c5895a070 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x54000~2000: 0x0~2000 Blob(0x558c579d6070 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c5895a0e0 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a0e0) get_ref 0x0~2000 Blob(0x558c5895a0e0 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c5895a0e0 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x56000~2000: 0x0~2000 Blob(0x558c579d60e0 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c5895a150 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a150) get_ref 0x0~2000 Blob(0x558c5895a150 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a150) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c5895a150 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x58000~2000: 0x0~2000 Blob(0x558c579d6150 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c5895a1c0 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a1c0) get_ref 0x0~2000 Blob(0x558c5895a1c0 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c5895a1c0 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c579d61c0 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c5895a230 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a230) get_ref 0x0~2000 Blob(0x558c5895a230 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a230) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c5895a230 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c579d6230 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup new Blob(0x558c5895a2a0 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a2a0) get_ref 0x0~2000 Blob(0x558c5895a2a0 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.blob(0x558c5895a2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c5895a2a0 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=3)))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) dirty_range mark inline shard dirty 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# -> 2#1:bca18876:::smithi13140796-30:head# = 0 2019-12-07T19:34:32.657+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:32.657+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.657+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000055.00000000000000000027' <- 0000000055.00000000000000000027 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58540c00 cost 5830 (1 ios * 4000 + 1830 bytes) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58540c00 onodes 0x558c54e13900,0x558c585cf180 shared_blobs 0x558c54cf8230,0x558c550604d0,0x558c574d2cb0,0x558c574fd9d0,0x558c578f00e0,0x558c578f0150,0x558c578f1110,0x558c578f1180,0x558c578f1340,0x558c5794d9d0,0x558c57a06bd0,0x558c57a07f80,0x558c57bea540,0x558c57beac40,0x558c57db5500,0x558c57db5570,0x558c57db5960,0x558c57e63d50,0x558c57f512d0,0x558c57f51340,0x558c57f51810,0x558c57ff4310,0x558c5809b180,0x558c5809b3b0,0x558c58c67c00,0x558c58c67ce0,0x558c58c67dc0,0x558c58c67ea0,0x558c58c67f80,0x558c58c78070,0x558c58c78150,0x558c58c78230,0x558c58c78310,0x558c58c783f0,0x558c58c784d0,0x558c58c785b0,0x558c58c78690,0x558c58c78770,0x558c58c78850,0x558c58c78930,0x558c58c78a10,0x558c58c78af0,0x558c58c78bd0,0x558c58c78cb0,0x558c58c78d90,0x558c58c78e70,0x558c58c78f50 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update 2#1:bca18876:::smithi13140796-30:head#1b 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update shard 0x0 is 489 bytes (was 489) from 18 extents 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update shard 0x24000 is 491 bytes (was 491) from 18 extents 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) update shard 0x48000 is 302 bytes (was 302) from 11 extents 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head#1b is 636 (634 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) update inline shard 1272 bytes from 47 extents 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore.extentmap(0x558c585cf288) reshard 0x[0,ffffffff) of 0 shards on 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard new shard 0x0 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard new shard 0x24000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard new shard 0x48000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard old [] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) update 2#1:bca18876:::smithi13140796-30:head# force 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) update shard 0x48000 is 302 bytes (was 0) from 11 extents 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 636 (634 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e8 is 11 SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ea is 11 SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f2 is 11 SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e6 is 11 SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e2 is 11 SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e1 is 11 SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e0 is 11 SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e4 is 11 SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e7 is 11 SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ed is 11 SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ee is 11 SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f3 is 11 SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f0 is 11 SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f5 is 11 SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f1 is 11 SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ef is 11 SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29eb is 11 SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e5 is 11 SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e3 is 11 SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29df is 11 SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e9 is 11 SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ec is 11 SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f6 is 11 SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f4 is 11 SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f7 is 11 SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f8 is 11 SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f9 is 11 SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fa is 11 SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fb is 11 SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fc is 11 SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fd is 11 SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fe is 11 SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ff is 11 SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 458 ==== MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=270, reqid=client.4540.0:405, at_version=55'41, trim_to=0'0, roll_forward_to=53'38)) v2 ==== 195011+0+0 (crc 0 0 0) 0x558c579e1000 con 0x558c54c64000 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a00 is 11 SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a01 is 11 SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a02 is 11 SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f24457700 15 osd.7 55 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000049 epoch 55 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=270, reqid=client.4540.0:405, at_version=55'41, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a03 is 11 SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a04 is 11 SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=270, reqid=client.4540.0:405, at_version=55'41, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a05 is 11 SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a06 is 11 SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a07 is 11 SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a08 is 11 SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a09 is 11 SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0a is 11 SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0b is 11 SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0c is 11 SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0d is 11 SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=3))) 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58540c00 allocated 0x[] released 0x[] 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 prepare 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58540c00 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 io_done 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'27 (0'0,55'27] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'26 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'27 2019-12-07T19:34:32.657+0000 7f8f0217d700 10 osd.7 55 dequeue_op 0x558c57efe2c0 finish 2019-12-07T19:34:32.657+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=270, reqid=client.4540.0:405, at_version=55'41, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=270, reqid=client.4540.0:405, at_version=55'41, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) pg 0x558c5500a000 2019-12-07T19:34:32.657+0000 7f8f08189700 10 osd.7 55 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000142 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=270, reqid=client.4540.0:405, at_version=55'41, trim_to=0'0, roll_forward_to=53'38)) v2 pg pg[1.9s4( v 55'40 (0'0,55'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.657+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'40 (0'0,55'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=270, reqid=client.4540.0:405, at_version=55'41, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:32.657+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'40 (0'0,55'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'40], crt=53'38) [55'41 (55'40) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:405 2019-12-07T19:34:32.653855+0000 0 ObjectCleanRegions clean_offsets: [0~1933312,2703360~18446744073706848255], clean_omap: 1, new_object: 0] 2019-12-07T19:34:32.657+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'41 (0'0,55'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'40 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'41 (55'40) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:405 2019-12-07T19:34:32.653855+0000 0 ObjectCleanRegions clean_offsets: [0~1933312,2703360~18446744073706848255], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.657+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'41 (0'0,55'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'40 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 41 2019-12-07T19:34:32.657+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'41 (0'0,55'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'40 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.657+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.657+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'41, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c58113800 seq 79 2019-12-07T19:34:32.657+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:32.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:32.657+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:32.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:32.657+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x76000~2f000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:9b8adabe:::smithi13140796-6:head# 0x76000~2f000 - have 0x76000 (483328) bytes fadvise_flags 0x20 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:32.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x76000~2f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58113800 24 blobs 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.657+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2f000/1000,2f000,0 2019-12-07T19:34:32.657+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x38d000~2d000/1000,2f000,0 2019-12-07T19:34:32.657+0000 7f8f08189700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb4a000~2000/1000,2f000,0 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x38d000~2d000,0xb4a000~2000] 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583eab60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583eab60 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970f50 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c583eab60) get_ref 0x0~2000 Blob(0x558c583eab60 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970f50 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c583eab60) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x76000~2000: 0x0~2000 Blob(0x558c583eab60 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970f50 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57970f68 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f71f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0af0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f71f0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c578f0af0) get_ref 0x0~2000 Blob(0x558c578f0af0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f71f0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c578f0af0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x78000~2000: 0x0~2000 Blob(0x558c578f0af0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f71f0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7208 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574fa4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574fa4d0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c574fa4d0) get_ref 0x0~2000 Blob(0x558c574fa4d0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c574fa4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x7a000~2000: 0x0~2000 Blob(0x558c574fa4d0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c578df668 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d030 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7110 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c5794d030) get_ref 0x0~2000 Blob(0x558c5794d030 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7110 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c5794d030) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x7c000~2000: 0x0~2000 Blob(0x558c5794d030 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7110 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7128 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f73b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7340 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f73b0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7340) get_ref 0x0~2000 Blob(0x558c575f7340 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f73b0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7340) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x7e000~2000: 0x0~2000 Blob(0x558c575f7340 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f73b0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f73c8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7420 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7490 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7420) get_ref 0x0~2000 Blob(0x558c575f7420 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7490 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7420) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x80000~2000: 0x0~2000 Blob(0x558c575f7420 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7490 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f74a8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7500 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7570 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7500) get_ref 0x0~2000 Blob(0x558c575f7500 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7570 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7500) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x82000~2000: 0x0~2000 Blob(0x558c575f7500 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7570 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7588 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f75e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f75e0 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7650 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f75e0) get_ref 0x0~2000 Blob(0x558c575f75e0 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7650 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f75e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x84000~2000: 0x0~2000 Blob(0x558c575f75e0 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7650 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7668 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f76c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f76c0 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7730 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f76c0) get_ref 0x0~2000 Blob(0x558c575f76c0 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7730 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f76c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x86000~2000: 0x0~2000 Blob(0x558c575f76c0 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7730 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7748 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f77a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f77a0 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7810 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f77a0) get_ref 0x0~2000 Blob(0x558c575f77a0 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7810 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f77a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x88000~2000: 0x0~2000 Blob(0x558c575f77a0 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7810 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7828 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f78f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7880 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f78f0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7880) get_ref 0x0~2000 Blob(0x558c575f7880 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f78f0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7880) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8a000~2000: 0x0~2000 Blob(0x558c575f7880 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f78f0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7908 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f79d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7960 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f79d0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7960) get_ref 0x0~2000 Blob(0x558c575f7960 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f79d0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.blob(0x558c575f7960) get_ref init 0x2000, 1000 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8c000~2000: 0x0~2000 Blob(0x558c575f7960 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f79d0 sbid 0x0)) 2019-12-07T19:34:32.657+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f79e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7a40 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ab0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c575f7a40) get_ref 0x0~2000 Blob(0x558c575f7a40 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ab0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c575f7a40) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8e000~2000: 0x0~2000 Blob(0x558c575f7a40 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7ab0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7ac8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c575f7b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c575f7b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c575f7b20 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7b90 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c575f7b20) get_ref 0x0~2000 Blob(0x558c575f7b20 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7b90 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c575f7b20) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x90000~2000: 0x0~2000 Blob(0x558c575f7b20 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7b90 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c575f7ba8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895ae00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895ae70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895ae00 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895ae70 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895ae00) get_ref 0x0~2000 Blob(0x558c5895ae00 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895ae70 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895ae00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x92000~2000: 0x0~2000 Blob(0x558c5895ae00 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895ae70 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895ae88 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895aee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895af50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895aee0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895af50 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895aee0) get_ref 0x0~2000 Blob(0x558c5895aee0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895af50 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895aee0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x94000~2000: 0x0~2000 Blob(0x558c5895aee0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895af50 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895af68 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895afc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895afc0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b030 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895afc0) get_ref 0x0~2000 Blob(0x558c5895afc0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b030 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895afc0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x96000~2000: 0x0~2000 Blob(0x558c5895afc0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b030 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b048 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895b0a0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b110 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b0a0) get_ref 0x0~2000 Blob(0x558c5895b0a0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b110 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x98000~2000: 0x0~2000 Blob(0x558c5895b0a0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b110 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b128 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895b180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895b180 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b1f0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b180) get_ref 0x0~2000 Blob(0x558c5895b180 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b1f0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b180) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x9a000~2000: 0x0~2000 Blob(0x558c5895b180 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b1f0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b208 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895b260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895b260 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b2d0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b260) get_ref 0x0~2000 Blob(0x558c5895b260 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b2d0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b260) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x9c000~2000: 0x0~2000 Blob(0x558c5895b260 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b2d0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b2e8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895b340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895b340 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b3b0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b340) get_ref 0x0~2000 Blob(0x558c5895b340 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b3b0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b340) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x9e000~2000: 0x0~2000 Blob(0x558c5895b340 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b3b0 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b3c8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895b420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895b420 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b490 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b420) get_ref 0x0~2000 Blob(0x558c5895b420 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b490 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b420) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa0000~2000: 0x0~2000 Blob(0x558c5895b420 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b490 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b4a8 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895b500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895b500 blob([0x3b9000~1000,0xb4a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b570 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b500) get_ref 0x0~2000 Blob(0x558c5895b500 blob([0x3b9000~1000,0xb4a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b570 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b500) get_ref init 0x2000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa2000~2000: 0x0~2000 Blob(0x558c5895b500 blob([0x3b9000~1000,0xb4a000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b570 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b588 in 0x558c53a59a40) _discard 0x0~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5895b5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5895b650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5895b5e0 blob([0xb4b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b650 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b5e0) get_ref 0x0~1000 Blob(0x558c5895b5e0 blob([0xb4b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b650 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.blob(0x558c5895b5e0) get_ref init 0x1000, 1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa4000~1000: 0x0~1000 Blob(0x558c5895b5e0 blob([0xb4b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5895b650 sbid 0x0)) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5895b668 in 0x558c53a59a40) _discard 0x0~1000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xa5000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dirty_range mark shard 0x4f000 dirty 2019-12-07T19:34:32.658+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 0x76000~2f000 = 0 2019-12-07T19:34:32.658+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.658+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.658+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000055.00000000000000000041' <- 0000000055.00000000000000000041 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.658+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:32.658+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58113800 cost 298662 (26 ios * 4000 + 194662 bytes) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58113800 onodes 0x558c57a35400 shared_blobs 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update shard 0x4f000 is 862 bytes (was 403) from 44 extents 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 123473 (123471 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.658+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58113800 allocated 0x[38d000~2d000,b4a000~2000] released 0x[] 2019-12-07T19:34:32.658+0000 7f8f08189700 10 freelist allocate 0x38d000~2d000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:32.658+0000 7f8f08189700 10 freelist allocate 0xb4a000~2000 2019-12-07T19:34:32.658+0000 7f8f08189700 20 freelist _xor first_key 0xb00000 last_key 0xb00000 2019-12-07T19:34:32.658+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113800 prepare 2019-12-07T19:34:32.658+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58113800 2019-12-07T19:34:32.658+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'41 (0'0,55'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'40 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'41 2019-12-07T19:34:32.658+0000 7f8f08189700 10 osd.7 55 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:32.658+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.658+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 267 ==== MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=150, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c579de000 con 0x558c54c5bc00 2019-12-07T19:34:32.658+0000 7f8f23c56700 15 osd.7 55 enqueue_op 0x558c578d2c60 prio 127 cost 0 latency 0.000022 epoch 55 MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=150, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.658+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=150, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) 2019-12-07T19:34:32.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.658+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 268 ==== MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=151, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c57c05b80 con 0x558c54c5bc00 2019-12-07T19:34:32.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=150, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.658+0000 7f8f23c56700 15 osd.7 55 enqueue_op 0x558c578d3b80 prio 127 cost 0 latency 0.000012 epoch 55 MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=151, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.658+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=151, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) 2019-12-07T19:34:32.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=150, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) pg 0x558c54d5c000 2019-12-07T19:34:32.658+0000 7f8f05984700 10 osd.7 55 dequeue_op 0x558c578d2c60 prio 127 cost 0 latency 0.000075 MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=150, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.658+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 459 ==== MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=274, reqid=client.4540.0:406, at_version=55'42, trim_to=0'0, roll_forward_to=53'38)) v2 ==== 2465+0+0 (crc 0 0 0) 0x558c579e2000 con 0x558c54c64000 2019-12-07T19:34:32.658+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=150, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.658+0000 7f8f24457700 15 osd.7 55 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000020 epoch 55 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=274, reqid=client.4540.0:406, at_version=55'42, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:32.658+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=274, reqid=client.4540.0:406, at_version=55'42, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.658+0000 7f8f05984700 25 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:32.658+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~29000 size 0x29000 (167936) 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e4d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e5b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=151, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0cb0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0bd0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0af0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0a10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f07e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.658+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5742a1c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:32.658+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 = 167936 2019-12-07T19:34:32.658+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read read request=167936 r=167936 len=167936 2019-12-07T19:34:32.658+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: Getting attr on 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.658+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: not in cache 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.658+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:34:32.658+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: found on disk, size 167936 2019-12-07T19:34:32.658+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key 2019-12-07T19:34:32.658+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key = 0 2019-12-07T19:34:32.658+0000 7f8f05984700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read: Checking hash of 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.658+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 55/28 ECSubReadReply(tid=150, attrs_read=0)) v2 -- 0x558c55112780 con 0x558c54c5bc00 2019-12-07T19:34:32.658+0000 7f8f05984700 10 osd.7 55 dequeue_op 0x558c578d2c60 finish 2019-12-07T19:34:32.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:32.658+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113800 aio_wait 2019-12-07T19:34:32.658+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58113800 2019-12-07T19:34:32.658+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.658+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113800 io_done 2019-12-07T19:34:32.658+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=274, reqid=client.4540.0:406, at_version=55'42, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.658+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.658+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=274, reqid=client.4540.0:406, at_version=55'42, trim_to=0'0, roll_forward_to=53'38)) v2) prio 127 cost 0 e55) pg 0x558c5500a000 2019-12-07T19:34:32.658+0000 7f8f04181700 10 osd.7 55 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000248 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=274, reqid=client.4540.0:406, at_version=55'42, trim_to=0'0, roll_forward_to=53'38)) v2 pg pg[1.9s4( v 55'41 (0'0,55'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.658+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'41 (0'0,55'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=274, reqid=client.4540.0:406, at_version=55'42, trim_to=0'0, roll_forward_to=53'38)) v2 2019-12-07T19:34:32.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.658+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=151, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) pg 0x558c54d5c000 2019-12-07T19:34:32.658+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'41 (0'0,55'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'41], crt=53'38) [55'42 (55'41) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:406 2019-12-07T19:34:32.653894+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:32.658+0000 7f8f0197c700 10 osd.7 55 dequeue_op 0x558c578d3b80 prio 127 cost 0 latency 0.000323 MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=151, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.658+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'41 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'42 (55'41) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:406 2019-12-07T19:34:32.653894+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.658+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'41 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 42 2019-12-07T19:34:32.658+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpRead(1.6s2 55/28 ECSubRead(tid=151, to_read={1:639abb38:::smithi13140796-40:head=0,167936,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.658+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'41 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.659+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.659+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'42, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.659+0000 7f8f0197c700 25 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:32.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~29000 size 0x29000 (167936) 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57538cb0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552658f0 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e1c0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e2a0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e380 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e460 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e4d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e540 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e5b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57956900 seq 80 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e620 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e690 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5757e700 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5757e770 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 3 keys 2019-12-07T19:34:32.659+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 3 keys = 0 2019-12-07T19:34:32.659+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.659+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0e00 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0d20 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0cb0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0c40 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0bd0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000055.00000000000000000042' <- 0000000055.00000000000000000042 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0b60 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0af0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0a80 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0a10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f09a0 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f0930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f08c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574f07e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57956900 cost 6116 (1 ios * 4000 + 2116 bytes) 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57956900 onodes 0x558c57a35400 shared_blobs 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f0850 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54ea3810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c552798f0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c55279500 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a070 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742a0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5742a150 blob([0x1ce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5742a1c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:32.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 0x0~29000 = 167936 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 123473 (123471 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read read request=167936 r=167936 len=167936 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57956900 allocated 0x[] released 0x[] 2019-12-07T19:34:32.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: Getting attr on 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.659+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956900 prepare 2019-12-07T19:34:32.659+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57956900 2019-12-07T19:34:32.659+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956900 io_done 2019-12-07T19:34:32.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: not in cache 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:34:32.659+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'41 crt=53'38 lcod 55'40 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'42 2019-12-07T19:34:32.659+0000 7f8f0197c700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: found on disk, size 167936 2019-12-07T19:34:32.659+0000 7f8f04181700 10 osd.7 55 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:32.659+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key 2019-12-07T19:34:32.659+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.659+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.6s2_head 2#1:639abb38:::smithi13140796-40:head# hinfo_key = 0 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read: Checking hash of 1:639abb38:::smithi13140796-40:head 2019-12-07T19:34:32.659+0000 7f8f0197c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 55/28 ECSubReadReply(tid=151, attrs_read=0)) v2 -- 0x558c585cfb80 con 0x558c54c5bc00 2019-12-07T19:34:32.659+0000 7f8f0197c700 10 osd.7 55 dequeue_op 0x558c578d3b80 finish 2019-12-07T19:34:32.659+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:32.659+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 187 ==== MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=72, reqid=client.4540.0:410, at_version=55'28, trim_to=0'0, roll_forward_to=54'24)) v2 ==== 2437+0+0 (crc 0 0 0) 0x558c551dd800 con 0x558c54c19800 2019-12-07T19:34:32.659+0000 7f8f24c58700 15 osd.7 55 enqueue_op 0x558c57efe6e0 prio 127 cost 0 latency 0.000029 epoch 55 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=72, reqid=client.4540.0:410, at_version=55'28, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:32.659+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=72, reqid=client.4540.0:410, at_version=55'28, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.659+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.659+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=72, reqid=client.4540.0:410, at_version=55'28, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.659+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.659+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=72, reqid=client.4540.0:410, at_version=55'28, trim_to=0'0, roll_forward_to=54'24)) v2) prio 127 cost 0 e55) pg 0x558c55141000 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 55 dequeue_op 0x558c57efe6e0 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=72, reqid=client.4540.0:410, at_version=55'28, trim_to=0'0, roll_forward_to=54'24)) v2 pg pg[1.ds2( v 55'27 (0'0,55'27] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'27 (0'0,55'27] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=72, reqid=client.4540.0:410, at_version=55'28, trim_to=0'0, roll_forward_to=54'24)) v2 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'27 (0'0,55'27] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'27], crt=54'24) [55'28 (55'27) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:410 2019-12-07T19:34:32.653973+0000 0 ObjectCleanRegions clean_offsets: [0~1540096,1540096~18446744073708011519], clean_omap: 1, new_object: 0] 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'27 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'28 (55'27) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:410 2019-12-07T19:34:32.653973+0000 0 ObjectCleanRegions clean_offsets: [0~1540096,1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'27 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 28 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'27 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.659+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.659+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'28, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5853a600 seq 61 2019-12-07T19:34:32.659+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:32.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:32.659+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6308 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000055.00000000000000000028' <- 0000000055.00000000000000000028 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:32.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853a600 cost 6087 (1 ios * 4000 + 2087 bytes) 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853a600 onodes 0x558c585cf180 shared_blobs 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore.extentmap(0x558c585cf288) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 636 (634 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853a600 allocated 0x[] released 0x[] 2019-12-07T19:34:32.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a600 prepare 2019-12-07T19:34:32.659+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853a600 2019-12-07T19:34:32.659+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a600 io_done 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'27 crt=54'24 lcod 55'26 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'28 2019-12-07T19:34:32.659+0000 7f8f06185700 10 osd.7 55 dequeue_op 0x558c57efe6e0 finish 2019-12-07T19:34:32.659+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13900 had 1 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585cf180 had 2 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 2 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 26290 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 59058 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 91826 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 124594 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 132002 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 2 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 157362 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 190130 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 222898 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 255666 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 256705 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 256728 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 256728 < min_flush_size 524288 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x370000 mtime 2019-12-07T19:34:32.656823+0000 allocated 370000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~340000]) 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x36194e~3ead8 to file(ino 16 size 0x370000 mtime 2019-12-07T19:34:32.656823+0000 allocated 370000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~340000]) 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x36194e 0x36194e~3ead8 to file(ino 16 size 0x370000 mtime 2019-12-07T19:34:32.656823+0000 allocated 370000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~340000]) 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _allocate len 0x30426 from 0 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _allocate len 0x30426 from 1 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x40000/10000,40000,acd4a0000 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd4a0000~40000/10000,40000,acd4a0000 2019-12-07T19:34:32.659+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x3b0000 to include allocated 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 68 (was clean) 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~380000 x_off 0x33194e 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x94e 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x426 and padding block with 0xbda 2019-12-07T19:34:32.659+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x3b0000 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3a0426 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (68) on file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]), flushing log 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 68 len 0x4a crc 0xf05a1fee) 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x43000~1000 to file(ino 1 size 0x43000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x43000 0x43000~1000 to file(ino 1 size 0x43000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x44000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x43000 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x44000 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000029 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 68 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000922448s (0.000061166s flush + 0.000861282s kv commit) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58540c00,0x558c58113800,0x558c57956900> 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 kv_submitted 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58540c00 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1815 < min_flush_size 524288 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 finishing 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58540c00 onodes 0x558c54e13900,0x558c585cf180 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540c00 done 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853a600 kv_queued 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58540c00 [] 2019-12-07T19:34:32.660+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585cf180 had 1 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1838 < min_flush_size 524288 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1838 < min_flush_size 524288 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113800 kv_submitted 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58113800 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3a0426~72e to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113800 finishing 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3a0426 0x3a0426~72e to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58113800 onodes 0x558c57a35400 2019-12-07T19:34:32.660+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 55/41 ECSubWriteReply(tid=71, last_complete=55'27, committed=1, applied=1)) v2 -- 0x558c5841f800 con 0x558c54c19800 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7128 in 0x558c53a59a40) _finish_write discard buffer(0x558c57983ef0 space 0x558c575f7128 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~380000 x_off 0x370426 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7208 in 0x558c53a59a40) _finish_write discard buffer(0x558c5796fc20 space 0x558c575f7208 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x426 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f73c8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57983200 space 0x558c575f73c8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb54 and padding block with 0x4ac 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f74a8 in 0x558c53a59a40) _finish_write discard buffer(0x558c57983170 space 0x558c575f74a8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7588 in 0x558c53a59a40) _finish_write discard buffer(0x558c57982990 space 0x558c575f7588 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7668 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eacf0 space 0x558c575f7668 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7748 in 0x558c53a59a40) _finish_write discard buffer(0x558c585ead80 space 0x558c575f7748 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7828 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eae10 space 0x558c575f7828 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7908 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eaea0 space 0x558c575f7908 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 55/41 ECSubWriteReply(tid=270, last_complete=55'41, committed=1, applied=1)) v2 -- 0x558c57950400 con 0x558c54c64000 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f79e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eaf30 space 0x558c575f79e8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7ac8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eafc0 space 0x558c575f7ac8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c575f7ba8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb050 space 0x558c575f7ba8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578df668 in 0x558c53a59a40) _finish_write discard buffer(0x558c57983e60 space 0x558c578df668 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970f68 in 0x558c53a59a40) _finish_write discard buffer(0x558c5796fb90 space 0x558c57970f68 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895ae88 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb0e0 space 0x558c5895ae88 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895af68 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb170 space 0x558c5895af68 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b048 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb200 space 0x558c5895b048 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b128 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb290 space 0x558c5895b128 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b208 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb320 space 0x558c5895b208 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b2e8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb3b0 space 0x558c5895b2e8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b3c8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb440 space 0x558c5895b3c8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b4a8 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb4d0 space 0x558c5895b4a8 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b588 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb560 space 0x558c5895b588 0x0~2000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5895b668 in 0x558c53a59a40) _finish_write discard buffer(0x558c585eb5f0 space 0x558c5895b668 0x0~1000 writing nocache) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3a0b54 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000094540s (0.000000154s flush + 0.000094386s kv commit) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58113800 done 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57956900 kv_submitted 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58113800 [] 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956900 kv_submitted 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57956900 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956900 finishing 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57956900 onodes 0x558c57a35400 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57956900 done 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57956900 [] 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853a600> 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a600 kv_submitted 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853a600 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a600 finishing 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853a600 onodes 0x558c585cf180 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853a600 done 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853a600 [] 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.660+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 55/41 ECSubWriteReply(tid=274, last_complete=55'42, committed=1, applied=1)) v2 -- 0x558c57950200 con 0x558c54c64000 2019-12-07T19:34:32.660+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 55/41 ECSubWriteReply(tid=72, last_complete=55'28, committed=1, applied=1)) v2 -- 0x558c580b9e00 con 0x558c54c19800 2019-12-07T19:34:32.660+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 269 ==== pg_update_log_missing(1.fs5 epoch 55/41 rep_tid 149 entries 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 29'12) v3 ==== 258+0+0 (crc 0 0 0) 0x558c579d8240 con 0x558c54c5bc00 2019-12-07T19:34:32.660+0000 7f8f23c56700 15 osd.7 55 enqueue_op 0x558c578d2000 prio 127 cost 0 latency 0.000020 epoch 55 pg_update_log_missing(1.fs5 epoch 55/41 rep_tid 149 entries 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 29'12) v3 2019-12-07T19:34:32.660+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=pg_update_log_missing(1.fs5 epoch 55/41 rep_tid 149 entries 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 29'12) v3) prio 127 cost 0 e55) 2019-12-07T19:34:32.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=pg_update_log_missing(1.fs5 epoch 55/41 rep_tid 149 entries 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 29'12) v3) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=pg_update_log_missing(1.fs5 epoch 55/41 rep_tid 149 entries 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 29'12) v3) prio 127 cost 0 e55) pg 0x558c57383000 2019-12-07T19:34:32.660+0000 7f8f05183700 10 osd.7 55 dequeue_op 0x558c578d2000 prio 127 cost 0 latency 0.000058 pg_update_log_missing(1.fs5 epoch 55/41 rep_tid 149 entries 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 29'12) v3 pg pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.660+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: pg_update_log_missing(1.fs5 epoch 55/41 rep_tid 149 entries 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 29'12) v3 2019-12-07T19:34:32.660+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.fs5( v 54'14 (0'0,54'14] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] do_update_log_missing op_trim_to = -- op_roll_forward_to = 29'12 2019-12-07T19:34:32.660+0000 7f8f05183700 20 update missing, append 55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.660+0000 7f8f05183700 20 osd.7 pg_epoch: 55 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=54'14 crt=54'14 lcod 54'14 active mbc={} ps=[e~1,13~1,19~2]] append_log_entries_update_missing trim_to bool = 0 trim_to = 0'0 2019-12-07T19:34:32.660+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'15, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.660+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c583b6300 seq 27 2019-12-07T19:34:32.660+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000055.00000000000000000015' <- 0000000055.00000000000000000015 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._fastinfo' <- _fastinfo 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.660+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:32.660+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b6300 cost 4735 (1 ios * 4000 + 735 bytes) 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b6300 onodes shared_blobs 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b6300 allocated 0x[] released 0x[] 2019-12-07T19:34:32.660+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6300 prepare 2019-12-07T19:34:32.660+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b6300 2019-12-07T19:34:32.660+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6300 io_done 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.660+0000 7f8f05183700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=54'14 crt=54'14 lcod 54'14 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'15 2019-12-07T19:34:32.660+0000 7f8f05183700 10 osd.7 55 dequeue_op 0x558c578d2000 finish 2019-12-07T19:34:32.660+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 591 < min_flush_size 524288 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 614 < min_flush_size 524288 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 614 < min_flush_size 524288 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3a0b54~266 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3a0b54 0x3a0b54~266 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~380000 x_off 0x370b54 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb54 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdba and padding block with 0x246 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3a0dba 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000087202s (0.000000106s flush + 0.000087096s kv commit) 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583b6300> 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6300 kv_submitted 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b6300 2019-12-07T19:34:32.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6300 finishing 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b6300 onodes 2019-12-07T19:34:32.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b6300 done 2019-12-07T19:34:32.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:32.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b6300 [] 2019-12-07T19:34:32.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.661+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.661+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_update_log_missing_reply(1.fs0 epoch 55/41 rep_tid 149 lcod 55'15) v3 -- 0x558c58513e00 con 0x558c54c5bc00 2019-12-07T19:34:32.661+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:32.664+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 270 ==== MOSDECSubOpRead(1.1s1 55/41 ECSubRead(tid=152, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c585cfb80 con 0x558c54c5bc00 2019-12-07T19:34:32.664+0000 7f8f23c56700 15 osd.7 55 enqueue_op 0x558c581ccb00 prio 127 cost 0 latency 0.000022 epoch 55 MOSDECSubOpRead(1.1s1 55/41 ECSubRead(tid=152, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.664+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 55/41 ECSubRead(tid=152, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) 2019-12-07T19:34:32.664+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 55/41 ECSubRead(tid=152, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.664+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 55/41 ECSubRead(tid=152, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e55) pg 0x558c5736a000 2019-12-07T19:34:32.664+0000 7f8f08189700 10 osd.7 55 dequeue_op 0x558c581ccb00 prio 127 cost 0 latency 0.000154 MOSDECSubOpRead(1.1s1 55/41 ECSubRead(tid=152, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.664+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpRead(1.1s1 55/41 ECSubRead(tid=152, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:32.664+0000 7f8f08189700 25 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:32.664+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~4d000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~4d000 size 0x4d000 (315392) 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcb570 blob([0x740000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbb90 blob([0x742000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaa80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbb20 blob([0x744000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x746000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74f50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ad260 blob([0x748000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbc70 blob([0x74a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaa10 blob([0x74c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbc00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbce0 blob([0x74e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbd50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbdc0 blob([0x750000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbe30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbea0 blob([0x752000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbf10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbf80 blob([0x754000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800d7a0 blob([0x756000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800d880 blob([0x758000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d8f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800d960 blob([0x75a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d9d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800da40 blob([0x75c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800db20 blob([0x75e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800db90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800dc00 blob([0x760000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dc70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800dce0 blob([0x762000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dd50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ddc0 blob([0x764000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800de30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800dea0 blob([0x766000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800df10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800df80 blob([0x768000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66070 blob([0x76a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f660e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66150 blob([0x76c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f661c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66230 blob([0x76e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f662a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66310 blob([0x770000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f663f0 blob([0x772000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f664d0 blob([0x774000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66540 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f665b0 blob([0x776000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66690 blob([0x778000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66770 blob([0x77a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f667e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66850 blob([0x77c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f668c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66930 blob([0x77e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f669a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576acfc0 blob([0x780000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dd50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c000 blob([0x782000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54df8a10 blob([0x784000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cc40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ccb0 blob([0x786000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cd20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800cd90 blob([0x788000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ce00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ce70 blob([0x78a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800cf50 blob([0x78c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800cfc0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54d3d9d0 blob([0x746000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74f50 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c54df8a10 blob([0x784000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cc40 sbid 0x0)) need {<0x0, 0x2000> : [0x44000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x44000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576acfc0 blob([0x780000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dd50 sbid 0x0)) need {<0x0, 0x2000> : [0x40000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x40000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c576ad260 blob([0x748000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb650 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5792c000 blob([0x782000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c380 sbid 0x0)) need {<0x0, 0x2000> : [0x42000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x42000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcaa10 blob([0x74c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbc00 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcb570 blob([0x740000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb500 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcbb20 blob([0x744000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d960 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcbb90 blob([0x742000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaa80 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcbc70 blob([0x74a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74ee0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcbce0 blob([0x74e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbd50 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcbdc0 blob([0x750000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbe30 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcbea0 blob([0x752000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbf10 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57dcbf80 blob([0x754000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d730 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66070 blob([0x76a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f660e0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66150 blob([0x76c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f661c0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66230 blob([0x76e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f662a0 sbid 0x0)) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66310 blob([0x770000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66380 sbid 0x0)) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f663f0 blob([0x772000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66460 sbid 0x0)) need {<0x0, 0x2000> : [0x32000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x32000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f664d0 blob([0x774000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66540 sbid 0x0)) need {<0x0, 0x2000> : [0x34000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x34000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f665b0 blob([0x776000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66620 sbid 0x0)) need {<0x0, 0x2000> : [0x36000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x36000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66690 blob([0x778000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66700 sbid 0x0)) need {<0x0, 0x2000> : [0x38000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x38000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66770 blob([0x77a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f667e0 sbid 0x0)) need {<0x0, 0x2000> : [0x3a000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3a000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66850 blob([0x77c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f668c0 sbid 0x0)) need {<0x0, 0x2000> : [0x3c000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3c000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57f66930 blob([0x77e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f669a0 sbid 0x0)) need {<0x0, 0x2000> : [0x3e000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x3e000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800ccb0 blob([0x786000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cd20 sbid 0x0)) need {<0x0, 0x2000> : [0x46000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x46000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800cd90 blob([0x788000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ce00 sbid 0x0)) need {<0x0, 0x2000> : [0x48000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x48000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800ce70 blob([0x78a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cee0 sbid 0x0)) need {<0x0, 0x2000> : [0x4a000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4a000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800cf50 blob([0x78c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800cfc0 sbid 0x0)) need {<0x0, 0x1000> : [0x4c000:0~1000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x4c000: 0x0 reading 0x0~1000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800d7a0 blob([0x756000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d810 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800d880 blob([0x758000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d8f0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800d960 blob([0x75a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d9d0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800da40 blob([0x75c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dab0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800db20 blob([0x75e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800db90 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800dc00 blob([0x760000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dc70 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800dce0 blob([0x762000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dd50 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800ddc0 blob([0x764000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800de30 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800dea0 blob([0x766000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800df10 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c5800df80 blob([0x768000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66000 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:32.664+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54d3d9d0 blob([0x746000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74f50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d74f68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.665+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ea1b0 space 0x558c54d74f68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c54df8a10 blob([0x784000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cc40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x44000:0~2000]} 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800cc58 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.665+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eb710 space 0x558c5800cc58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576acfc0 blob([0x780000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dd50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x40000:0~2000]} 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5794dd68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.665+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eb7a0 space 0x558c5794dd68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c576ad260 blob([0x748000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcb668 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.665+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eb830 space 0x558c57dcb668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.665+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5792c000 blob([0x782000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x42000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5792c398 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eb560 space 0x558c5792c398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcaa10 blob([0x74c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbc00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcbc18 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eb8c0 space 0x558c57dcbc18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcb570 blob([0x740000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcb518 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eb950 space 0x558c57dcb518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcbb20 blob([0x744000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d3d978 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eb9e0 space 0x558c54d3d978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcbb90 blob([0x742000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaa80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcaa98 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585eba70 space 0x558c57dcaa98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcbc70 blob([0x74a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c54d74ef8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebb00 space 0x558c54d74ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcbce0 blob([0x74e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbd50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcbd68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebb90 space 0x558c57dcbd68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcbdc0 blob([0x750000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbe30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcbe48 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebc20 space 0x558c57dcbe48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcbea0 blob([0x752000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbf10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57dcbf28 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebcb0 space 0x558c57dcbf28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57dcbf80 blob([0x754000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800d748 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebd40 space 0x558c5800d748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66070 blob([0x76a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f660e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f660f8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebdd0 space 0x558c57f660f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66150 blob([0x76c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f661c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f661d8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebe60 space 0x558c57f661d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66230 blob([0x76e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f662a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f662b8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c585ebef0 space 0x558c57f662b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66310 blob([0x770000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f66398 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c57f763f0 space 0x558c57f66398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f663f0 blob([0x772000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x32000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f66478 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c5813b170 space 0x558c57f66478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f664d0 blob([0x774000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66540 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x34000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f66558 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c54dba090 space 0x558c57f66558 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f665b0 blob([0x776000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x36000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f66638 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2000 space 0x558c57f66638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66690 blob([0x778000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x38000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f66718 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2090 space 0x558c57f66718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66770 blob([0x77a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f667e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3a000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f667f8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2120 space 0x558c57f667f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66850 blob([0x77c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f668c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3c000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f668d8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f21b0 space 0x558c57f668d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57f66930 blob([0x77e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f669a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x3e000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f669b8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2240 space 0x558c57f669b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800ccb0 blob([0x786000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cd20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x46000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800cd38 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f22d0 space 0x558c5800cd38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800cd90 blob([0x788000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ce00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x48000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800ce18 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2360 space 0x558c5800ce18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800ce70 blob([0x78a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4a000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800cef8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f23f0 space 0x558c5800cef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800cf50 blob([0x78c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800cfc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4c000:0~1000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800cfd8 in 0x558c53a59500) _discard 0x0~1000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2480 space 0x558c5800cfd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800d7a0 blob([0x756000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800d828 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2510 space 0x558c5800d828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800d880 blob([0x758000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d8f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800d908 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f25a0 space 0x558c5800d908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800d960 blob([0x75a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d9d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800d9e8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2630 space 0x558c5800d9e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800da40 blob([0x75c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800dac8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f26c0 space 0x558c5800dac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800db20 blob([0x75e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800db90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800dba8 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2750 space 0x558c5800dba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800dc00 blob([0x760000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dc70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800dc88 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f27e0 space 0x558c5800dc88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800dce0 blob([0x762000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dd50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800dd68 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2870 space 0x558c5800dd68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800ddc0 blob([0x764000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800de30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800de48 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2900 space 0x558c5800de48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800dea0 blob([0x766000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800df10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c5800df28 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2990 space 0x558c5800df28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c5800df80 blob([0x768000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:32.666+0000 7f8f08189700 20 bluestore.BufferSpace(0x558c57f66018 in 0x558c53a59500) _discard 0x0~2000 2019-12-07T19:34:32.666+0000 7f8f08189700 20 _add level 1 near 0 on buffer(0x558c580f2a20 space 0x558c57f66018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:32.666+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~4d000 = 315392 2019-12-07T19:34:32.666+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read read request=315392 r=315392 len=315392 2019-12-07T19:34:32.666+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:32.666+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:32.666+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:32.666+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:32.666+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:34:32.666+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:34:32.666+0000 7f8f08189700 20 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_sub_read: Checking hash of 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:32.666+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 55/41 ECSubReadReply(tid=152, attrs_read=0)) v2 -- 0x558c5895c000 con 0x558c54c5bc00 2019-12-07T19:34:32.666+0000 7f8f08189700 10 osd.7 55 dequeue_op 0x558c581ccb00 finish 2019-12-07T19:34:32.666+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.667+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 460 ==== MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=278, reqid=client.4540.0:406, at_version=0'0, trim_to=0'0, roll_forward_to=55'42)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c579e0800 con 0x558c54c64000 2019-12-07T19:34:32.667+0000 7f8f24457700 15 osd.7 55 enqueue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000022 epoch 55 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=278, reqid=client.4540.0:406, at_version=0'0, trim_to=0'0, roll_forward_to=55'42)) v2 2019-12-07T19:34:32.667+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=278, reqid=client.4540.0:406, at_version=0'0, trim_to=0'0, roll_forward_to=55'42)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=278, reqid=client.4540.0:406, at_version=0'0, trim_to=0'0, roll_forward_to=55'42)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=278, reqid=client.4540.0:406, at_version=0'0, trim_to=0'0, roll_forward_to=55'42)) v2) prio 127 cost 0 e55) pg 0x558c5500a000 2019-12-07T19:34:32.667+0000 7f8f04181700 10 osd.7 55 dequeue_op 0x558c54e78dc0 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=278, reqid=client.4540.0:406, at_version=0'0, trim_to=0'0, roll_forward_to=55'42)) v2 pg pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.667+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.9s4 55/41 ECSubWrite(tid=278, reqid=client.4540.0:406, at_version=0'0, trim_to=0'0, roll_forward_to=55'42)) v2 2019-12-07T19:34:32.667+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=53'38 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'42], crt=53'38) [] 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'39 (53'38) clone 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:18.620957+0000 0 snaps [20,1b,15] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'40 (53'38) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:404 2019-12-07T19:34:32.649174+0000 0 ObjectCleanRegions clean_offsets: [0~1261568,1933312~18446744073707618303], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'41 (55'40) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:405 2019-12-07T19:34:32.653855+0000 0 ObjectCleanRegions clean_offsets: [0~1933312,2703360~18446744073706848255], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'42 (55'41) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:406 2019-12-07T19:34:32.653894+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.667+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 42 2019-12-07T19:34:32.667+0000 7f8f04181700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.667+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.667+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.667+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:32.667+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57956c00 seq 81 2019-12-07T19:34:32.667+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:32.667+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:32.667+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.667+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.667+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:32.667+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57956c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:32.667+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57956c00 onodes shared_blobs 2019-12-07T19:34:32.667+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57956c00 allocated 0x[] released 0x[] 2019-12-07T19:34:32.667+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956c00 prepare 2019-12-07T19:34:32.667+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57956c00 2019-12-07T19:34:32.667+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956c00 io_done 2019-12-07T19:34:32.667+0000 7f8f04181700 10 osd.7 55 dequeue_op 0x558c54e78dc0 finish 2019-12-07T19:34:32.667+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3a0dba~471 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3a0dba 0x3a0dba~471 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~380000 x_off 0x370dba 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdba 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x22b and padding block with 0xdd5 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3a122b 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.668+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000244653s (0.000000166s flush + 0.000244487s kv commit) 2019-12-07T19:34:32.668+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57956c00> 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956c00 kv_submitted 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57956c00 2019-12-07T19:34:32.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956c00 finishing 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57956c00 onodes 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57956c00 done 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:32.668+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57956c00 [] 2019-12-07T19:34:32.668+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.668+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.668+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 55/41 ECSubWriteReply(tid=278, last_complete=55'42, committed=1, applied=1)) v2 -- 0x558c58512600 con 0x558c54c64000 2019-12-07T19:34:32.668+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.668+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:32.668+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 188 ==== MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=75, reqid=client.4540.0:410, at_version=0'0, trim_to=0'0, roll_forward_to=55'28)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c575f4000 con 0x558c54c19800 2019-12-07T19:34:32.668+0000 7f8f24c58700 15 osd.7 55 enqueue_op 0x558c57efe580 prio 127 cost 0 latency 0.000020 epoch 55 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=75, reqid=client.4540.0:410, at_version=0'0, trim_to=0'0, roll_forward_to=55'28)) v2 2019-12-07T19:34:32.668+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=75, reqid=client.4540.0:410, at_version=0'0, trim_to=0'0, roll_forward_to=55'28)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=75, reqid=client.4540.0:410, at_version=0'0, trim_to=0'0, roll_forward_to=55'28)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=75, reqid=client.4540.0:410, at_version=0'0, trim_to=0'0, roll_forward_to=55'28)) v2) prio 127 cost 0 e55) pg 0x558c55141000 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 osd.7 55 dequeue_op 0x558c57efe580 prio 127 cost 0 latency 0.000337 MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=75, reqid=client.4540.0:410, at_version=0'0, trim_to=0'0, roll_forward_to=55'28)) v2 pg pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.ds2 55/41 ECSubWrite(tid=75, reqid=client.4540.0:410, at_version=0'0, trim_to=0'0, roll_forward_to=55'28)) v2 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=54'24 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'28], crt=54'24) [] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'25 (54'24) clone 1:bca18876:::smithi13140796-30:20 by unknown.0.0:0 2019-12-07T19:34:16.632044+0000 0 snaps [20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'26 (54'24) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:408 2019-12-07T19:34:32.653937+0000 0 ObjectCleanRegions clean_offsets: [0~1540096,1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'27 (55'26) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:409 2019-12-07T19:34:32.653968+0000 0 ObjectCleanRegions clean_offsets: [1540096~18446744073708011519], clean_omap: 0, new_object: 0 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'28 (55'27) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:410 2019-12-07T19:34:32.653973+0000 0 ObjectCleanRegions clean_offsets: [0~1540096,1540096~18446744073708011519], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 28 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.669+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58540f00 seq 62 2019-12-07T19:34:32.669+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bca18876:::smithi13140796-30:head#1b onode 0x558c54e13900 txc 0x558c58540f00 2019-12-07T19:34:32.669+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bca18876:::smithi13140796-30:head#1b 0x0 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c575836c0) put_ref 0x0~2000 Blob(0x558c575836c0 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c575835e0) put_ref 0x0~2000 Blob(0x558c575835e0 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57583260) put_ref 0x0~2000 Blob(0x558c57583260 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57538a80) put_ref 0x0~2000 Blob(0x558c57538a80 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c575389a0) put_ref 0x0~2000 Blob(0x558c575389a0 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c575388c0) put_ref 0x0~2000 Blob(0x558c575388c0 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c575387e0) put_ref 0x0~2000 Blob(0x558c575387e0 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57538690) put_ref 0x0~2000 Blob(0x558c57538690 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c546772d0) put_ref 0x0~2000 Blob(0x558c546772d0 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c5809b420) put_ref 0x0~2000 Blob(0x558c5809b420 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57c42460) put_ref 0x0~2000 Blob(0x558c57c42460 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c5794c690) put_ref 0x0~2000 Blob(0x558c5794c690 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c580ec380) put_ref 0x0~2000 Blob(0x558c580ec380 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57bea1c0) put_ref 0x0~2000 Blob(0x558c57bea1c0 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c55060230) put_ref 0x0~2000 Blob(0x558c55060230 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c574d3ab0) put_ref 0x0~2000 Blob(0x558c574d3ab0 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57db55e0) put_ref 0x0~2000 Blob(0x558c57db55e0 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57db5340) put_ref 0x0~2000 Blob(0x558c57db5340 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c574d2310) put_ref 0x0~2000 Blob(0x558c574d2310 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57881260) put_ref 0x0~2000 Blob(0x558c57881260 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57efc0e0) put_ref 0x0~2000 Blob(0x558c57efc0e0 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57a07960) put_ref 0x0~2000 Blob(0x558c57a07960 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c57ff4150) put_ref 0x0~2000 Blob(0x558c57ff4150 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c54df8930) put_ref 0x0~2000 Blob(0x558c54df8930 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c580a5500) put_ref 0x0~2000 Blob(0x558c580a5500 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67c70) put_ref 0x0~2000 Blob(0x558c58c67c70 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67d50) put_ref 0x0~2000 Blob(0x558c58c67d50 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67e30) put_ref 0x0~2000 Blob(0x558c58c67e30 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c67f10) put_ref 0x0~2000 Blob(0x558c58c67f10 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78000) put_ref 0x0~2000 Blob(0x558c58c78000 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c780e0) put_ref 0x0~2000 Blob(0x558c58c780e0 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c781c0) put_ref 0x0~2000 Blob(0x558c58c781c0 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c782a0) put_ref 0x0~2000 Blob(0x558c58c782a0 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78380) put_ref 0x0~2000 Blob(0x558c58c78380 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78460) put_ref 0x0~2000 Blob(0x558c58c78460 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78540) put_ref 0x0~2000 Blob(0x558c58c78540 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78620) put_ref 0x0~2000 Blob(0x558c58c78620 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78700) put_ref 0x0~2000 Blob(0x558c58c78700 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c787e0) put_ref 0x0~2000 Blob(0x558c58c787e0 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c788c0) put_ref 0x0~2000 Blob(0x558c58c788c0 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c789a0) put_ref 0x0~2000 Blob(0x558c58c789a0 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78a80) put_ref 0x0~2000 Blob(0x558c58c78a80 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78b60) put_ref 0x0~2000 Blob(0x558c58c78b60 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78c40) put_ref 0x0~2000 Blob(0x558c58c78c40 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78d20) put_ref 0x0~2000 Blob(0x558c58c78d20 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78e00) put_ref 0x0~2000 Blob(0x558c58c78e00 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.blob(0x558c58c78ee0) put_ref 0x0~2000 Blob(0x558c58c78ee0 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dirty_range mark shard 0x24000 dirty 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e13a08) dirty_range mark shard 0x48000 dirty 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c575836c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xae6000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c575835e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xae8000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57583260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaea000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57538a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaec000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c575389a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaee000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c575388c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf0000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c575387e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf2000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57538690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf4000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c546772d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf6000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c5809b420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf8000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57c42460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xafa000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c5794c690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xafc000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c580ec380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xafe000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57bea1c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb00000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c55060230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb02000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c574d3ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb04000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c57db55e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb06000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57db5340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb08000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c574d2310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb0a000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c57881260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb0c000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c57efc0e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb0e000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c57a07960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb10000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c57ff4150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb12000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c54df8930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb14000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c580a5500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb16000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c58c67c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb18000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c58c67d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb1a000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~2000: 0x0~2000 Blob(0x558c58c67e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb1c000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x38000~2000: 0x0~2000 Blob(0x558c58c67f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb1e000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3a000~2000: 0x0~2000 Blob(0x558c58c78000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb20000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3c000~2000: 0x0~2000 Blob(0x558c58c780e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb22000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3e000~2000: 0x0~2000 Blob(0x558c58c781c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb24000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x40000~2000: 0x0~2000 Blob(0x558c58c782a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb26000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x42000~2000: 0x0~2000 Blob(0x558c58c78380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb28000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x44000~2000: 0x0~2000 Blob(0x558c58c78460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb2a000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x46000~2000: 0x0~2000 Blob(0x558c58c78540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb2c000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x48000~2000: 0x0~2000 Blob(0x558c58c78620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb2e000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4a000~2000: 0x0~2000 Blob(0x558c58c78700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb30000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~2000: 0x0~2000 Blob(0x558c58c787e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb32000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4e000~2000: 0x0~2000 Blob(0x558c58c788c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb34000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x50000~2000: 0x0~2000 Blob(0x558c58c789a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb36000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x52000~2000: 0x0~2000 Blob(0x558c58c78a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb38000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x54000~2000: 0x0~2000 Blob(0x558c58c78b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb3a000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x56000~2000: 0x0~2000 Blob(0x558c58c78c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb3c000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x58000~2000: 0x0~2000 Blob(0x558c58c78d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb3e000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5a000~2000: 0x0~2000 Blob(0x558c58c78e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb40000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5c000~2000: 0x0~2000 Blob(0x558c58c78ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=3)))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb42000~2000] 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x24000 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x48000 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bca18876:::smithi13140796-30:head#1b = 0 2019-12-07T19:34:32.669+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58540f00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58540f00 onodes shared_blobs 0x558c54cf8230,0x558c550604d0,0x558c574d2cb0,0x558c574fd9d0,0x558c578f00e0,0x558c578f0150,0x558c578f1110,0x558c578f1180,0x558c578f1340,0x558c5794d9d0,0x558c57a06bd0,0x558c57a07f80,0x558c57bea540,0x558c57beac40,0x558c57db5500,0x558c57db5570,0x558c57db5960,0x558c57e63d50,0x558c57f512d0,0x558c57f51340,0x558c57f51810,0x558c57ff4310,0x558c5809b180,0x558c5809b3b0,0x558c58c67c00,0x558c58c67ce0,0x558c58c67dc0,0x558c58c67ea0,0x558c58c67f80,0x558c58c78070,0x558c58c78150,0x558c58c78230,0x558c58c78310,0x558c58c783f0,0x558c58c784d0,0x558c58c785b0,0x558c58c78690,0x558c58c78770,0x558c58c78850,0x558c58c78930,0x558c58c78a10,0x558c58c78af0,0x558c58c78bd0,0x558c58c78cb0,0x558c58c78d90,0x558c58c78e70,0x558c58c78f50 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e8 is 11 SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ea is 11 SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f2 is 11 SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e6 is 11 SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e2 is 11 SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e1 is 11 SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e0 is 11 SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e4 is 11 SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e7 is 11 SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ed is 11 SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ee is 11 SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f3 is 11 SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f0 is 11 SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f5 is 11 SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f1 is 11 SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ef is 11 SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29eb is 11 SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e5 is 11 SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e3 is 11 SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29df is 11 SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e9 is 11 SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ec is 11 SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f6 is 11 SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f4 is 11 SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f7 is 11 SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f8 is 11 SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f9 is 11 SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fa is 11 SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fb is 11 SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fc is 11 SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fd is 11 SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fe is 11 SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ff is 11 SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a00 is 11 SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a01 is 11 SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a02 is 11 SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a03 is 11 SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a04 is 11 SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a05 is 11 SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a06 is 11 SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a07 is 11 SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a08 is 11 SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a09 is 11 SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0a is 11 SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0b is 11 SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0c is 11 SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0d is 11 SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2))) 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58540f00 allocated 0x[] released 0x[] 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540f00 prepare 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58540f00 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540f00 io_done 2019-12-07T19:34:32.669+0000 7f8f0217d700 10 osd.7 55 dequeue_op 0x558c57efe580 finish 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2526 < min_flush_size 524288 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13900 had 1 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2549 < min_flush_size 524288 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2549 < min_flush_size 524288 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3a122b~9f5 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3a122b 0x3a122b~9f5 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~380000 x_off 0x37122b 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x22b 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc20 and padding block with 0x3e0 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3a1c20 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.669+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000152883s (0.000000115s flush + 0.000152768s kv commit) 2019-12-07T19:34:32.669+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58540f00> 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540f00 kv_submitted 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58540f00 2019-12-07T19:34:32.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540f00 finishing 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58540f00 onodes 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540f00 done 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:32.669+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58540f00 [] 2019-12-07T19:34:32.669+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.669+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 55/41 ECSubWriteReply(tid=75, last_complete=55'28, committed=1, applied=1)) v2 -- 0x558c58332000 con 0x558c54c19800 2019-12-07T19:34:32.669+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.669+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.669+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:32.678+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 285 ==== osd_map(55..55 src has 1..55) v4 ==== 633+0+0 (unknown 0 0 0) 0x558c579d9200 con 0x558c54c5b400 2019-12-07T19:34:32.678+0000 7f8f1199c700 20 osd.7 55 OSD::ms_dispatch: osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.678+0000 7f8f1199c700 10 osd.7 55 do_waiters -- start 2019-12-07T19:34:32.678+0000 7f8f1199c700 10 osd.7 55 do_waiters -- finish 2019-12-07T19:34:32.678+0000 7f8f1199c700 20 osd.7 55 _dispatch 0x558c579d9200 osd_map(55..55 src has 1..55) v4 2019-12-07T19:34:32.678+0000 7f8f1199c700 3 osd.7 55 handle_osd_map epochs [55,55], i have 55, src has [1,55] 2019-12-07T19:34:32.678+0000 7f8f1199c700 10 osd.7 55 no new maps here, dropping 2019-12-07T19:34:32.678+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 286 ==== MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=107, reqid=client.4540.0:421, at_version=55'20, trim_to=0'0, roll_forward_to=53'19)) v2 ==== 371555+0+0 (unknown 0 0 0) 0x558c57e00000 con 0x558c54c5b400 2019-12-07T19:34:32.678+0000 7f8f23c56700 15 osd.7 55 enqueue_op 0x558c581cd340 prio 127 cost 0 latency 0.000033 epoch 55 MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=107, reqid=client.4540.0:421, at_version=55'20, trim_to=0'0, roll_forward_to=53'19)) v2 2019-12-07T19:34:32.678+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=107, reqid=client.4540.0:421, at_version=55'20, trim_to=0'0, roll_forward_to=53'19)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.678+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.678+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=107, reqid=client.4540.0:421, at_version=55'20, trim_to=0'0, roll_forward_to=53'19)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.678+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.678+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=107, reqid=client.4540.0:421, at_version=55'20, trim_to=0'0, roll_forward_to=53'19)) v2) prio 127 cost 0 e55) pg 0x558c5513e000 2019-12-07T19:34:32.678+0000 7f8f07187700 10 osd.7 55 dequeue_op 0x558c581cd340 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=107, reqid=client.4540.0:421, at_version=55'20, trim_to=0'0, roll_forward_to=53'19)) v2 pg pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.678+0000 7f8f07187700 10 osd.7 55 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 54 -> 55 (as per caller) 2019-12-07T19:34:32.678+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=107, reqid=client.4540.0:421, at_version=55'20, trim_to=0'0, roll_forward_to=53'19)) v2 2019-12-07T19:34:32.678+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 53'19 (0'0,53'19] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,53'19], crt=53'19) [55'20 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:421 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0] 2019-12-07T19:34:32.678+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=53'19 crt=53'19 lcod 53'19 active mbc={} ps=[e~1,13~1,19~2]] add_log_entry 55'20 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:421 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.678+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=53'19 crt=53'19 lcod 53'19 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 20 2019-12-07T19:34:32.678+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=53'19 crt=53'19 lcod 53'19 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.678+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.678+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 55'20, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.678+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c58523800 seq 65 2019-12-07T19:34:32.678+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1265 2019-12-07T19:34:32.678+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# = 0 2019-12-07T19:34:32.678+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 23 keys 2019-12-07T19:34:32.678+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 23 keys = 0 2019-12-07T19:34:32.678+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:32.678+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:32.678+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 0x0~4d000 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:c87708e9:::smithi13140796-17:head# 0x0~4d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:32.678+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~4d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58523800 39 blobs 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:32.678+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x4d000/1000,4d000,0 2019-12-07T19:34:32.678+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb4c000~4d000/1000,4d000,0 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb4c000~4d000] 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57c420e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579d76c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57c420e0 blob([0xb4c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579d76c0 sbid 0x0)) 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore.blob(0x558c57c420e0) get_ref 0x0~2000 Blob(0x558c57c420e0 blob([0xb4c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579d76c0 sbid 0x0)) 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore.blob(0x558c57c420e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.678+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57c420e0 blob([0xb4c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579d76c0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579d76d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d110 blob([0xb4e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c5829d110) get_ref 0x0~2000 Blob(0x558c5829d110 blob([0xb4e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c5829d110) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c5829d110 blob([0xb4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5809a948 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5898b5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2e00 blob([0xb50000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5898b5e0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c574d2e00) get_ref 0x0~2000 Blob(0x558c574d2e00 blob([0xb50000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5898b5e0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c574d2e00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c574d2e00 blob([0xb50000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5898b5e0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5898b5f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57396620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829c690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57396620 blob([0xb52000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c690 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57396620) get_ref 0x0~2000 Blob(0x558c57396620 blob([0xb52000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c690 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57396620) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c57396620 blob([0xb52000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829c690 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5829c6a8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794d2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829c620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794d2d0 blob([0xb54000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c620 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c5794d2d0) get_ref 0x0~2000 Blob(0x558c5794d2d0 blob([0xb54000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c620 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c5794d2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5794d2d0 blob([0xb54000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829c620 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5829c638 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829dc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de1c0 blob([0xb56000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829dc00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578de1c0) get_ref 0x0~2000 Blob(0x558c578de1c0 blob([0xb56000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829dc00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578de1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c578de1c0 blob([0xb56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829dc00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5829dc18 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971490 blob([0xb58000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971490) get_ref 0x0~2000 Blob(0x558c57971490 blob([0xb58000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1c00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971490) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c57971490 blob([0xb58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1c00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c578f1c18 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579708c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c421c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579708c0 blob([0xb5a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c421c0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579708c0) get_ref 0x0~2000 Blob(0x558c579708c0 blob([0xb5a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c421c0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579708c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c579708c0 blob([0xb5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c421c0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57c421d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579713b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac2a0 blob([0xb5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579713b0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c576ac2a0) get_ref 0x0~2000 Blob(0x558c576ac2a0 blob([0xb5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579713b0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c576ac2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c576ac2a0 blob([0xb5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579713b0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579713c8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55136cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55136cb0 blob([0xb5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c55136cb0) get_ref 0x0~2000 Blob(0x558c55136cb0 blob([0xb5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c55136cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c55136cb0 blob([0xb5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970a10 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970a28 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579709a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971500 blob([0xb60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579709a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971500) get_ref 0x0~2000 Blob(0x558c57971500 blob([0xb60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579709a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971500) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c57971500 blob([0xb60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579709a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579709b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55136e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55136e00 blob([0xb62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42070 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c55136e00) get_ref 0x0~2000 Blob(0x558c55136e00 blob([0xb62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42070 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c55136e00) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c55136e00 blob([0xb62000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42070 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57c42088 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac230 blob([0xb64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971420 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c576ac230) get_ref 0x0~2000 Blob(0x558c576ac230 blob([0xb64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971420 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c576ac230) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c576ac230 blob([0xb64000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971420 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971438 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971110 blob([0xb66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971110) get_ref 0x0~2000 Blob(0x558c57971110 blob([0xb66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971110) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57971110 blob([0xb66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad7a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ad7b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54677260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54677260 blob([0xb68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac380 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c54677260) get_ref 0x0~2000 Blob(0x558c54677260 blob([0xb68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac380 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c54677260) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c54677260 blob([0xb68000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac380 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac398 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57dcb3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57dcb3b0 blob([0xb6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57dcb3b0) get_ref 0x0~2000 Blob(0x558c57dcb3b0 blob([0xb6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57dcb3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57dcb3b0 blob([0xb6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac310 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac328 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579716c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579716c0 blob([0xb6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818e70 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579716c0) get_ref 0x0~2000 Blob(0x558c579716c0 blob([0xb6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818e70 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579716c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c579716c0 blob([0xb6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818e70 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57818e88 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b27e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b27e0 blob([0xb6e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579b27e0) get_ref 0x0~2000 Blob(0x558c579b27e0 blob([0xb6e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579b27e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c579b27e0 blob([0xb6e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2770 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579b2788 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579fcf50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971650 blob([0xb70000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcf50 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971650) get_ref 0x0~2000 Blob(0x558c57971650 blob([0xb70000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579fcf50 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971650) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57971650 blob([0xb70000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579fcf50 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c579fcf68 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b28c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b28c0 blob([0xb72000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579b28c0) get_ref 0x0~2000 Blob(0x558c579b28c0 blob([0xb72000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579b28c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c579b28c0 blob([0xb72000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970948 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579b2850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579b2850 blob([0xb74000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971340 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579b2850) get_ref 0x0~2000 Blob(0x558c579b2850 blob([0xb74000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971340 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579b2850) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c579b2850 blob([0xb74000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971340 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57971358 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579715e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55136d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579715e0 blob([0xb76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136d90 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579715e0) get_ref 0x0~2000 Blob(0x558c579715e0 blob([0xb76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136d90 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579715e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c579715e0 blob([0xb76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55136d90 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55136da8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e42700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573965b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e42700 blob([0xb78000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c54e42700) get_ref 0x0~2000 Blob(0x558c54e42700 blob([0xb78000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c54e42700) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c54e42700 blob([0xb78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573965b0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c573965c8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55136d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971570 blob([0xb7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136d20 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971570) get_ref 0x0~2000 Blob(0x558c57971570 blob([0xb7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55136d20 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971570) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57971570 blob([0xb7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55136d20 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c55136d38 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579707e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579707e0 blob([0xb7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579707e0) get_ref 0x0~2000 Blob(0x558c579707e0 blob([0xb7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579707e0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c579707e0 blob([0xb7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970850 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57970868 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578de380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578de380 blob([0xb7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a770 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578de380) get_ref 0x0~2000 Blob(0x558c578de380 blob([0xb7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a770 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578de380) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c578de380 blob([0xb7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a770 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5809a788 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c38930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971260 blob([0xb80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971260) get_ref 0x0~2000 Blob(0x558c57971260 blob([0xb80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c38930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57971260) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c57971260 blob([0xb80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38930 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57c38948 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ac1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ac1c0 blob([0xb82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c576ac1c0) get_ref 0x0~2000 Blob(0x558c576ac1c0 blob([0xb82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c576ac1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c576ac1c0 blob([0xb82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac460 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c576ac478 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579711f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794d650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579711f0 blob([0xb84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d650 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579711f0) get_ref 0x0~2000 Blob(0x558c579711f0 blob([0xb84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794d650 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c579711f0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c579711f0 blob([0xb84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d650 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c5794d668 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819960 blob([0xb86000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57819960) get_ref 0x0~2000 Blob(0x558c57819960 blob([0xb86000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57819960) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c57819960 blob([0xb86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0a10 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c578f0a28 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818af0 blob([0xb88000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818af0) get_ref 0x0~2000 Blob(0x558c57818af0 blob([0xb88000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818af0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c57818af0 blob([0xb88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818b60 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57818b78 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578198f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f09a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578198f0 blob([0xb8a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f09a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578198f0) get_ref 0x0~2000 Blob(0x558c578198f0 blob([0xb8a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f09a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578198f0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c578198f0 blob([0xb8a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f09a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c578f09b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818a10 blob([0xb8c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818a10) get_ref 0x0~2000 Blob(0x558c57818a10 blob([0xb8c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818a10) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c57818a10 blob([0xb8c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818a80 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57818a98 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f50e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819880 blob([0xb8e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50e70 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57819880) get_ref 0x0~2000 Blob(0x558c57819880 blob([0xb8e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f50e70 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57819880) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c57819880 blob([0xb8e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f50e70 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c57f50e88 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578189a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818930 blob([0xb90000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818930) get_ref 0x0~2000 Blob(0x558c57818930 blob([0xb90000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818930) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c57818930 blob([0xb90000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578189a0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c578189b8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57819810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578f15e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57819810 blob([0xb92000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f15e0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57819810) get_ref 0x0~2000 Blob(0x558c57819810 blob([0xb92000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f15e0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57819810) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c57819810 blob([0xb92000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f15e0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c578f15f8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57818850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c578188c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57818850 blob([0xb94000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818850) get_ref 0x0~2000 Blob(0x558c57818850 blob([0xb94000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c57818850) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c57818850 blob([0xb94000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578188c0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c578188d8 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578197a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f4e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578197a0 blob([0xb96000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f4e00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578197a0) get_ref 0x0~2000 Blob(0x558c578197a0 blob([0xb96000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f4e00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c578197a0) get_ref init 0x2000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c578197a0 blob([0xb96000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580f4e00 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c580f4e18 in 0x558c53a596c0) _discard 0x0~2000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580f4e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f4ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580f4e70 blob([0xb98000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f4ee0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c580f4e70) get_ref 0x0~1000 Blob(0x558c580f4e70 blob([0xb98000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f4ee0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.blob(0x558c580f4e70) get_ref init 0x1000, 1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~1000: 0x0~1000 Blob(0x558c580f4e70 blob([0xb98000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580f4ee0 sbid 0x0)) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c580f4ef8 in 0x558c53a596c0) _discard 0x0~1000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4d000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e45288) dirty_range mark inline shard dirty 2019-12-07T19:34:32.679+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# 0x0~4d000 = 0 2019-12-07T19:34:32.679+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) 2019-12-07T19:34:32.679+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.3s4_head 4#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:32.679+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.0000000055.00000000000000000020' <- 0000000055.00000000000000000020 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.679+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:32.679+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58523800 cost 532013 (40 ios * 4000 + 372013 bytes) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58523800 onodes 0x558c54e45180 shared_blobs 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e45288) update 4#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e45288) update inline shard 741 bytes from 39 extents 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:c87708e9:::smithi13140796-17:head# is 55137 (54390 bytes onode + 2 bytes spanning blobs + 745 bytes inline extents) 2019-12-07T19:34:32.679+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58523800 allocated 0x[b4c000~4d000] released 0x[] 2019-12-07T19:34:32.679+0000 7f8f07187700 10 freelist allocate 0xb4c000~4d000 2019-12-07T19:34:32.679+0000 7f8f07187700 20 freelist _xor first_key 0xb00000 last_key 0xb80000 2019-12-07T19:34:32.679+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523800 prepare 2019-12-07T19:34:32.679+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58523800 2019-12-07T19:34:32.680+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 lua=53'19 crt=53'19 lcod 53'19 active mbc={} ps=[e~1,13~1,19~2]] op_applied version 55'20 2019-12-07T19:34:32.680+0000 7f8f07187700 10 osd.7 55 dequeue_op 0x558c581cd340 finish 2019-12-07T19:34:32.680+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:32.680+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523800 aio_wait 2019-12-07T19:34:32.680+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58523800 2019-12-07T19:34:32.680+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523800 io_done 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 25568 < min_flush_size 524288 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 56918 < min_flush_size 524288 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e45180 had 1 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 56941 < min_flush_size 524288 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 56941 < min_flush_size 524288 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3a1c20~de6d to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3a1c20 0x3a1c20~de6d to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~380000 x_off 0x371c20 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc20 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa8d and padding block with 0x573 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3afa8d 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.680+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000225400s (0.000035953s flush + 0.000189447s kv commit) 2019-12-07T19:34:32.680+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58523800> 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523800 kv_submitted 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58523800 2019-12-07T19:34:32.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523800 finishing 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58523800 onodes 0x558c54e45180 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55136d38 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fdd0 space 0x558c55136d38 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55136da8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c17ef0 space 0x558c55136da8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573965c8 in 0x558c53a596c0) _finish_write discard buffer(0x558c5813b560 space 0x558c573965c8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac328 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84510 space 0x558c576ac328 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac398 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b845a0 space 0x558c576ac398 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac478 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fef0 space 0x558c576ac478 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad7b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84630 space 0x558c576ad7b8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578188d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8f8c0 space 0x558c578188d8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578189b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fcb0 space 0x558c578189b8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818a98 in 0x558c53a596c0) _finish_write discard buffer(0x558c585ea090 space 0x558c57818a98 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818b78 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fb00 space 0x558c57818b78 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818e88 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84480 space 0x558c57818e88 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f09b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c585ea120 space 0x558c578f09b8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f0a28 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fd40 space 0x558c578f0a28 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f15f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fa70 space 0x558c578f15f8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578f1c18 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84a20 space 0x558c578f1c18 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794d668 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c17e60 space 0x558c5794d668 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970868 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fe60 space 0x558c57970868 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970948 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8f950 space 0x558c57970948 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579709b8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b847e0 space 0x558c579709b8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 55/41 ECSubWriteReply(tid=107, last_complete=55'20, committed=1, applied=1)) v2 -- 0x558c54d76000 con 0x558c54c5b400 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970a28 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84870 space 0x558c57970a28 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971358 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8f7a0 space 0x558c57971358 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579713c8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84900 space 0x558c579713c8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971438 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b846c0 space 0x558c57971438 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2788 in 0x558c53a596c0) _finish_write discard buffer(0x558c57f779e0 space 0x558c579b2788 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579d76d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84d80 space 0x558c579d76d8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579fcf68 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8f9e0 space 0x558c579fcf68 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c38948 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fc20 space 0x558c57c38948 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42088 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84750 space 0x558c57c42088 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c421d8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84990 space 0x558c57c421d8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f50e88 in 0x558c53a596c0) _finish_write discard buffer(0x558c585ea000 space 0x558c57f50e88 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a788 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c8fb90 space 0x558c5809a788 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a948 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84cf0 space 0x558c5809a948 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f4e18 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c173b0 space 0x558c580f4e18 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f4ef8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58c17200 space 0x558c580f4ef8 0x0~1000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829c638 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84b40 space 0x558c5829c638 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829c6a8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84bd0 space 0x558c5829c6a8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829dc18 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84ab0 space 0x558c5829dc18 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5898b5f8 in 0x558c53a596c0) _finish_write discard buffer(0x558c58b84c60 space 0x558c5898b5f8 0x0~2000 writing nocache) 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58523800 done 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:32.680+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58523800 [] 2019-12-07T19:34:32.680+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.680+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.682+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 287 ==== MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=108, reqid=client.4540.0:421, at_version=0'0, trim_to=0'0, roll_forward_to=55'20)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54e41800 con 0x558c54c5b400 2019-12-07T19:34:32.682+0000 7f8f23c56700 15 osd.7 55 enqueue_op 0x558c54e798c0 prio 127 cost 0 latency 0.000022 epoch 55 MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=108, reqid=client.4540.0:421, at_version=0'0, trim_to=0'0, roll_forward_to=55'20)) v2 2019-12-07T19:34:32.682+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=108, reqid=client.4540.0:421, at_version=0'0, trim_to=0'0, roll_forward_to=55'20)) v2) prio 127 cost 0 e55) 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=108, reqid=client.4540.0:421, at_version=0'0, trim_to=0'0, roll_forward_to=55'20)) v2) prio 127 cost 0 e55) queued 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGOpItem(op=MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=108, reqid=client.4540.0:421, at_version=0'0, trim_to=0'0, roll_forward_to=55'20)) v2) prio 127 cost 0 e55) pg 0x558c5513e000 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 osd.7 55 dequeue_op 0x558c54e798c0 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=108, reqid=client.4540.0:421, at_version=0'0, trim_to=0'0, roll_forward_to=55'20)) v2 pg pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] _handle_message: MOSDECSubOpWrite(1.3s4 55/41 ECSubWrite(tid=108, reqid=client.4540.0:421, at_version=0'0, trim_to=0'0, roll_forward_to=55'20)) v2 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=53'19 active mbc={} ps=[e~1,13~1,19~2]] append_log log((0'0,55'20], crt=53'19) [] 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] rollforward: entry=55'20 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:421 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] append_log approx pg log length = 20 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] append_log transaction_applied = 1 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:32.682+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c5810f800 seq 66 2019-12-07T19:34:32.682+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810f800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810f800 onodes shared_blobs 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810f800 allocated 0x[] released 0x[] 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f800 prepare 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810f800 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f800 io_done 2019-12-07T19:34:32.682+0000 7f8f0317f700 10 osd.7 55 dequeue_op 0x558c54e798c0 finish 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3afa8d~471 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3afa8d 0x3afa8d~471 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~380000 x_off 0x37fa8d 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa8d 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xefe and padding block with 0x102 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3afefe 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:32.682+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119852s (0.000000124s flush + 0.000119728s kv commit) 2019-12-07T19:34:32.682+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5810f800> 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:32.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f800 kv_submitted 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810f800 2019-12-07T19:34:32.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810f800 finishing 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810f800 onodes 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810f800 done 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:32.682+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810f800 [] 2019-12-07T19:34:32.682+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:32.682+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:32.682+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 55/41 ECSubWriteReply(tid=108, last_complete=55'20, committed=1, applied=1)) v2 -- 0x558c58513e00 con 0x558c54c5b400 2019-12-07T19:34:32.682+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:32.682+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:32.687+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 10 osd.7 55 tick_without_osd_lock 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f467000/0x40000000/0x1680000000, data 0xc77033/0xb97000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:32.576231+0000 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576218+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576154+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 20 osd.7 55 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 20 osd.7 55 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 20 osd.7 55 scrub_load_below_threshold loadavg per cpu 0.0375 < max 5 = yes 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 20 osd.7 55 sched_scrub load_is_low=1 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 10 osd.7 55 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:08.239446+0000 > 2019-12-07T19:34:32.734647+0000 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 20 osd.7 55 sched_scrub done 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 10 osd.7 55 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 20 osd.7 55 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:32.733+0000 7f8f1fbbe700 10 osd.7 55 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:32.737+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.788+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.838+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.888+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.938+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:32.988+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.038+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.047+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:33.062+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 12 ==== osd_ping(ping e55 up_from 13 ping_stamp 2019-12-07T19:34:33.063473+0000/56.512699559s send_stamp 56.512699559s delta_ub 0.014002835s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c54c65400 2019-12-07T19:34:33.062+0000 7f8f24c58700 20 osd.7 55 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013586089s,0.014002835s]) 2019-12-07T19:34:33.062+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e55 up_from 13 ping_stamp 2019-12-07T19:34:33.063473+0000/56.512699559s send_stamp 56.499113470s delta_ub 18446744073.695965527s) v5 -- 0x558c580b9e00 con 0x558c54c65400 2019-12-07T19:34:33.062+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 12 ==== osd_ping(ping e55 up_from 13 ping_stamp 2019-12-07T19:34:33.063473+0000/56.512699559s send_stamp 56.512699559s delta_ub 0.014002835s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c54c65000 2019-12-07T19:34:33.062+0000 7f8f24c58700 20 osd.7 55 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013525666s,0.014002835s]) 2019-12-07T19:34:33.062+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e55 up_from 13 ping_stamp 2019-12-07T19:34:33.063473+0000/56.512699559s send_stamp 56.499173893s delta_ub 18446744073.696025950s) v5 -- 0x558c580b9e00 con 0x558c54c65000 2019-12-07T19:34:33.088+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.138+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.188+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.238+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.288+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.318+0000 7f8f21451700 10 osd.7 55 tick 2019-12-07T19:34:33.318+0000 7f8f21451700 10 osd.7 55 do_waiters -- start 2019-12-07T19:34:33.318+0000 7f8f21451700 10 osd.7 55 do_waiters -- finish 2019-12-07T19:34:33.318+0000 7f8f21451700 20 osd.7 55 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:33.338+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.350+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:33.388+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.438+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.488+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.539+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.563+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 13 ==== osd_ping(ping e55 up_from 13 ping_stamp 2019-12-07T19:34:33.563824+0000/57.013050243s send_stamp 57.013050243s delta_ub 0.013845196s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c54c65400 2019-12-07T19:34:33.563+0000 7f8f24c58700 20 osd.7 55 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013641837s,0.013845196s]) 2019-12-07T19:34:33.563+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e55 up_from 13 ping_stamp 2019-12-07T19:34:33.563824+0000/57.013050243s send_stamp 56.999408406s delta_ub 18446744073.695909779s) v5 -- 0x558c580b9e00 con 0x558c54c65400 2019-12-07T19:34:33.563+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 13 ==== osd_ping(ping e55 up_from 13 ping_stamp 2019-12-07T19:34:33.563824+0000/57.013050243s send_stamp 57.013050243s delta_ub 0.013845196s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c54c65000 2019-12-07T19:34:33.563+0000 7f8f24c58700 20 osd.7 55 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013595194s,0.013845196s]) 2019-12-07T19:34:33.563+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e55 up_from 13 ping_stamp 2019-12-07T19:34:33.563824+0000/57.013050243s send_stamp 56.999455049s delta_ub 18446744073.695956422s) v5 -- 0x558c580b9e00 con 0x558c54c65000 2019-12-07T19:34:33.589+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 461 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:33.606+0000 7f8f24457700 15 osd.7 55 enqueue_peering_evt 1.8s5 epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:33.606+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e56) pg 0x558c57380000 2019-12-07T19:34:33.606+0000 7f8f0898a700 20 osd.7 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 56, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:33.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 271 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:33.606+0000 7f8f23c56700 15 osd.7 55 enqueue_peering_evt 1.1s1 epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:33.606+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 272 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:33.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e56) pg 0x558c5736a000 2019-12-07T19:34:33.606+0000 7f8f08189700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 56, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:33.606+0000 7f8f23c56700 15 osd.7 55 enqueue_peering_evt 1.6s2 epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:33.606+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e56) pg 0x558c54d5c000 2019-12-07T19:34:33.606+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.6s2 peering, item epoch is 56, will wait on OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 273 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57378700 con 0x558c54c5bc00 2019-12-07T19:34:33.606+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:33.606+0000 7f8f23c56700 15 osd.7 55 enqueue_peering_evt 1.fs5 epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:33.606+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 462 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:33.606+0000 7f8f24457700 15 osd.7 55 enqueue_peering_evt 1.0s3 epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:33.606+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e56) pg 0x558c57383000 2019-12-07T19:34:33.606+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 56, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:33.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 463 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:33.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e56) pg 0x558c5500d000 2019-12-07T19:34:33.606+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 56, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:33.606+0000 7f8f24457700 15 osd.7 55 enqueue_peering_evt 1.es4 epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:33.606+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e56) pg 0x558c54d98000 2019-12-07T19:34:33.606+0000 7f8f0197c700 20 osd.7 op_wq _add_slot_waiter 1.es4 peering, item epoch is 56, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:33.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 464 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:33.606+0000 7f8f24457700 15 osd.7 55 enqueue_peering_evt 1.7s4 epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:33.606+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 465 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:33.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e56) pg 0x558c54d5f000 2019-12-07T19:34:33.606+0000 7f8f05183700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 56, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:33.606+0000 7f8f24457700 15 osd.7 55 enqueue_peering_evt 1.9s4 epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:33.606+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e56) pg 0x558c5500a000 2019-12-07T19:34:33.606+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 56, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:33.607+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 466 ==== pg_info2(1.as5 1.as5 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c574e6a00 con 0x558c54c64000 2019-12-07T19:34:33.607+0000 7f8f24457700 15 osd.7 55 enqueue_peering_evt 1.as5 epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:33.607+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e56) pg 0x558c57e8b000 2019-12-07T19:34:33.607+0000 7f8f07988700 20 osd.7 op_wq _add_slot_waiter 1.as5 peering, item epoch is 56, will wait on OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.607+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:33.608+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 288 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:33.608+0000 7f8f23c56700 15 osd.7 55 enqueue_peering_evt 1.bs1 epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:33.608+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e56) pg 0x558c575ef000 2019-12-07T19:34:33.608+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 56, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.608+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 289 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c57378700 con 0x558c54c5b400 2019-12-07T19:34:33.608+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:33.608+0000 7f8f23c56700 15 osd.7 55 enqueue_peering_evt 1.3s4 epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:33.608+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e56) pg 0x558c5513e000 2019-12-07T19:34:33.608+0000 7f8f0317f700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 56, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.608+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:33.614+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 189 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c19800 2019-12-07T19:34:33.614+0000 7f8f24c58700 15 osd.7 55 enqueue_peering_evt 1.ds2 epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:33.614+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e56) pg 0x558c55141000 2019-12-07T19:34:33.614+0000 7f8f0217d700 20 osd.7 op_wq _add_slot_waiter 1.ds2 peering, item epoch is 56, will wait on OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.614+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:33.614+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 190 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57378000 con 0x558c54c19800 2019-12-07T19:34:33.614+0000 7f8f24c58700 15 osd.7 55 enqueue_peering_evt 1.5s4 epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:33.614+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e56) pg 0x558c57522000 2019-12-07T19:34:33.614+0000 7f8f06185700 20 osd.7 op_wq _add_slot_waiter 1.5s4 peering, item epoch is 56, will wait on OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e56) 2019-12-07T19:34:33.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:33.639+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.690+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:34:33.690+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:33.690+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 44 0x2000~1000 2019-12-07T19:34:33.690+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc e2299622 2019-12-07T19:34:33.690+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 44 0x1b000~1000 2019-12-07T19:34:33.690+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc 618bf9df 2019-12-07T19:34:33.690+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1377268 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.690+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:33.690+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 10 osd.7 55 tick_without_osd_lock 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f467000/0x40000000/0x1680000000, data 0xc77033/0xb97000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:32.576231+0000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576218+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576154+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 25 osd.7 55 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 osd.7 55 tick_without_osd_lock max_waiting_epoch 56, requesting new map 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=56}) v3 -- 0x558c580b9a00 con 0x558c5476d000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 osd.7 55 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 osd.7 55 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 osd.7 55 scrub_load_below_threshold loadavg per cpu 0.0375 < max 5 = yes 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 osd.7 55 sched_scrub load_is_low=1 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 10 osd.7 55 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:08.239446+0000 > 2019-12-07T19:34:33.729609+0000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 osd.7 55 sched_scrub done 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 10 osd.7 55 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 20 osd.7 55 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:33.729+0000 7f8f1fbbe700 10 osd.7 55 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 31 ==== osd_map(56..56 src has 1..56) v4 ==== 677+0+0 (crc 0 0 0) 0x558c579d9440 con 0x558c5476d000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 osd.7 55 OSD::ms_dispatch: osd_map(56..56 src has 1..56) v4 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 osd.7 55 do_waiters -- start 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 osd.7 55 do_waiters -- finish 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 osd.7 55 _dispatch 0x558c579d9440 osd_map(56..56 src has 1..56) v4 2019-12-07T19:34:33.729+0000 7f8f164bb700 3 osd.7 55 handle_osd_map epochs [56,56], i have 55, src has [1,56] 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 osd.7 55 handle_osd_map got inc map for epoch 56 2019-12-07T19:34:33.729+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:35f4e826:::osdmap.55:0# 0x0~0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c578805b0 blob([0x33b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5794c1c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57396460 blob([!~1000,0x33c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c5809ac40 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:35f4e826:::osdmap.55:0# 0x0~1448 = 5192 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 osd.7 55 add_map_bl 55 5192 bytes 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 osd.7 55 got_full_map 56, nothing requested 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 osd.7 55 handle_osd_map pg_num_history pg_num_history(e56 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {56={1=[13~1,19~1]}} 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) lower_bound to PSN__1_0000000000000012 key 0x00000000000000000000000000000403'.PSN__1_0000000000000012' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000017' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 12 found [17,18), no overlap 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) lower_bound to PSN__1_0000000000000014 key 0x00000000000000000000000000000403'.PSN__1_0000000000000014' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000017' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 14 found [17,18), no overlap 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) lower_bound to PSN__1_0000000000000018 key 0x00000000000000000000000000000403'.PSN__1_0000000000000018' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 18 key 'PSN__1_0000000000000018' lower_bound not found 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) lower_bound to PSN__1_000000000000001a key 0x00000000000000000000000000000403'.PSN__1_000000000000001a' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b500) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 1a key 'PSN__1_000000000000001a' lower_bound not found 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 2 keys 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 osd.7 55 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e56 [1,56] lci=[13,56]) 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c582b6600 seq 47 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c667972f:::inc_osdmap.56:0# key 0x7f7fffffffffffffffc66797'/!inc_osdmap.56!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c667972f:::inc_osdmap.56:0# 0x0~279 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1266 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c667972f:::inc_osdmap.56:0# 0x0~279 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~279 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd87 on front/back, now 0x0~1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6600 1 blobs 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb99000~1000/1000,1000,0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb99000~1000] 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829c850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5742bea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829c850 blob([0xb99000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bea0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c5829c850) get_ref 0x0~279 Blob(0x558c5829c850 blob([0xb99000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5742bea0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c5829c850) get_ref init 0x1000, 1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~279: 0x0~279 Blob(0x558c5829c850 blob([0xb99000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x279) SharedBlob(0x558c5742bea0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5742beb8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x279 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b508) dirty_range mark inline shard dirty 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c667972f:::inc_osdmap.56:0# 0x0~279 = 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3ef4e826:::osdmap.56:0# key 0x7f7fffffffffffffff3ef4e8'&!osdmap.56!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ef4e826:::osdmap.56:0# 0x0~1464 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1267 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3ef4e826:::osdmap.56:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6600 2 blobs 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb9a000~2000/1000,2000,0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb9a000~2000] 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b2380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829c1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b2380 blob([0xb9a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c1c0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c550b2380) get_ref 0x0~1000 Blob(0x558c550b2380 blob([0xb9a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c1c0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c550b2380) get_ref init 0x1000, 1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c550b2380 blob([0xb9a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5829c1c0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5829c1d8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efce00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57efce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efce00 blob([!~1000,0xb9b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efce70 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c57efce00) get_ref 0x1000~464 Blob(0x558c57efce00 blob([!~1000,0xb9b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57efce70 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c57efce00) get_ref init 0x2000, 1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c57efce00 blob([!~1000,0xb9b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57efce70 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c57efce88 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b788) dirty_range mark inline shard dirty 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3ef4e826:::osdmap.56:0# 0x0~1464 = 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6600 0 blobs 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000013' <- PSN__1_0000000000000013 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000019' <- PSN__1_0000000000000019 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6600 0 blobs 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b6600 cost 23444 (4 ios * 4000 + 7444 bytes) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b6600 onodes 0x558c53a91900,0x558c55276c80,0x558c5795b400,0x558c5795b680 shared_blobs 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b508) update #-1:c667972f:::inc_osdmap.56:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b508) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c667972f:::inc_osdmap.56:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b788) update #-1:3ef4e826:::osdmap.56:0# 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795b788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3ef4e826:::osdmap.56:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b6600 allocated 0x[b99000~3000] released 0x[] 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 freelist allocate 0xb99000~3000 2019-12-07T19:34:33.729+0000 7f8f164bb700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6600 prepare 2019-12-07T19:34:33.729+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c582b6600 2019-12-07T19:34:33.729+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6600 aio_wait 2019-12-07T19:34:33.729+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b6600 2019-12-07T19:34:33.729+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6600 io_done 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 258 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8938 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795b400 had 1 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795b680 had 1 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3856990208 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997796) bluestore 86 GiB free (0.959874), bluefs_ratio 0.0399208 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8985 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8985 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3afefe~2319 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3afefe 0x3afefe~2319 to file(ino 16 size 0x3b0000 mtime 2019-12-07T19:34:32.661123+0000 allocated 3b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~380000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _allocate len 0x2217 from 0 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _allocate len 0x2217 from 1 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd4e0000 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd4e0000~10000/10000,10000,acd4e0000 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x3c0000 to include allocated 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 69 (was clean) 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x37fefe 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xefe 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x217 and padding block with 0xde9 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x3c0000 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b2217 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (69) on file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]), flushing log 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 69 len 0x4a crc 0x8e959c29) 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x44000~1000 to file(ino 1 size 0x44000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x44000 0x44000~1000 to file(ino 1 size 0x44000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x45000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x44000 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x45000 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000019 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 69 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000416075s (0.000043337s flush + 0.000372738s kv commit) 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b6600> 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c58141400> 2019-12-07T19:34:33.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6600 kv_submitted 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b6600 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c582b6600 osr 0x558c54ea4b40 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7 55 _committed_osd_maps 56..56 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7 55 advance to epoch 56 (<= last 56 <= newest_map 56) 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 45 0x1b000~1000 crc f368d8c4 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 45 0x2000~1000 crc 6396d5f 2019-12-07T19:34:33.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6300 deferred_cleanup 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b6300 onodes 0x558c53a91900,0x558c55276c80,0x558c5795af00,0x558c5795b180 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de558 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2c60 space 0x558c578de558 0x0~1000 writing nocache) 2019-12-07T19:34:33.729+0000 7f8f141a1700 7 osd.7 56 consume_map version 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.8s5 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.0s3 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a2788 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2cf0 space 0x558c580a2788 0x0~1000 writing nocache) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.1s1 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.9s4 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.as5 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec248 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2d80 space 0x558c580ec248 0x1000~1000 writing nocache) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.2s0 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.bs1 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.3s4 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.cs0 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.4s0 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6300 done 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.5s4 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d3200 deferred_cleanup 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.ds2 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b6300 [] 2019-12-07T19:34:33.729+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.es4 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.6s2 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.fs5 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 identify_splits_and_merges 1.7s4 e55 to e56 pg_nums {18=16} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 56 prune_pg_created 2019-12-07T19:34:33.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d3200 deferred_cleanup 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:0.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d3200 onodes 0x558c53a91900,0x558c55276c80,0x558c5851e780,0x558c5851ea00 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c1d8 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8f050 space 0x558c5794c1d8 0x0~1000 writing nocache) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809ac58 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8f0e0 space 0x558c5809ac58 0x1000~1000 writing nocache) 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809b748 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8efc0 space 0x558c5809b748 0x0~1000 writing nocache) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d3200 done 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6600 deferred_queued 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:33.729+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d3200 [] 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:1.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:2.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.as5 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e56) pg 0x558c5500d000 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:3.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e56) pg 0x558c5500a000 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:4.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:5.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e56) pg 0x558c57e8b000 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.5s4 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:5._wake_pg_slot _wake_pg_slot 1.ds2 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:6.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e56) pg 0x558c5513e000 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.6s2 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f0898a700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7:7.consume_map 56 (was 55) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:33.729+0000 7f8f0898a700 10 osd.7 pg_epoch: 55 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 56 <= 56, requeueing 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {56=} 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e56) pg 0x558c54d5c000 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e56) pg 0x558c55141000 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e56) pg 0x558c54d5f000 2019-12-07T19:34:33.729+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f141a1700 10 osd.7 56 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.8s5 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.0s3 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.1s1 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.9s4 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.as5 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.2s0 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.729+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:33.729+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.bs1 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 55 while current is 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.3s4 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.cs0 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.9s0( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.as0( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.4s0 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 55 while current is 56 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:33.729+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.5s4 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:33.729+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.ds2 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.es4 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.729+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:33.729+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c58113b00 seq 82 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.6s2 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:33.729+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.729+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.fs5 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:33.729+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c585f6900 seq 25 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.6s0( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:33.729+0000 7f8f141a1700 15 osd.7 56 enqueue_peering_evt 1.7s4 epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c57676000 2019-12-07T19:34:33.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c58523b00 seq 67 2019-12-07T19:34:33.729+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:33.729+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:33.729+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.729+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) 2019-12-07T19:34:33.729+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.729+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:33.729+0000 7f8f141a1700 7 osd.7 56 activate_map version 56 2019-12-07T19:34:33.729+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:33.729+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:33.729+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.729+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:33.729+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:33.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:33.729+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58113b00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.729+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:33.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:33.729+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58113b00 onodes shared_blobs 2019-12-07T19:34:33.729+0000 7f8f0297e700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:33.729+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:33.729+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58113b00 allocated 0x[] released 0x[] 2019-12-07T19:34:33.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:33.729+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585f6900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.729+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:33.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58523b00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.729+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585f6900 onodes shared_blobs 2019-12-07T19:34:33.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58523b00 onodes shared_blobs 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c575c4f00 seq 106 2019-12-07T19:34:33.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58523b00 allocated 0x[] released 0x[] 2019-12-07T19:34:33.729+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585f6900 allocated 0x[] released 0x[] 2019-12-07T19:34:33.729+0000 7f8f0297e700 10 osd.7 pg_epoch: 55 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:33.729+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113b00 prepare 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58113b00 2019-12-07T19:34:33.729+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523b00 prepare 2019-12-07T19:34:33.729+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58523b00 2019-12-07T19:34:33.729+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523b00 io_done 2019-12-07T19:34:33.729+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 prepare 2019-12-07T19:34:33.729+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:33.729+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585f6900 2019-12-07T19:34:33.729+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 io_done 2019-12-07T19:34:33.729+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113b00 io_done 2019-12-07T19:34:33.729+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:33.729+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,13~1,19~2]] state: Active advmap 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.729+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.729+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c4f00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e56) pg 0x558c575ef000 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c4f00 onodes shared_blobs 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c4f00 allocated 0x[] released 0x[] 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e56) pg 0x558c5736a000 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 55 while current is 56 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f0918b700 20 osd.7 56 agent_entry empty queue 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4f00 prepare 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c57e8b000 2019-12-07T19:34:33.729+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c4f00 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:33.729+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4f00 io_done 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.ds0( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.729+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c57e88000 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3402 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e56) pg 0x558c54d98000 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:33.729+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:33.729+0000 7f8f0117b700 20 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 55 while current is 56 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58541200 seq 63 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3425 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3425 < min_flush_size 524288 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 55 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:33.729+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.7s0( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 55 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b2217~d61 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:33.729+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b2217 0x3b2217~d61 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58541200 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x382217 2019-12-07T19:34:33.729+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 55 -> 56 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58541200 onodes shared_blobs 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x217 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58541200 allocated 0x[] released 0x[] 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 55 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:33.729+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf78 and padding block with 0x88 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:33.729+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 55 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541200 prepare 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58541200 2019-12-07T19:34:33.729+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541200 io_done 2019-12-07T19:34:33.729+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,13~1,19~2]] state: Active advmap 2019-12-07T19:34:33.729+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] on_active_advmap new purged_snaps [13~1,19~1], now [e~1,1a~1] 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] share_pg_info 2019-12-07T19:34:33.729+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e56) pg 0x558c57522000 2019-12-07T19:34:33.729+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.729+0000 7f8f08189700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f07187700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.729+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] on_active_advmap new purged_snaps [13~1,19~1], now [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b2f78 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] share_pg_info 2019-12-07T19:34:33.730+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.730+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.729+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:33.730+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 55 -> 56 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c57959800 seq 84 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000119132s (0.000000071s flush + 0.000119061s kv commit) 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 55 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:33.730+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:33.730+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c57379c00 con 0x558c54c64000 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 4(0) info: 1.bs0( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:33.730+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57959800 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57959800 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57959800 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:33.730+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c5737b100 con 0x558c54c19800 2019-12-07T19:34:33.730+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959800 prepare 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57959800 2019-12-07T19:34:33.730+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959800 io_done 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c579d2c00 seq 23 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.730+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:33.730+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c5737b800 con 0x558c54c5bc00 2019-12-07T19:34:33.730+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.730+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b2f78~8f3 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e56) pg 0x558c57383000 2019-12-07T19:34:33.730+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b2f78 0x3b2f78~8f3 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c579d2c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c548a1c00 con 0x558c54c5b400 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x382f78 2019-12-07T19:34:33.730+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c579d2c00 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf78 2019-12-07T19:34:33.730+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c579d2c00 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x86b and padding block with 0x795 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2c00 prepare 2019-12-07T19:34:33.730+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c579d2c00 2019-12-07T19:34:33.730+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2c00 io_done 2019-12-07T19:34:33.730+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c58125800 con 0x558c54c19c00 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.730+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 55 -> 56 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c575c5200 seq 21 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c5736a000 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 55 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:33.730+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f03980700 15 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] publish_stats_to_osd 54:159 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.730+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:33.730+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c58541500 seq 22 2019-12-07T19:34:33.730+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c5200 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c5200 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c5200 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:33.730+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c5500a000 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.730+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5200 prepare 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:33.730+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c5200 2019-12-07T19:34:33.730+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5200 io_done 2019-12-07T19:34:33.730+0000 7f8f08189700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:33.730+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58541500 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58541500 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58541500 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586e0000 seq 47 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541500 prepare 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c54d98000 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58541500 2019-12-07T19:34:33.730+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541500 io_done 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c57522000 2019-12-07T19:34:33.730+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 55 while current is 56 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c54d5c000 2019-12-07T19:34:33.730+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e0000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e0000 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e0000 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 3(0) info: 1.fs0( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c55141000 2019-12-07T19:34:33.730+0000 7f8f05984700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f03980700 7 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:33.730+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0000 prepare 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e0000 2019-12-07T19:34:33.730+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0000 io_done 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f03980700 15 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] publish_stats_to_osd 55:160 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.729+0000 7f8f0898a700 20 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c575ef000 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c57959b00 seq 28 2019-12-07T19:34:33.730+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b386b 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:33.730+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:33.730+0000 7f8f03980700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:33.730+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57959b00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57959b00 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57959b00 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c58260000 seq 29 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c578a6c00 seq 48 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000200194s (0.000000092s flush + 0.000200102s kv commit) 2019-12-07T19:34:33.730+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959b00 prepare 2019-12-07T19:34:33.730+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57959b00 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959b00 io_done 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a6c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e56/56) v1 -- 0x558c58539100 con 0x558c54c64000 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a6c00 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c5513e000 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6c00 prepare 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a6c00 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6c00 io_done 2019-12-07T19:34:33.730+0000 7f8f07187700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3402 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e56) pg 0x558c57380000 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4536 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58523b00,0x558c585f6900,0x558c58113b00,0x558c575c4f00,0x558c58541200> 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523b00 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58523b00 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e56/56) v1 -- 0x558c58539800 con 0x558c54c19800 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5670 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58523b00 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58523b00 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58523b00 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58523b00 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585f6900 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 55 -> 56 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5693 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585f6900 finishing 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5693 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585f6900 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585f6900 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585f6900 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e56/56) v1 -- 0x558c55054300 con 0x558c54c5bc00 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113b00 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58113b00 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 55 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b386b~163d to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58113b00 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58113b00 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58113b00 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:33.730+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58113b00 [] 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b386b 0x3b386b~163d to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] state: Started advmap 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x38386b 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4f00 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x86b 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c4f00 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e56/56) v1 -- 0x558c55054a00 con 0x558c54c5b400 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xea8 and padding block with 0x158 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c4f00 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c4f00 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c4f00 done 2019-12-07T19:34:33.730+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c4f00 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541200 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58541200 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] plpg_on_pool_change 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541200 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58541200 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58541200 done 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e56/56) v1 -- 0x558c55053c00 con 0x558c54c19c00 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58541200 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] hit_set_clear 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:33.730+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] agent_stop 2019-12-07T19:34:33.730+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] activate_map 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] publish_stats_to_osd 54:183 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] do_peering_event: epoch_sent: 56 epoch_requested: 56 MInfoRec from 1(0) info: 1.8s0( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,13~1,19~2]] proc_primary_info updating purged_snaps to [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c578a6f00 seq 49 2019-12-07T19:34:33.730+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a6f00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a6f00 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a6f00 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6f00 prepare 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a6f00 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6f00 io_done 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c57380000 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c5500d000 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0297e700 7 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58260000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58260000 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58260000 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260000 prepare 2019-12-07T19:34:33.730+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58260000 2019-12-07T19:34:33.730+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260000 io_done 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] publish_stats_to_osd 55:184 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b4ea8 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 5 cleaned 0 in 0.000224438s (0.000000078s flush + 0.000224360s kv commit) 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57959800,0x558c579d2c00,0x558c575c5200,0x558c58541500,0x558c586e0000> 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959800 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57959800 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959800 finishing 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57959800 onodes 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57959800 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57959800 [] 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57bd2300 seq 57 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2c00 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c579d2c00 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2c00 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c579d2c00 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c579d2c00 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:33.730+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c579d2c00 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5200 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c5200 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5200 finishing 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3402 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c5200 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c5200 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c5200 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd2300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541500 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd2300 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58541500 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd2300 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541500 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58541500 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58541500 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58541500 [] 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0000 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e0000 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2300 prepare 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd2300 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4536 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0000 finishing 2019-12-07T19:34:33.730+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e0000 onodes 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2300 io_done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0000 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:33.730+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e0000 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:33.730+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4559 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4559 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c548ab000 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b4ea8~11cf to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b4ea8 0x3b4ea8~11cf to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x384ea8 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xea8 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x77 and padding block with 0xf89 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] handle_advance_map: 56 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 55 -> 56 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 55 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,13~1,19~2]] state: Active advmap 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] on_active_advmap new purged_snaps [13~1,19~1], now [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] share_pg_info 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c55053500 con 0x558c54c64000 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c55052e00 con 0x558c54c19800 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c55052700 con 0x558c54c5bc00 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b6077 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c55055100 con 0x558c54c5b400 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000168731s (0.000000089s flush + 0.000168642s kv commit) 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57959b00,0x558c578a6c00,0x558c578a6f00,0x558c58260000> 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959b00 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57959b00 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57959b00 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57959b00 onodes 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57959b00 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:33.730+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e56/56) v1 -- 0x558c57a53500 con 0x558c54c19c00 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57959b00 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6c00 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a6c00 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6c00 finishing 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a6c00 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a6c00 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a6c00 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6f00 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a6f00 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a6f00 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a6f00 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a6f00 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a6f00 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b6077~485 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260000 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58260000 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b6077 0x3b6077~485 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260000 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58260000 onodes 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58260000 done 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] publish_stats_to_osd 54:147 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x386077 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x77 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58260000 [] 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4fc and padding block with 0xb04 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:33.730+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:33.730+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:33.730+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 56 advance_pg new pool opts old pool opts 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] handle_activate_map: 56 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:33.730+0000 7f8f0297e700 7 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b64fc 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] publish_stats_to_osd 55:148 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 54 while current is 56 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116523s (0.000000070s flush + 0.000116453s kv commit) 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c57bd2600 seq 46 2019-12-07T19:34:33.730+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57bd2300> 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2300 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd2300 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2300 finishing 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd2600 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd2300 onodes 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd2600 onodes shared_blobs 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2300 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd2600 allocated 0x[] released 0x[] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd2300 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:33.730+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2600 prepare 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd2600 2019-12-07T19:34:33.730+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2600 io_done 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b64fc~485 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b64fc 0x3b64fc~485 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c57383000 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x3864fc 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4fc 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x981 and padding block with 0x67f 2019-12-07T19:34:33.730+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) queued 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:33.730+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 56 epoch_requested: 56 NullEvt) prio 255 cost 10 e56) pg 0x558c54d5f000 2019-12-07T19:34:33.730+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f05183700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 56 epoch_requested: 56 NullEvt 2019-12-07T19:34:33.730+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:33.730+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b6981 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:33.730+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127701s (0.000000092s flush + 0.000127609s kv commit) 2019-12-07T19:34:33.730+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57bd2600> 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2600 kv_submitted 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd2600 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2600 finishing 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd2600 onodes 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2600 done 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd2600 [] 2019-12-07T19:34:33.730+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:33.730+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:33.730+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:33.730+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:33.740+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.753+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:33.755+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 16 ==== osd_ping(ping e56 up_from 13 ping_stamp 2019-12-07T19:34:33.756910+0000/57.213834051s send_stamp 57.213834051s delta_ub 0.021842388s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c54c19000 2019-12-07T19:34:33.755+0000 7f8f23c56700 20 osd.7 56 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021374449s,0.021842388s]) 2019-12-07T19:34:33.755+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e56 up_from 13 ping_stamp 2019-12-07T19:34:33.756910+0000/57.213834051s send_stamp 57.192459602s delta_ub 18446744073.688177167s) v5 -- 0x558c580b9a00 con 0x558c54c19000 2019-12-07T19:34:33.755+0000 7f8f23c56700 10 osd.7 56 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 55 -> 56 (as per caller) 2019-12-07T19:34:33.755+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 16 ==== osd_ping(ping e56 up_from 13 ping_stamp 2019-12-07T19:34:33.756910+0000/57.213834051s send_stamp 57.213834051s delta_ub 0.021842388s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513e00 con 0x558c54c18800 2019-12-07T19:34:33.755+0000 7f8f23c56700 20 osd.7 56 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021315202s,0.021842388s]) 2019-12-07T19:34:33.755+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e56 up_from 13 ping_stamp 2019-12-07T19:34:33.756910+0000/57.213834051s send_stamp 57.192518849s delta_ub 18446744073.688236414s) v5 -- 0x558c580b9a00 con 0x558c54c18800 2019-12-07T19:34:33.790+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.840+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.854+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:33.890+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.939+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:33.990+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.017+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c58158000 0x558c574b9700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:34.017+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c58158000 0x558c574b9700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:34.017+0000 7f8f24457700 10 osd.7 56 ms_handle_authentication new session 0x558c57bfa4e0 con 0x558c58158000 entity client.admin addr 2019-12-07T19:34:34.017+0000 7f8f24457700 10 osd.7 56 ms_handle_authentication session 0x558c57bfa4e0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:34.017+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1870964600 conn(0x558c58158000 0x558c574b9700 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4526 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:34.017+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4526 172.21.15.131:0/1870964600 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c54df72c0 con 0x558c58158000 2019-12-07T19:34:34.017+0000 7f8f164bb700 20 osd.7 56 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:34.017+0000 7f8f164bb700 10 osd.7 56 do_waiters -- start 2019-12-07T19:34:34.017+0000 7f8f164bb700 10 osd.7 56 do_waiters -- finish 2019-12-07T19:34:34.017+0000 7f8f164bb700 20 osd.7 56 _dispatch 0x558c54df72c0 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:34.018+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1870964600 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce1dc0 con 0x558c58158000 2019-12-07T19:34:34.040+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.040+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4526 172.21.15.131:0/1870964600 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c54df7860 con 0x558c58158000 2019-12-07T19:34:34.040+0000 7f8f164bb700 20 osd.7 56 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:34.040+0000 7f8f164bb700 10 osd.7 56 do_waiters -- start 2019-12-07T19:34:34.040+0000 7f8f164bb700 10 osd.7 56 do_waiters -- finish 2019-12-07T19:34:34.040+0000 7f8f164bb700 20 osd.7 56 _dispatch 0x558c54df7860 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:34.042+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1870964600 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce08c0 con 0x558c58158000 2019-12-07T19:34:34.045+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1870964600 conn(0x558c58158000 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 72 2019-12-07T19:34:34.045+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1870964600 conn(0x558c58158000 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:34.045+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1870964600 conn(0x558c58158000 0x558c574b9700 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:34.045+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1870964600 conn(0x558c58158000 0x558c574b9700 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:34.045+0000 7f8f164bb700 2 osd.7 56 ms_handle_reset con 0x558c58158000 session 0x558c57bfa4e0 2019-12-07T19:34:34.090+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.141+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.191+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.240+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.281+0000 7f8f21451700 10 osd.7 56 tick 2019-12-07T19:34:34.281+0000 7f8f21451700 10 osd.7 56 do_waiters -- start 2019-12-07T19:34:34.281+0000 7f8f21451700 10 osd.7 56 do_waiters -- finish 2019-12-07T19:34:34.281+0000 7f8f21451700 20 osd.7 56 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:34.291+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.340+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.390+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.440+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.491+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.509+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:34.541+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.591+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1379336 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 32 ==== osd_map(56..57 src has 1..57) v4 ==== 1302+0+0 (crc 0 0 0) 0x558c579d9680 con 0x558c5476d000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 osd.7 56 OSD::ms_dispatch: osd_map(56..57 src has 1..57) v4 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 osd.7 56 do_waiters -- start 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 osd.7 56 do_waiters -- finish 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 osd.7 56 _dispatch 0x558c579d9680 osd_map(56..57 src has 1..57) v4 2019-12-07T19:34:34.613+0000 7f8f164bb700 3 osd.7 56 handle_osd_map epochs [56,57], i have 56, src has [1,57] 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 osd.7 56 handle_osd_map got inc map for epoch 57 2019-12-07T19:34:34.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ef4e826:::osdmap.56:0# 0x0~0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c550b2380 blob([0xb9a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5829c1c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57efce00 blob([!~1000,0xb9b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c57efce70 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3ef4e826:::osdmap.56:0# 0x0~1464 = 5220 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 osd.7 56 add_map_bl 56 5220 bytes 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 osd.7 56 got_full_map 57, nothing requested 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 osd.7 56 handle_osd_map pg_num_history pg_num_history(e57 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {57={}} 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 osd.7 56 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e57 [1,57] lci=[13,57]) 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c582b6900 seq 48 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cce7972f:::inc_osdmap.57:0# key 0x7f7fffffffffffffffcce797'/!inc_osdmap.57!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cce7972f:::inc_osdmap.57:0# 0x0~269 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1268 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cce7972f:::inc_osdmap.57:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6900 1 blobs 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb9c000~1000/1000,1000,0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb9c000~1000] 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5898b7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7a40 blob([0xb9c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5898b7a0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7a40) get_ref 0x0~269 Blob(0x558c580d7a40 blob([0xb9c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5898b7a0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7a40) get_ref init 0x1000, 1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c580d7a40 blob([0xb9c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5898b7a0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5898b7b8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795ba08) dirty_range mark inline shard dirty 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cce7972f:::inc_osdmap.57:0# 0x0~269 = 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3034e826:::osdmap.57:0# key 0x7f7fffffffffffffff3034e8'&!osdmap.57!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3034e826:::osdmap.57:0# 0x0~1464 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1269 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3034e826:::osdmap.57:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6900 2 blobs 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb9d000~2000/1000,2000,0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xb9d000~2000] 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579d7570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7340 blob([0xb9d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579d7570 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7340) get_ref 0x0~1000 Blob(0x558c580d7340 blob([0xb9d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579d7570 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7340) get_ref init 0x1000, 1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c580d7340 blob([0xb9d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579d7570 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c579d7588 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580f6e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f6ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580f6e70 blob([!~1000,0xb9e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f6ee0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c580f6e70) get_ref 0x1000~464 Blob(0x558c580f6e70 blob([!~1000,0xb9e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c580f6ee0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c580f6e70) get_ref init 0x2000, 1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c580f6e70 blob([!~1000,0xb9e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c580f6ee0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c580f6ef8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795bc88) dirty_range mark inline shard dirty 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3034e826:::osdmap.57:0# 0x0~1464 = 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6900 0 blobs 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6900 0 blobs 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b6900 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b6900 onodes 0x558c53a91900,0x558c55276c80,0x558c5795b900,0x558c5795bb80 shared_blobs 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795ba08) update #-1:cce7972f:::inc_osdmap.57:0# 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795ba08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cce7972f:::inc_osdmap.57:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795bc88) update #-1:3034e826:::osdmap.57:0# 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore.extentmap(0x558c5795bc88) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3034e826:::osdmap.57:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b6900 allocated 0x[b9c000~3000] released 0x[] 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 freelist allocate 0xb9c000~3000 2019-12-07T19:34:34.613+0000 7f8f164bb700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6900 prepare 2019-12-07T19:34:34.613+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c582b6900 2019-12-07T19:34:34.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6900 aio_wait 2019-12-07T19:34:34.614+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b6900 2019-12-07T19:34:34.614+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6900 io_done 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5759 < min_flush_size 524288 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795b900 had 1 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795bb80 had 1 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b6981~2277 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b6981 0x3b6981~2277 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x386981 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x981 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbf8 and padding block with 0x408 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b8bf8 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000155244s (0.000041204s flush + 0.000114040s kv commit) 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b6900> 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.614+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6900 kv_submitted 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b6900 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c582b6900 osr 0x558c54ea4b40 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 45 0x1b000~1000 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7 56 _committed_osd_maps 57..57 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7 56 advance to epoch 57 (<= last 57 <= newest_map 57) 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 46 0x1b000~1000 crc e9b77439 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 45 0x2000~1000 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 46 0x2000~1000 crc bd027962 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.614+0000 7f8f141a1700 7 osd.7 57 consume_map version 57 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.8s5 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.0s3 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.1s1 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.9s4 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.as5 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.2s0 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.bs1 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.3s4 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.cs0 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.4s0 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.5s4 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.ds2 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.es4 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.6s2 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.fs5 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 identify_splits_and_merges 1.7s4 e56 to e57 pg_nums {18=16} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 57 prune_pg_created 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:0.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:1.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:2.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:3.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:4.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:5.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:6.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7:7.consume_map 57 (was 56) 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:34.614+0000 7f8f141a1700 10 osd.7 57 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.8s5 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.0s3 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.1s1 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.9s4 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.as5 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c57380000 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.2s0 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.bs1 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c5736a000 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.3s4 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.cs0 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.4s0 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c57e8b000 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.5s4 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.ds2 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.es4 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.6s2 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.fs5 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c57676000 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f141a1700 15 osd.7 57 enqueue_peering_evt 1.7s4 epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c54d98000 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 56 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 56 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:34.614+0000 7f8f141a1700 7 osd.7 57 activate_map version 57 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 56 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:34.614+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 56 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 56 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean ps=[e~1,1a~1]] state: Active advmap 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] on_active_advmap new removed_snaps [15~1], snap_trimq now [15~1] 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:34.614+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0918b700 20 osd.7 57 agent_entry empty queue 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c54d5c000 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c5500a000 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c5500d000 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c57e88000 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 56 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 56 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 56 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s0 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 56 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,1a~1]] state: Active advmap 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c57522000 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] exit NotTrimming 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] on_active_advmap new removed_snaps [15~1], snap_trimq now [15~1] 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c57383000 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 56 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f06986700 15 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 56:185 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.614+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:34.614+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06986700 7 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.614+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c575ef000 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f06986700 15 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 56:186 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c54d5f000 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 56 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c548ab000 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:34.614+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0b990700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] exit NotTrimming 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0b990700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: NotTrimming: trimming 15 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c55141000 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.4s0 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 56 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 56 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.614+0000 7f8f07988700 15 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 56:161 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,1a~1]] state: Active advmap 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0b990700 15 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:187 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 NullEvt) prio 255 cost 10 e57) pg 0x558c5513e000 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] on_active_advmap new removed_snaps [15~1], snap_trimq now [15~1] 2019-12-07T19:34:34.614+0000 7f8f07988700 7 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 56 -> 57 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f0b990700 10 osd.7 57 queueing pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean trimq=[15~1] ps=[e~1,1a~1]] for snaptrim 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=57) prio 5 cost 1048576 e57) 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 56 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:34.614+0000 7f8f07988700 15 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 56:162 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0b990700 15 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:188 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.614+0000 7f8f0b990700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.614+0000 7f8f0b990700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: NotTrimming: trimming 15 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=57) prio 5 cost 1048576 e57) queued 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=57) prio 5 cost 1048576 e57) pg 0x558c57676000 2019-12-07T19:34:34.614+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:34.614+0000 7f8f0b990700 15 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:163 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 15 2019-12-07T19:34:34.614+0000 7f8f0b990700 10 osd.7 57 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[15~1] ps=[e~1,1a~1]] for snaptrim 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=57) prio 5 cost 1048576 e57) 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] exit NotTrimming 2019-12-07T19:34:34.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=57) prio 5 cost 1048576 e57) queued 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:34.614+0000 7f8f0b990700 15 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:164 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming 2019-12-07T19:34:34.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=57) prio 5 cost 1048576 e57) pg 0x558c57e88000 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:34.614+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:34.614+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 15 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:34.614+0000 7f8f06986700 15 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 56:149 2019-12-07T19:34:34.614+0000 7f8f06986700 7 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57bcc980) upper_bound after SNA_1_0000000000000015_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.C' 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57bcc980) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:34.614+0000 7f8f06986700 15 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 56:150 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c57bcca00) upper_bound after SNA_1_0000000000000015_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.2' 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore.OmapIteratorImpl(0x558c57bcca00) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:34.614+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000015_.0_0000000000000001.C) returns 0 SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239600 in raw 0x558c55239600 len 69 nref 1) ) 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:34.614+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.614+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f03980700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000015_.0_0000000000000001.2) returns 0 SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55238f20 in raw 0x558c55238f20 len 69 nref 1) ) 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.614+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:34.614+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57bcc980) upper_bound after SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. key 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:34.614+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,1a~1]] SnapTrimmer state: adding snap 15 to purged_snaps 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57bcc980) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50..' 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..) returns 0 SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:34.614+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.614+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1]] SnapTrimmer state: purged_snaps now [e~1,15~1,1a~1], snap_trimq now [] 2019-12-07T19:34:34.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c58260300 seq 30 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_object_context: found obc in cache: 0x558c5738f8c0 2019-12-07T19:34:34.614+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_object_context: 0x558c5738f8c0 1:33f5bf01:::smithi13140796-2:19 rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:19(53'39 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[15]} 2019-12-07T19:34:34.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:34.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58260300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58260300 onodes shared_blobs 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58260300 allocated 0x[] released 0x[] 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:34.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260300 prepare 2019-12-07T19:34:34.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58260300 2019-12-07T19:34:34.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260300 io_done 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(53'40 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{19=[15]} 2019-12-07T19:34:34.614+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1]] share_pg_info 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 1:33f5bf01:::smithi13140796-2:19 old_snaps 15 old snapset 19=[]:{19=[15]} 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] simple_opc_create 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b8bf8~495 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58536700 con 0x558c54c64000 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b8bf8 0x3b8bf8~495 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x388bf8 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbf8 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 1:33f5bf01:::smithi13140796-2:19 snaps 15 -> ... deleting 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8d and padding block with 0xf73 2019-12-07T19:34:34.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58538a00 con 0x558c54c19800 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 1:33f5bf01:::smithi13140796-2:19 new snapset 19=[]:{} on 1:33f5bf01:::smithi13140796-2:head(53'40 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 1:33f5bf01:::smithi13140796-2:19 writing updated snapset on 1:33f5bf01:::smithi13140796-2:head, snapset is 19=[]:{} 2019-12-07T19:34:34.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58536e00 con 0x558c54c5bc00 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] new_repop rep_tid 47 (no op) 2019-12-07T19:34:34.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c5829f500 con 0x558c54c5b400 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] new_repop: repgather(0x558c576b6fc0 0'0 rep_tid=47 committed?=0 r=0) 2019-12-07T19:34:34.614+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] simple_opc_submit 0x558c576b6fc0 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b908d 2019-12-07T19:34:34.614+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c582a1800 con 0x558c54c19c00 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.614+0000 7f8f0297e700 7 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] issue_repop rep_tid 47 o 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000106389s (0.000000081s flush + 0.000106308s kv commit) 2019-12-07T19:34:34.614+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:34.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.614+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58260300> 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.614+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1]] exit Trimming 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260300 kv_submitted 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58260300 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260300 finishing 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58260300 onodes 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58260300 done 2019-12-07T19:34:34.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:19 v=57'42 tt=0'0 tid=47 reqid=osd.7.0:47 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58260300 [] 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.615+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.615+0000 7f8f03980700 15 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:165 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# 2019-12-07T19:34:34.615+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] enter NotTrimming 2019-12-07T19:34:34.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:34.615+0000 7f8f0b990700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: NotTrimming: trimming 15 2019-12-07T19:34:34.615+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_hash_info: found on disk, size 225280 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key 2019-12-07T19:34:34.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[15~1] ps=[e~1,1a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:34.615+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# hinfo_key = 0 2019-12-07T19:34:34.615+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] snap_trimmer complete 2019-12-07T19:34:34.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.615+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:34.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:151 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 projected size 0 2019-12-07T19:34:34.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:34.615+0000 7f8f0b990700 10 osd.7 57 queueing pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[15~1] ps=[e~1,1a~1]] for snaptrim 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:34.615+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=57) prio 5 cost 1048576 e57) 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:34.615+0000 7f8f0b990700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=57) prio 5 cost 1048576 e57) queued 2019-12-07T19:34:34.615+0000 7f8f0b990700 15 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:152 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:34.615+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=57) prio 5 cost 1048576 e57) pg 0x558c548ab000 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=57'42 tt=0'0 tid=47 reqid=osd.7.0:47 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 15 2019-12-07T19:34:34.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:19 v=57'42 tt=0'0 tid=47 reqid=osd.7.0:47 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=57'42 tt=0'0 tid=47 reqid=osd.7.0:47 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54e88780) upper_bound after SNA_1_0000000000000015_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.4' 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c54e88780) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' 2019-12-07T19:34:34.615+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000015_.0_0000000000000001.4) returns 0 SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239810 in raw 0x558c55239810 len 69 nref 1) ) 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:19, current size is 901120 buffers are {} 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,1a~1]] SnapTrimmer state: adding snap 15 to purged_snaps 2019-12-07T19:34:34.615+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1]] SnapTrimmer state: purged_snaps now [e~1,15~1,1a~1], snap_trimq now [] 2019-12-07T19:34:34.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c58748000 seq 47 2019-12-07T19:34:34.615+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): new_size start 0 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:34.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:34.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58748000 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58748000 onodes shared_blobs 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58748000 allocated 0x[] released 0x[] 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:34.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748000 prepare 2019-12-07T19:34:34.615+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58748000 2019-12-07T19:34:34.615+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748000 io_done 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1]] share_pg_info 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): 1:33f5bf01:::smithi13140796-2:19 resetting hinfo to logical size 0 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:34.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58134000 con 0x558c54c64000 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58137100 con 0x558c54c19800 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b908d~495 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b908d 0x3b908d~495 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x38908d 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8d 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x522 and padding block with 0xade 2019-12-07T19:34:34.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58134700 con 0x558c54c5bc00 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): new_size start 1622016 2019-12-07T19:34:34.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58135500 con 0x558c54c5b400 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:34.615+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 -- 0x558c58134e00 con 0x558c54c19c00 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1]] exit Trimming 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:34.615+0000 7f8f06986700 15 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:153 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] enter NotTrimming 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:19={},1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:34.615+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] snap_trimmer complete 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=57'42 tt=0'0 tid=47 reqid=osd.7.0:47 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:19,{} 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3b9522 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000117367s (0.000000102s flush + 0.000117265s kv commit) 2019-12-07T19:34:34.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58748000> 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748000 kv_submitted 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58748000 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748000 finishing 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58748000 onodes 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58748000 done 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58748000 [] 2019-12-07T19:34:34.615+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.615+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 send_incremental_map 56 -> 57 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cce7972f:::inc_osdmap.57:0# 0x0~0 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d7a40 blob([0xb9c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5898b7a0 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cce7972f:::inc_osdmap.57:0# 0x0~269 = 617 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 add_map_inc_bl 57 617 bytes 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(57..57 src has 1..57) v4 -- 0x558c579d8d80 con 0x558c54c64000 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 56 -> 57 (shared) 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 57/28 ECSubWrite(tid=47, reqid=osd.7.0:47, at_version=57'42, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c58154800 con 0x558c54c64000 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 send_incremental_map 55 -> 57 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c667972f:::inc_osdmap.56:0# 0x0~0 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~279 size 0x279 (633) 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5829c850 blob([0xb99000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x279) SharedBlob(0x558c5742bea0 sbid 0x0)) need 0x0~279 cache has 0x[0~279] 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c667972f:::inc_osdmap.56:0# 0x0~279 = 633 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 add_map_inc_bl 56 633 bytes 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(56..57 src has 1..57) v4 -- 0x558c579d8b40 con 0x558c54c19800 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 55 -> 57 (shared) 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 57/28 ECSubWrite(tid=47, reqid=osd.7.0:47, at_version=57'42, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c58155800 con 0x558c54c19800 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 send_incremental_map 55 -> 57 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(56..57 src has 1..57) v4 -- 0x558c579d8900 con 0x558c54c5bc00 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 55 -> 57 (shared) 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 57/28 ECSubWrite(tid=47, reqid=osd.7.0:47, at_version=57'42, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c575c0800 con 0x558c54c5bc00 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 send_incremental_map 55 -> 57 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(56..57 src has 1..57) v4 -- 0x558c579d86c0 con 0x558c54c5b400 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 55 -> 57 (shared) 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 57/28 ECSubWrite(tid=47, reqid=osd.7.0:47, at_version=57'42, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c575bf000 con 0x558c54c5b400 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 send_incremental_map 54 -> 57 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cb67972f:::inc_osdmap.55:0# 0x0~0 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57e63f80 blob([0x33a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5809b730 sbid 0x0)) need 0x0~24d cache has 0x[] 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc blob Blob(0x558c57e63f80 blob([0x33a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5809b730 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~24d]} 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read waiting for aio 2019-12-07T19:34:34.614+0000 7f8f06185700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] handle_advance_map: 57 2019-12-07T19:34:34.615+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 56 -> 57 2019-12-07T19:34:34.615+0000 7f8f06185700 10 osd.7 pg_epoch: 56 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:34.615+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] state: Started advmap 2019-12-07T19:34:34.615+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:34.615+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:34.615+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:34.615+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] hit_set_clear 2019-12-07T19:34:34.615+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] agent_stop 2019-12-07T19:34:34.615+0000 7f8f06185700 20 osd.7 57 advance_pg new pool opts old pool opts 2019-12-07T19:34:34.615+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] handle_activate_map: 57 2019-12-07T19:34:34.615+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] activate_map 2019-12-07T19:34:34.615+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] activate_map: Not dirtying info: last_persisted is 56 while current is 57 2019-12-07T19:34:34.615+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:34.615+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 NullEvt 2019-12-07T19:34:34.615+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:34.615+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _generate_read_result_bl blob Blob(0x558c57e63f80 blob([0x33a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5809b730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~24d]} 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore.BufferSpace(0x558c5809b748 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 _add level 1 near 0 on buffer(0x558c54c9b0e0 space 0x558c5809b748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cb67972f:::inc_osdmap.55:0# 0x0~24d = 589 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 add_map_inc_bl 55 589 bytes 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(55..57 src has 1..57) v4 -- 0x558c579d8000 con 0x558c54c19c00 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 57 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 54 -> 57 (shared) 2019-12-07T19:34:34.615+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 57/28 ECSubWrite(tid=47, reqid=osd.7.0:47, at_version=57'42, trim_to=0'0, roll_forward_to=53'40)) v2 -- 0x558c5748e800 con 0x558c54c19c00 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 snap_mapper.remove_oid 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 snap_mapper._remove_oid 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 snap_mapper.get_snaps 1:33f5bf01:::smithi13140796-2:19 15 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 snap_mapper.clear_snaps 1:33f5bf01:::smithi13140796-2:19 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 snap_mapper.clear_snaps rm OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 snap_mapper._remove_oid rm SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 trim 57'41 (53'39) delete 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 trim 57'42 (53'40) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 53'40 (0'0,53'40] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] append_log log((0'0,53'40], crt=53'40) [57'41 (53'39) delete 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'42 (53'40) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'41 (0'0,57'41] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 lua=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] add_log_entry 57'41 (53'39) delete 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 lua=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] add_log_entry 57'42 (53'40) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 lua=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] append_log approx pg log length = 42 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 lua=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.615+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'41, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57bd2900 seq 58 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:19#29 key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0x00000000000000190000000000000029'o' 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# -> 0#1:33f5bf01:::smithi13140796-2:19#29 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19# -> 0#1:33f5bf01:::smithi13140796-2:19#29 = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- OBJ_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2..' <- SNA_1_0000000000000015_.0_0000000000000001.CCFADF08.19.smithi13140796-2.. 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000057.00000000000000000041' <- 0000000057.00000000000000000041 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000057.00000000000000000042' <- 0000000057.00000000000000000042 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:34.615+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd2900 cost 7526 (1 ios * 4000 + 3526 bytes) 2019-12-07T19:34:34.615+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd2900 onodes 0x558c58295180,0x558c58295400 shared_blobs 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295288) update 0#1:33f5bf01:::smithi13140796-2:19#29 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:19#29 is 1172 (410 bytes onode + 2 bytes spanning blobs + 760 bytes inline extents) 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1441 (485 bytes onode + 2 bytes spanning blobs + 954 bytes inline extents) 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd2900 allocated 0x[] released 0x[] 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2900 prepare 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd2900 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2900 io_done 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 lua=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] op_applied version 57'42 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5060 < min_flush_size 524288 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] eval_repop repgather(0x558c576b6fc0 57'42 rep_tid=47 committed?=0 r=0) (no op) 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295180 had 1 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a3400 had 1 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] calc_trim_to_aggressive limit = 53'40 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5083 < min_flush_size 524288 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5083 < min_flush_size 524288 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3b9522~13db to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/WaitRepops 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3b9522 0x3b9522~13db to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x389522 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x522 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] snap_trimmer complete 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8fd and padding block with 0x703 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3ba8fd 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000356085s (0.000000087s flush + 0.000355998s kv commit) 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57bd2900> 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2900 kv_submitted 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd2900 2019-12-07T19:34:34.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2900 finishing 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd2900 onodes 0x558c58295180,0x558c58295400 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2900 done 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:34.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd2900 [] 2019-12-07T19:34:34.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 290 ==== MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58513200 con 0x558c54c5b400 2019-12-07T19:34:34.616+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c54e79a20 prio 196 cost 0 latency 0.000014 epoch 56 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.616+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) 2019-12-07T19:34:34.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 274 ==== MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b9a00 con 0x558c54c5bc00 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) queued 2019-12-07T19:34:34.616+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c5511f600 prio 196 cost 0 latency 0.000008 epoch 56 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.616+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) pg 0x558c57676000 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c54e79a20 prio 196 cost 0 latency 0.000056 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) queued 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) pg 0x558c57676000 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c5511f600 prio 196 cost 0 latency 0.000051 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.616+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c5511f600 finish 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.616+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.616+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 107 ==== MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512800 con 0x558c54c19c00 2019-12-07T19:34:34.616+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c54e78580 prio 196 cost 0 latency 0.000017 epoch 56 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.616+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) queued 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) pg 0x558c57676000 2019-12-07T19:34:34.616+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c54e78580 prio 196 cost 0 latency 0.000063 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 2019-12-07T19:34:34.616+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.616+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:34.616+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.617+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 191 ==== MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c54c19800 2019-12-07T19:34:34.617+0000 7f8f24c58700 15 osd.7 57 enqueue_op 0x558c573358c0 prio 196 cost 0 latency 0.000014 epoch 56 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.617+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) 2019-12-07T19:34:34.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) queued 2019-12-07T19:34:34.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) pg 0x558c57676000 2019-12-07T19:34:34.617+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c573358c0 prio 196 cost 0 latency 0.000914 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 2019-12-07T19:34:34.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.617+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c573358c0 finish 2019-12-07T19:34:34.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 291 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c58134700 con 0x558c54c5b400 2019-12-07T19:34:34.618+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.3s4 epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:34.618+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s)) prio 255 cost 10 e57) pg 0x558c5513e000 2019-12-07T19:34:34.618+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.056879179s ub 67.056879179s int 16s) 2019-12-07T19:34:34.618+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.618+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:34.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:34.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c586e0300 seq 68 2019-12-07T19:34:34.618+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:34.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:34.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:34.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:34.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:34.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e0300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:34.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e0300 onodes shared_blobs 2019-12-07T19:34:34.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e0300 allocated 0x[] released 0x[] 2019-12-07T19:34:34.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0300 prepare 2019-12-07T19:34:34.618+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e0300 2019-12-07T19:34:34.618+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0300 io_done 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3ba8fd~495 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3ba8fd 0x3ba8fd~495 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~390000 x_off 0x38a8fd 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8fd 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd92 and padding block with 0x26e 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3bad92 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.618+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000115794s (0.000000125s flush + 0.000115669s kv commit) 2019-12-07T19:34:34.618+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 292 ==== MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=109, reqid=osd.4.0:109, at_version=57'39, trim_to=0'0, roll_forward_to=53'37)) v2 ==== 2656+0+0 (unknown 0 0 0) 0x558c575c0800 con 0x558c54c5b400 2019-12-07T19:34:34.618+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c578d2420 prio 127 cost 0 latency 0.000025 epoch 57 MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=109, reqid=osd.4.0:109, at_version=57'39, trim_to=0'0, roll_forward_to=53'37)) v2 2019-12-07T19:34:34.618+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=109, reqid=osd.4.0:109, at_version=57'39, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=109, reqid=osd.4.0:109, at_version=57'39, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=109, reqid=osd.4.0:109, at_version=57'39, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e57) pg 0x558c575ef000 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 57 dequeue_op 0x558c578d2420 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=109, reqid=osd.4.0:109, at_version=57'39, trim_to=0'0, roll_forward_to=53'37)) v2 pg pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=109, reqid=osd.4.0:109, at_version=57'39, trim_to=0'0, roll_forward_to=53'37)) v2 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 snap_mapper.remove_oid 1:dce63cc3:::smithi13140796-50:15 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 snap_mapper._remove_oid 1:dce63cc3:::smithi13140796-50:15 2019-12-07T19:34:34.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 467 ==== MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57950200 con 0x558c54c64000 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 snap_mapper.get_snaps 1:dce63cc3:::smithi13140796-50:15 15 2019-12-07T19:34:34.618+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511e420 prio 196 cost 0 latency 0.000014 epoch 56 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 snap_mapper.clear_snaps 1:dce63cc3:::smithi13140796-50:15 2019-12-07T19:34:34.618+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 snap_mapper.clear_snaps rm OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 snap_mapper._remove_oid rm SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) queued 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e56) pg 0x558c57676000 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 53'37 (0'0,53'37] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,53'37], crt=53'37) [57'38 (53'34) delete 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:34.618977+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'39 (53'35) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:34.618977+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c5511e420 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] 2019-12-07T19:34:34.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 468 ==== MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=281, reqid=osd.1.0:281, at_version=57'44, trim_to=0'0, roll_forward_to=55'42)) v2 ==== 3417+0+0 (crc 0 0 0) 0x558c5748e800 con 0x558c54c64000 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 56/28 ECSubWriteReply(tid=47, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.618+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511edc0 prio 127 cost 0 latency 0.000025 epoch 57 MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=281, reqid=osd.1.0:281, at_version=57'44, trim_to=0'0, roll_forward_to=55'42)) v2 2019-12-07T19:34:34.618+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=281, reqid=osd.1.0:281, at_version=57'44, trim_to=0'0, roll_forward_to=55'42)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:19 v=57'42 tt=0'0 tid=47 reqid=osd.7.0:47 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=53'40 crt=53'40 lcod 53'38 mlcod 53'38 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] repop_all_committed: repop tid 47 all committed 2019-12-07T19:34:34.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'38 (0'0,57'38] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'37 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'38 (53'34) delete 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:34.618977+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=281, reqid=osd.1.0:281, at_version=57'44, trim_to=0'0, roll_forward_to=55'42)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] eval_repop repgather(0x558c576b6fc0 57'42 rep_tid=47 committed?=1 r=0) (no op) 2019-12-07T19:34:34.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=281, reqid=osd.1.0:281, at_version=57'44, trim_to=0'0, roll_forward_to=55'42)) v2) prio 127 cost 0 e57) pg 0x558c5500a000 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] commit: repgather(0x558c576b6fc0 57'42 rep_tid=47 committed?=1 r=0) 2019-12-07T19:34:34.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 469 ==== MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=280, reqid=osd.1.0:280, at_version=57'53, trim_to=0'0, roll_forward_to=54'51)) v2 ==== 3362+0+0 (crc 0 0 0) 0x558c58154800 con 0x558c54c64000 2019-12-07T19:34:34.618+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511ef20 prio 127 cost 0 latency 0.000020 epoch 57 MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=280, reqid=osd.1.0:280, at_version=57'53, trim_to=0'0, roll_forward_to=54'51)) v2 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.618+0000 7f8f08189700 10 osd.7 57 dequeue_op 0x558c5511edc0 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=281, reqid=osd.1.0:281, at_version=57'44, trim_to=0'0, roll_forward_to=55'42)) v2 pg pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.618+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=280, reqid=osd.1.0:280, at_version=57'53, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.618+0000 7f8f06986700 15 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:189 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'37 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'39 (53'35) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:34.618977+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.618+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=281, reqid=osd.1.0:281, at_version=57'44, trim_to=0'0, roll_forward_to=55'42)) v2 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] removing repgather(0x558c576b6fc0 57'42 rep_tid=47 committed?=1 r=0) 2019-12-07T19:34:34.618+0000 7f8f08189700 20 snap_mapper.update_snaps 1:9b8adabe:::smithi13140796-6:20 1b,20 was 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] q front is repgather(0x558c576b6fc0 57'42 rep_tid=47 committed?=1 r=0) 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'37 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 39 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] exit Trimming/WaitRepops 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'37 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:34.618+0000 7f8f08189700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:20 15,1b,20 2019-12-07T19:34:34.618+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'38, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:34.618+0000 7f8f08189700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:20 1b,20 2019-12-07T19:34:34.618+0000 7f8f08189700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:34.618+0000 7f8f08189700 20 snap_mapper.update_snaps rm SNA_1_0000000000000015_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 57 queueing pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] for snaptrim 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=57) prio 5 cost 1048576 e57) 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,1a~1] 2019-12-07T19:34:34.618+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 55'42 (0'0,55'42] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,55'42], crt=55'42) [57'43 (55'39) modify 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:34.618831+0000 0 snaps [1b,20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'44 (55'42) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:34.618831+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.618+0000 7f8f06986700 15 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] publish_stats_to_osd 57:190 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c5810fb00 seq 48 2019-12-07T19:34:34.618+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'43 (0'0,57'43] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'42 crt=55'42 lcod 55'42 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'43 (55'39) modify 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:34.618831+0000 0 snaps [1b,20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] remove_repop repgather(0x558c576b6fc0 57'42 rep_tid=47 committed?=1 r=0) 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:15#26 key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0x00000000000000150000000000000026'o' 2019-12-07T19:34:34.618+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'42 crt=55'42 lcod 55'42 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'44 (55'42) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:34.618831+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=57'42 tt=0'0 tid=47 reqid=osd.7.0:47 roll_forward_to=53'40 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:19=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:34.618+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'42 crt=55'42 lcod 55'42 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 44 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:34.618+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'42 crt=55'42 lcod 55'42 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# -> 1#1:dce63cc3:::smithi13140796-50:15#26 2019-12-07T19:34:34.618+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586e0300> 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.618+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'43, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15# -> 1#1:dce63cc3:::smithi13140796-50:15#26 = 0 2019-12-07T19:34:34.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0300 kv_submitted 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e0300 2019-12-07T19:34:34.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:34:34.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0300 finishing 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e0300 onodes 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0300 done 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=48 reqid=osd.7.0:47 roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:34.618+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e0300 [] 2019-12-07T19:34:34.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.618+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.618+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:34.618+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- OBJ_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50..' <- SNA_1_0000000000000015_.1_0000000000000001.B376C33C.15.smithi13140796-50.. 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.618+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c579d2f00 seq 83 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000057.00000000000000000038' <- 0000000057.00000000000000000038 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=48 reqid=osd.7.0:47 roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000057.00000000000000000039' <- 0000000057.00000000000000000039 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# 1 keys 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# 1 keys = 0 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# hinfo_key (42 bytes) 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5810fb00 cost 7551 (1 ios * 4000 + 3551 bytes) 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5810fb00 onodes 0x558c574a0a00,0x558c574a0c80 shared_blobs 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a0b08) update 1#1:dce63cc3:::smithi13140796-50:15#26 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- SNA_1_0000000000000015_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:34.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 57/28 ECSubWrite(tid=48, reqid=osd.7.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c58534000 con 0x558c54c64000 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:15#26 is 1120 (412 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a0d88) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:34.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 1193 (485 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000057.00000000000000000043' <- 0000000057.00000000000000000043 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5810fb00 allocated 0x[] released 0x[] 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000057.00000000000000000044' <- 0000000057.00000000000000000044 2019-12-07T19:34:34.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 57/28 ECSubWrite(tid=48, reqid=osd.7.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c58534800 con 0x558c54c19800 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810fb00 prepare 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5810fb00 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810fb00 io_done 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 57/28 ECSubWrite(tid=48, reqid=osd.7.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c58535000 con 0x558c54c5bc00 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:34.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c579d2f00 cost 8396 (1 ios * 4000 + 4396 bytes) 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c579d2f00 onodes 0x558c57573900,0x558c57a35400 shared_blobs 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore.extentmap(0x558c57573a08) update 4#1:9b8adabe:::smithi13140796-6:20# 2019-12-07T19:34:34.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 57/28 ECSubWrite(tid=48, reqid=osd.7.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c58535800 con 0x558c54c5b400 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=53'37 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] op_applied version 57'39 2019-12-07T19:34:34.618+0000 7f8f0317f700 10 osd.7 57 dequeue_op 0x558c578d2420 finish 2019-12-07T19:34:34.618+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 57/28 ECSubWrite(tid=48, reqid=osd.7.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c58532000 con 0x558c54c19c00 2019-12-07T19:34:34.618+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=53'40 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] append_log log((0'0,57'42], crt=53'40) [] 2019-12-07T19:34:34.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=280, reqid=osd.1.0:280, at_version=57'53, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.618+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=280, reqid=osd.1.0:280, at_version=57'53, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e57) pg 0x558c57380000 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:20# is 124356 (123297 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] rollforward: entry=57'41 (53'39) delete 1:33f5bf01:::smithi13140796-2:19 by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.618+0000 7f8f0898a700 10 osd.7 57 dequeue_op 0x558c5511ef20 prio 127 cost 0 latency 0.000297 MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=280, reqid=osd.1.0:280, at_version=57'53, trim_to=0'0, roll_forward_to=54'51)) v2 pg pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.618+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] rollforward: entry=57'42 (53'40) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:34.616102+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.618+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:34.618+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=280, reqid=osd.1.0:280, at_version=57'53, trim_to=0'0, roll_forward_to=54'51)) v2 2019-12-07T19:34:34.618+0000 7f8f0898a700 20 snap_mapper.update_snaps 1:10779340:::smithi13140796-32:1b 1b was 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] append_log approx pg log length = 42 2019-12-07T19:34:34.618+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.618+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:34.618+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.618+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 123465 (123463 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:34.619+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58748300 seq 59 2019-12-07T19:34:34.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19#29 onode 0x558c58295180 txc 0x558c58748300 2019-12-07T19:34:34.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19#29 0x0 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78700) put_ref 0x0~2000 Blob(0x558c58b78700 blob([0x2a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78690) put_ref 0x0~2000 Blob(0x558c58b78690 blob([0x2a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b785b0) put_ref 0x0~2000 Blob(0x558c58b785b0 blob([0x2aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78620) put_ref 0x0~2000 Blob(0x558c58b78620 blob([0x2ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78770) put_ref 0x0~2000 Blob(0x558c58b78770 blob([0x2ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b787e0) put_ref 0x0~2000 Blob(0x558c58b787e0 blob([0x2b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b788c0) put_ref 0x0~2000 Blob(0x558c58b788c0 blob([0x2b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78930) put_ref 0x0~2000 Blob(0x558c58b78930 blob([0x2b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b789a0) put_ref 0x0~2000 Blob(0x558c58b789a0 blob([0x350000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78a10) put_ref 0x0~2000 Blob(0x558c58b78a10 blob([0x352000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78a80) put_ref 0x0~2000 Blob(0x558c58b78a80 blob([0x354000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78af0) put_ref 0x0~2000 Blob(0x558c58b78af0 blob([0x356000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78b60) put_ref 0x0~2000 Blob(0x558c58b78b60 blob([0x358000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78bd0) put_ref 0x0~2000 Blob(0x558c58b78bd0 blob([0x35a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78c40) put_ref 0x0~2000 Blob(0x558c58b78c40 blob([0x35c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78cb0) put_ref 0x0~2000 Blob(0x558c58b78cb0 blob([0x35e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78d20) put_ref 0x0~2000 Blob(0x558c58b78d20 blob([0x360000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78d90) put_ref 0x0~2000 Blob(0x558c58b78d90 blob([0x362000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78e00) put_ref 0x0~2000 Blob(0x558c58b78e00 blob([0x364000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78e70) put_ref 0x0~2000 Blob(0x558c58b78e70 blob([0x366000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c579d2f00 allocated 0x[] released 0x[] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78ee0) put_ref 0x0~2000 Blob(0x558c58b78ee0 blob([0x368000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78f50) put_ref 0x0~2000 Blob(0x558c58b78f50 blob([0x36a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2f00 prepare 2019-12-07T19:34:34.619+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c579d2f00 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b78fc0) put_ref 0x0~2000 Blob(0x558c58b78fc0 blob([0x36c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2f00 io_done 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b79030) put_ref 0x0~2000 Blob(0x558c58b79030 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b790a0) put_ref 0x0~2000 Blob(0x558c58b790a0 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b79110) put_ref 0x0~2000 Blob(0x558c58b79110 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b79180) put_ref 0x0~2000 Blob(0x558c58b79180 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=55'42 crt=55'42 lcod 55'42 active mbc={} ps=[e~1,1a~1]] op_applied version 57'44 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.blob(0x558c58b791f0) put_ref 0x0~1000 Blob(0x558c58b791f0 blob([0x376000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295288) dirty_range mark inline shard dirty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c58b78700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map(0x2a6000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a6000~2000] 2019-12-07T19:34:34.619+0000 7f8f08189700 10 osd.7 57 dequeue_op 0x558c5511edc0 finish 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2a6000~2000] from SharedBlob(0x558c573963f0 loaded (sbid 0x2946 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2a6000~2000 2019-12-07T19:34:34.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c58b78690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map(0x2a8000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2a8000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2a8000~2000] from SharedBlob(0x558c57396150 loaded (sbid 0x2947 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2a8000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c58b785b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map(0x2aa000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2aa000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2aa000~2000] from SharedBlob(0x558c57880540 loaded (sbid 0x2948 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2aa000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c58b78620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map(0x2ac000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ac000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2ac000~2000] from SharedBlob(0x558c57880620 loaded (sbid 0x2949 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ac000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c58b78770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map(0x2ae000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2ae000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2ae000~2000] from SharedBlob(0x558c57880700 loaded (sbid 0x294a ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2ae000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c58b787e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map(0x2b0000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b0000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2b0000~2000] from SharedBlob(0x558c578807e0 loaded (sbid 0x294b ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b0000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c58b788c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map(0x2b2000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b2000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2b2000~2000] from SharedBlob(0x558c578808c0 loaded (sbid 0x294c ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b2000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c58b78930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map(0x2b4000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x2b4000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x2b4000~2000] from SharedBlob(0x558c578809a0 loaded (sbid 0x294d ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x2b4000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c58b789a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map(0x350000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x350000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x350000~2000] from SharedBlob(0x558c574d38f0 loaded (sbid 0x294e ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x350000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c58b78a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map(0x352000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x352000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x352000~2000] from SharedBlob(0x558c57880e70 loaded (sbid 0x294f ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x352000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c58b78a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map(0x354000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x354000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x354000~2000] from SharedBlob(0x558c54d75180 loaded (sbid 0x2950 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x354000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c58b78af0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map(0x356000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x356000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x356000~2000] from SharedBlob(0x558c54d75340 loaded (sbid 0x2951 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x356000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c58b78b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map(0x358000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x358000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x358000~2000] from SharedBlob(0x558c578de620 loaded (sbid 0x2952 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x358000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c58b78bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map(0x35a000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x35a000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x35a000~2000] from SharedBlob(0x558c578de460 loaded (sbid 0x2953 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x35a000~2000 2019-12-07T19:34:34.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c58b78c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map(0x35c000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x35c000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x35c000~2000] from SharedBlob(0x558c578de2a0 loaded (sbid 0x2954 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x35c000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c58b78cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map(0x35e000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x35e000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x35e000~2000] from SharedBlob(0x558c578de0e0 loaded (sbid 0x2955 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x35e000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c58b78d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map(0x360000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x360000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x360000~2000] from SharedBlob(0x558c57819f10 loaded (sbid 0x2956 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x360000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c58b78d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map(0x362000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x362000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x362000~2000] from SharedBlob(0x558c57819d50 loaded (sbid 0x2957 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x362000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c58b78e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map(0x364000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x364000~2000] 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x364000~2000] from SharedBlob(0x558c54cf8700 loaded (sbid 0x2958 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x364000~2000 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c58b78e70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map(0x366000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x366000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x366000~2000] from SharedBlob(0x558c54d3df80 loaded (sbid 0x2959 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x366000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c58b78ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map(0x368000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x368000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x368000~2000] from SharedBlob(0x558c5794c460 loaded (sbid 0x295a ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x368000~2000 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4784 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c58b78f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map(0x36a000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x36a000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x36a000~2000] from SharedBlob(0x558c5794c2a0 loaded (sbid 0x295b ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x36a000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c58b78fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map(0x36c000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x36c000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x36c000~2000] from SharedBlob(0x558c5794c230 loaded (sbid 0x295c ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x36c000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c58b79030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map(0x36e000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x36e000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x36e000~2000] from SharedBlob(0x558c5794c150 loaded (sbid 0x295d ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x36e000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c58b790a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map(0x370000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x370000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x370000~2000] from SharedBlob(0x558c5794ce00 loaded (sbid 0x295e ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x370000~2000 2019-12-07T19:34:34.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c58b79110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map(0x372000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x372000~2000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x372000~2000] from SharedBlob(0x558c574d32d0 loaded (sbid 0x295f ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x372000~2000 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c58b79180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map(0x374000~2000=1)))) 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0c80 had 1 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x374000~2000] 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x374000~2000] from SharedBlob(0x558c574d3880 loaded (sbid 0x2960 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57c04f00 had 1 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x374000~2000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~1000: 0x0~1000 Blob(0x558c58b791f0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map(0x376000~1000=1)))) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x376000~1000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [0x376000~1000] from SharedBlob(0x558c574d3730 loaded (sbid 0x2961 ref_map())) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish release 0x376000~1000 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 21102 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf8700,0x558c54d3df80,0x558c54d75180,0x558c54d75340,0x558c57396150,0x558c573963f0,0x558c574d32d0,0x558c574d3730,0x558c574d3880,0x558c574d38f0,0x558c57819d50,0x558c57819f10,0x558c57880540,0x558c57880620,0x558c57880700,0x558c578807e0,0x558c578808c0,0x558c578809a0,0x558c57880e70,0x558c578de0e0,0x558c578de2a0,0x558c578de460,0x558c578de620,0x558c5794c150,0x558c5794c230,0x558c5794c2a0,0x558c5794c460,0x558c5794ce00 2019-12-07T19:34:34.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:19#29 = 0 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=57) prio 5 cost 1048576 e57) queued 2019-12-07T19:34:34.619+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 53870 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:34.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58748300 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58748300 onodes shared_blobs 0x558c54cf8700,0x558c54d3df80,0x558c54d75180,0x558c54d75340,0x558c57396150,0x558c573963f0,0x558c574d32d0,0x558c574d3730,0x558c574d3880,0x558c574d38f0,0x558c57819d50,0x558c57819f10,0x558c57880540,0x558c57880620,0x558c57880700,0x558c578807e0,0x558c578808c0,0x558c578809a0,0x558c57880e70,0x558c578de0e0,0x558c578de2a0,0x558c578de460,0x558c578de620,0x558c5794c150,0x558c5794c230,0x558c5794c2a0,0x558c5794c460,0x558c5794ce00 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 86638 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2958 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2959 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2950 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2951 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2947 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2946 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295f is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2961 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2960 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294e is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2957 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2956 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2948 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2949 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294a is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294b is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294c is empty 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 119406 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294d is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x294f is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2955 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2954 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2953 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2952 is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295d is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295c is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295b is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295a is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x295e is empty 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58748300 allocated 0x[] released 0x[2a6000~10000,350000~27000] 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x2a6000~10000 2019-12-07T19:34:34.619+0000 7f8f06986700 10 freelist release 0x2a6000~10000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 152174 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv release 0x350000~27000 2019-12-07T19:34:34.619+0000 7f8f06986700 10 freelist release 0x350000~27000 2019-12-07T19:34:34.619+0000 7f8f06986700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:34.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748300 prepare 2019-12-07T19:34:34.619+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58748300 2019-12-07T19:34:34.619+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748300 io_done 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 184942 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c5511e420 finish 2019-12-07T19:34:34.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=57) prio 5 cost 1048576 e57) pg 0x558c57676000 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 217710 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 250478 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 15 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 255553 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573900 had 1 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 255576 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 255576 < min_flush_size 524288 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3bad92~3e658 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3bad92 0x3bad92~3e658 to file(ino 16 size 0x3c0000 mtime 2019-12-07T19:34:33.730537+0000 allocated 3c0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~390000]) 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _allocate len 0x393ea from 0 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _allocate len 0x393ea from 1 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x40000/10000,40000,acd4f0000 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd4f0000~40000/10000,40000,acd4f0000 2019-12-07T19:34:34.619+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x400000 to include allocated 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 70 (was clean) 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3d0000 x_off 0x38ad92 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd92 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3ea and padding block with 0xc16 2019-12-07T19:34:34.619+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x400000 2019-12-07T19:34:34.619+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:34.619+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.619+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:1b 15,1b 2019-12-07T19:34:34.619+0000 7f8f0898a700 20 snap_mapper.set_snaps 1:10779340:::smithi13140796-32:1b 1b 2019-12-07T19:34:34.619+0000 7f8f0898a700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:34.619+0000 7f8f0898a700 20 snap_mapper.update_snaps rm SNA_1_0000000000000015_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54ce5640) upper_bound after SNA_1_0000000000000015_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.0_0000000000000001.C' 2019-12-07T19:34:34.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 293 ==== MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=110, reqid=osd.4.0:110, at_version=57'41, trim_to=0'0, roll_forward_to=53'37)) v2 ==== 2656+0+0 (unknown 0 0 0) 0x558c58535000 con 0x558c54c5b400 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c54ce5640) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' 2019-12-07T19:34:34.619+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.000031 epoch 57 MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=110, reqid=osd.4.0:110, at_version=57'41, trim_to=0'0, roll_forward_to=53'37)) v2 2019-12-07T19:34:34.619+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=110, reqid=osd.4.0:110, at_version=57'41, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000015_.0_0000000000000001.C) returns 0 SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:34.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 54'51 (0'0,54'51] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,54'51], crt=54'51) [57'52 (54'50) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:34.618826+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'53 (54'51) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:34.618826+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim trimq=[15~1] ps=[e~1,1a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=110, reqid=osd.4.0:110, at_version=57'41, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim ps=[e~1,1a~1]] SnapTrimmer state: adding snap 15 to purged_snaps 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=110, reqid=osd.4.0:110, at_version=57'41, trim_to=0'0, roll_forward_to=53'37)) v2) prio 127 cost 0 e57) pg 0x558c575ef000 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim ps=[e~1,15~1,1a~1]] SnapTrimmer state: purged_snaps now [e~1,15~1,1a~1], snap_trimq now [] 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 osd.7 57 dequeue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=110, reqid=osd.4.0:110, at_version=57'41, trim_to=0'0, roll_forward_to=53'37)) v2 pg pg[1.bs1( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57bd2c00 seq 60 2019-12-07T19:34:34.619+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=110, reqid=osd.4.0:110, at_version=57'41, trim_to=0'0, roll_forward_to=53'37)) v2 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 snap_mapper.remove_oid 1:db85c185:::smithi13140796-18:15 2019-12-07T19:34:34.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'52 (0'0,57'52] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'51 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'52 (54'50) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:34.618826+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 snap_mapper._remove_oid 1:db85c185:::smithi13140796-18:15 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd2c00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd2c00 onodes shared_blobs 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd2c00 allocated 0x[] released 0x[] 2019-12-07T19:34:34.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2c00 prepare 2019-12-07T19:34:34.619+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd2c00 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2c00 io_done 2019-12-07T19:34:34.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'53 (0'0,57'53] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'51 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'53 (54'51) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:34.618826+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 snap_mapper.get_snaps 1:db85c185:::smithi13140796-18:15 15 2019-12-07T19:34:34.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim ps=[e~1,15~1,1a~1]] share_pg_info 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 snap_mapper.clear_snaps 1:db85c185:::smithi13140796-18:15 2019-12-07T19:34:34.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'53 (0'0,57'53] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'51 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 53 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 snap_mapper.clear_snaps rm OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:34.619+0000 7f8f0317f700 20 snap_mapper._remove_oid rm SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:34.619+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'53 (0'0,57'53] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'51 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.619+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e57/57) v1 -- 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:34.619+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'52, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'39 (0'0,57'39] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'39], crt=53'37) [57'40 (53'36) delete 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:34.619978+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'41 (53'37) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:34.619978+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e57/57) v1 -- 0x558c57a54300 con 0x558c54c19800 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'40 (0'0,57'40] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'39 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'40 (53'36) delete 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:34.619978+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'39 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'41 (53'37) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:34.619978+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'39 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 41 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'39 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.619+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.619+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e57/57) v1 -- 0x558c57a52700 con 0x558c54c5bc00 2019-12-07T19:34:34.619+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'40, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e57/57) v1 -- 0x558c57a52000 con 0x558c54c5b400 2019-12-07T19:34:34.620+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e57/57) v1 -- 0x558c57a55100 con 0x558c54c19c00 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c578a7200 seq 50 2019-12-07T19:34:34.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim ps=[e~1,15~1,1a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# 1 keys = 0 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean+snaptrim ps=[e~1,15~1,1a~1]] exit Trimming 2019-12-07T19:34:34.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:34.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a2000 seq 49 2019-12-07T19:34:34.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.620+0000 7f8f0297e700 15 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:191 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:db85c185:::smithi13140796-18:15#28 key 0x818000000000000001db85c18521736d'ithi13140796-18!='0x00000000000000150000000000000028'o' 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] enter NotTrimming 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- OBJ_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:34.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# -> 1#1:db85c185:::smithi13140796-18:15#28 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.80EE9C20.1b.smithi13140796-32..' <- SNA_1_0000000000000015_.5_0000000000000001.80EE9C20.1b.smithi13140796-32.. 2019-12-07T19:34:34.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.bs1_head 1#1:db85c185:::smithi13140796-18:15# -> 1#1:db85c185:::smithi13140796-18:15#28 = 0 2019-12-07T19:34:34.620+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:34.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:34:34.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] snap_trimmer complete 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000057.00000000000000000052' <- 0000000057.00000000000000000052 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000057.00000000000000000053' <- 0000000057.00000000000000000053 2019-12-07T19:34:34.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- OBJ_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a7200 cost 8335 (1 ios * 4000 + 4335 bytes) 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a7200 onodes 0x558c5840c500,0x558c5840c780 shared_blobs 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18..' <- SNA_1_0000000000000015_.1_0000000000000001.BD1A381A.15.smithi13140796-18.. 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c608) update 5#1:10779340:::smithi13140796-32:1b# 2019-12-07T19:34:34.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000057.00000000000000000040' <- 0000000057.00000000000000000040 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:1b# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000057.00000000000000000041' <- 0000000057.00000000000000000041 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a2000 cost 7533 (1 ios * 4000 + 3533 bytes) 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a7200 allocated 0x[] released 0x[] 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a2000 onodes 0x558c57573680,0x558c57e8ec80 shared_blobs 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57573788) update 1#1:db85c185:::smithi13140796-18:15#28 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:15#28 is 664 (412 bytes onode + 2 bytes spanning blobs + 250 bytes inline extents) 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57e8ed88) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7200 prepare 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a7200 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7200 io_done 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 1515 (486 bytes onode + 2 bytes spanning blobs + 1027 bytes inline extents) 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a2000 allocated 0x[] released 0x[] 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2000 prepare 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a2000 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2000 io_done 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'53 (0'0,57'53] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'51 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] op_applied version 57'53 2019-12-07T19:34:34.620+0000 7f8f0898a700 10 osd.7 57 dequeue_op 0x558c5511ef20 finish 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'39 crt=53'37 lcod 53'37 active mbc={} ps=[e~1,1a~1]] op_applied version 57'41 2019-12-07T19:34:34.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.620+0000 7f8f0317f700 10 osd.7 57 dequeue_op 0x558c581ccdc0 finish 2019-12-07T19:34:34.620+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3f93ea 2019-12-07T19:34:34.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 275 ==== MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=153, reqid=osd.3.0:153, at_version=57'71, trim_to=0'0, roll_forward_to=53'69)) v2 ==== 2656+0+0 (crc 0 0 0) 0x558c58535800 con 0x558c54c5bc00 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.620+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581cc840 prio 127 cost 0 latency 0.000021 epoch 57 MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=153, reqid=osd.3.0:153, at_version=57'71, trim_to=0'0, roll_forward_to=53'69)) v2 2019-12-07T19:34:34.620+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=153, reqid=osd.3.0:153, at_version=57'71, trim_to=0'0, roll_forward_to=53'69)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=153, reqid=osd.3.0:153, at_version=57'71, trim_to=0'0, roll_forward_to=53'69)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=153, reqid=osd.3.0:153, at_version=57'71, trim_to=0'0, roll_forward_to=53'69)) v2) prio 127 cost 0 e57) pg 0x558c54d5c000 2019-12-07T19:34:34.620+0000 7f8f05984700 10 osd.7 57 dequeue_op 0x558c581cc840 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=153, reqid=osd.3.0:153, at_version=57'71, trim_to=0'0, roll_forward_to=53'69)) v2 pg pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.620+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=153, reqid=osd.3.0:153, at_version=57'71, trim_to=0'0, roll_forward_to=53'69)) v2 2019-12-07T19:34:34.620+0000 7f8f05984700 20 snap_mapper.remove_oid 1:6336c6f2:::smithi13140796-16:17 2019-12-07T19:34:34.620+0000 7f8f05984700 20 snap_mapper._remove_oid 1:6336c6f2:::smithi13140796-16:17 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (70) on file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]), flushing log 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:34.620+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.620+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush cnt:2 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 70 len 0x4a crc 0xafc96ae6) 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x45000~1000 to file(ino 1 size 0x45000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x45000 0x45000~1000 to file(ino 1 size 0x45000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x46000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x45000 2019-12-07T19:34:34.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x46000 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:34.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:34.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 276 ==== MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58513e00 con 0x558c54c5bc00 2019-12-07T19:34:34.620+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000013 epoch 57 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.620+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) queued 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) pg 0x558c57676000 2019-12-07T19:34:34.620+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000188 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:34.620+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.620+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c581ccb00 finish 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 294 ==== MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d76000 con 0x558c54c5b400 2019-12-07T19:34:34.620+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581ccc60 prio 196 cost 0 latency 0.000011 epoch 57 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.620+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) queued 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) pg 0x558c57676000 2019-12-07T19:34:34.620+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c581ccc60 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:34.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.620+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c581ccc60 finish 2019-12-07T19:34:34.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 108 ==== MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57950400 con 0x558c54c19c00 2019-12-07T19:34:34.620+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511fb80 prio 196 cost 0 latency 0.000011 epoch 57 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) queued 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) pg 0x558c57676000 2019-12-07T19:34:34.620+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c5511fb80 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:34.620+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.620+0000 7f8f06986700 10 osd.7 57 dequeue_op 0x558c5511fb80 finish 2019-12-07T19:34:34.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 470 ==== MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=282, reqid=osd.1.0:282, at_version=57'55, trim_to=0'0, roll_forward_to=54'51)) v2 ==== 2648+0+0 (crc 0 0 0) 0x558c58532000 con 0x558c54c64000 2019-12-07T19:34:34.621+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511fe40 prio 127 cost 0 latency 0.000030 epoch 57 MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=282, reqid=osd.1.0:282, at_version=57'55, trim_to=0'0, roll_forward_to=54'51)) v2 2019-12-07T19:34:34.621+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=282, reqid=osd.1.0:282, at_version=57'55, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 192 ==== MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=77, reqid=osd.2.0:77, at_version=57'30, trim_to=0'0, roll_forward_to=55'28)) v2 ==== 3506+0+0 (crc 0 0 0) 0x558c58534800 con 0x558c54c19800 2019-12-07T19:34:34.621+0000 7f8f24c58700 15 osd.7 57 enqueue_op 0x558c57335e40 prio 127 cost 0 latency 0.000031 epoch 57 MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=77, reqid=osd.2.0:77, at_version=57'30, trim_to=0'0, roll_forward_to=55'28)) v2 2019-12-07T19:34:34.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=77, reqid=osd.2.0:77, at_version=57'30, trim_to=0'0, roll_forward_to=55'28)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=77, reqid=osd.2.0:77, at_version=57'30, trim_to=0'0, roll_forward_to=55'28)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=77, reqid=osd.2.0:77, at_version=57'30, trim_to=0'0, roll_forward_to=55'28)) v2) prio 127 cost 0 e57) pg 0x558c55141000 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 osd.7 57 dequeue_op 0x558c57335e40 prio 127 cost 0 latency 0.000082 MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=77, reqid=osd.2.0:77, at_version=57'30, trim_to=0'0, roll_forward_to=55'28)) v2 pg pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=77, reqid=osd.2.0:77, at_version=57'30, trim_to=0'0, roll_forward_to=55'28)) v2 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 snap_mapper.update_snaps 1:bca18876:::smithi13140796-30:1b 1b was 2019-12-07T19:34:34.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush cnt:2 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=282, reqid=osd.1.0:282, at_version=57'55, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000987 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=282, reqid=osd.1.0:282, at_version=57'55, trim_to=0'0, roll_forward_to=54'51)) v2) prio 127 cost 0 e57) pg 0x558c57380000 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 osd.7 57 dequeue_op 0x558c5511fe40 prio 127 cost 0 latency 0.000234 MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=282, reqid=osd.1.0:282, at_version=57'55, trim_to=0'0, roll_forward_to=54'51)) v2 pg pg[1.8s5( v 57'53 (0'0,57'53] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'53 (0'0,57'53] local-lis/les=41/42 n=8 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=282, reqid=osd.1.0:282, at_version=57'55, trim_to=0'0, roll_forward_to=54'51)) v2 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 snap_mapper.remove_oid 1:131421e8:::smithi13140796-7:15 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 snap_mapper._remove_oid 1:131421e8:::smithi13140796-7:15 2019-12-07T19:34:34.621+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush cnt:2 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 70 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.002222963s (0.000000122s flush + 0.002222841s kv commit) 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5810fb00,0x558c579d2f00> 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1614 < min_flush_size 524288 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810fb00 kv_submitted 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5810fb00 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5810fb00 finishing 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5810fb00 onodes 0x558c574a0a00,0x558c574a0c80 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5810fb00 done 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a2000 kv_queued 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5810fb00 [] 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2f00 kv_submitted 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c579d2f00 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2f00 finishing 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c579d2f00 onodes 0x558c57573900,0x558c57a35400 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c579d2f00 done 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c579d2f00 [] 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 2 2019-12-07T19:34:34.621+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 57/41 ECSubWriteReply(tid=281, last_complete=57'44, committed=1, applied=1)) v2 -- 0x558c57950600 con 0x558c54c64000 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295180 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2740 < min_flush_size 524288 2019-12-07T19:34:34.621+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 57/41 ECSubWriteReply(tid=109, last_complete=57'39, committed=1, applied=1)) v2 -- 0x558c54d77c00 con 0x558c54c5b400 2019-12-07T19:34:34.621+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:34.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6461 < min_flush_size 524288 2019-12-07T19:34:34.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c500 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 11087 < min_flush_size 524288 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573680 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8ec80 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57c05180 had 1 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 11110 < min_flush_size 524288 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 11110 < min_flush_size 524288 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3f93ea~2b66 to file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3f93ea 0x3f93ea~2b66 to file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3d0000 x_off 0x3c93ea 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3ea 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf50 and padding block with 0xb0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 snap_mapper.get_snaps 1:6336c6f2:::smithi13140796-16:17 15 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 snap_mapper.clear_snaps 1:6336c6f2:::smithi13140796-16:17 2019-12-07T19:34:34.621+0000 7f8f05984700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 snap_mapper.get_snaps 1:bca18876:::smithi13140796-30:1b 15,1b 2019-12-07T19:34:34.621+0000 7f8f05984700 20 snap_mapper._remove_oid rm SNA_1_0000000000000015_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 snap_mapper.set_snaps 1:bca18876:::smithi13140796-30:1b 1b 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 snap_mapper.update_snaps rm SNA_1_0000000000000015_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:34.621+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 53'69 (0'0,53'69] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,53'69], crt=53'69) [57'70 (53'68) delete 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:34.619778+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'71 (53'69) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:34.619778+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.621+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'70 (0'0,57'70] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'69 crt=53'69 lcod 53'69 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'70 (53'68) delete 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:34.619778+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.621+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'69 crt=53'69 lcod 53'69 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'71 (53'69) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:34.619778+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.621+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'69 crt=53'69 lcod 53'69 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 71 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 55'28 (0'0,55'28] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,55'28], crt=55'28) [57'29 (54'23) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:34.620843+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'30 (55'28) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:34.620843+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.621+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'69 crt=53'69 lcod 53'69 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.621+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.621+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'70, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'29 (0'0,57'29] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'28 crt=55'28 lcod 55'28 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'29 (54'23) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:34.620843+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'28 crt=55'28 lcod 55'28 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'30 (55'28) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:34.620843+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'28 crt=55'28 lcod 55'28 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 30 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'28 crt=55'28 lcod 55'28 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.621+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'29, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c575c5500 seq 107 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6336c6f2:::smithi13140796-16:17#46 key 0x8280000000000000016336c6f221736d'ithi13140796-16!='0x00000000000000170000000000000046'o' 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:34.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# -> 2#1:6336c6f2:::smithi13140796-16:17#46 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3fbf50 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17# -> 2#1:6336c6f2:::smithi13140796-16:17#46 = 0 2019-12-07T19:34:34.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000319765s (0.000000071s flush + 0.000319694s kv commit) 2019-12-07T19:34:34.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.621+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58541800 seq 64 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# 1 keys = 0 2019-12-07T19:34:34.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:34.621+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.621+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 193 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c58537500 con 0x558c54c19800 2019-12-07T19:34:34.621+0000 7f8f24c58700 15 osd.7 57 enqueue_peering_evt 1.5s4 epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:34.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58748300,0x558c57bd2c00,0x558c578a7200,0x558c586a2000> 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748300 kv_submitted 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58748300 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748300 finishing 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58748300 onodes 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58748300 done 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2c00 kv_submitted 2019-12-07T19:34:34.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58748300 [2a6000~10000,350000~27000] 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x2a6000~10000 2019-12-07T19:34:34.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release 0x350000~27000 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s)) prio 255 cost 10 e57) pg 0x558c57522000 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794ce00) put 0x558c5794ce00 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794ce18 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:15 15 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e142d0 space 0x558c5794ce18 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 snap_mapper.clear_snaps 1:131421e8:::smithi13140796-7:15 2019-12-07T19:34:34.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 194 ==== MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b9e00 con 0x558c54c19800 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794c460) put 0x558c5794c460 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c478 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e14510 space 0x558c5794c478 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 snap_mapper.clear_snaps rm OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:34.621+0000 7f8f24c58700 15 osd.7 57 enqueue_op 0x558c57334f20 prio 196 cost 0 latency 0.000011 epoch 57 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794c2a0) put 0x558c5794c2a0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c2b8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e14480 space 0x558c5794c2b8 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- OBJ_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:34.621+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.081869114s ub 67.081869114s int 16s) 2019-12-07T19:34:34.621+0000 7f8f0898a700 20 snap_mapper._remove_oid rm SNA_1_0000000000000015_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794c230) put 0x558c5794c230 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c248 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e143f0 space 0x558c5794c248 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c5794c150) put 0x558c5794c150 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794c168 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e14360 space 0x558c5794c168 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de620) put 0x558c578de620 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de638 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.2_0000000000000001.6CC636F4.17.smithi13140796-16..' <- SNA_1_0000000000000015_.2_0000000000000001.6CC636F4.17.smithi13140796-16.. 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbb9e0 space 0x558c578de638 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de460) put 0x558c578de460 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de478 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e141b0 space 0x558c578de478 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de2a0) put 0x558c578de2a0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de2b8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e14120 space 0x558c578de2b8 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578de0e0) put 0x558c578de0e0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000057.00000000000000000070' <- 0000000057.00000000000000000070 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'53 (0'0,57'53] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'53], crt=54'51) [57'54 (53'48) delete 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:34.621300+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'55 (53'49) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:34.621300+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5853a900 seq 23 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578de0f8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e14090 space 0x558c578de0f8 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57880e70) put 0x558c57880e70 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880e88 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000057.00000000000000000071' <- 0000000057.00000000000000000071 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbef0 space 0x558c57880e88 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578809a0) put 0x558c578809a0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578809b8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbdd0 space 0x558c578809b8 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:34.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578808c0) put 0x558c578808c0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578808d8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbd40 space 0x558c578808d8 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'54 (0'0,57'54] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'53 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'54 (53'48) delete 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:34.621300+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c578807e0) put 0x558c578807e0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c578807f8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbcb0 space 0x558c578807f8 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853a900 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:34.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853a900 onodes shared_blobs 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57880700) put 0x558c57880700 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853a900 allocated 0x[] released 0x[] 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880718 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbc20 space 0x558c57880718 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c5500 cost 7551 (1 ios * 4000 + 3551 bytes) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57880620) put 0x558c57880620 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880638 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'53 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'55 (53'49) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:34.621300+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c5500 onodes 0x558c54ce3900,0x558c57f8e500 shared_blobs 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbb90 space 0x558c57880638 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57880540) put 0x558c57880540 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3a08) update 2#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57880558 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a900 prepare 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbb00 space 0x558c57880558 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853a900 2019-12-07T19:34:34.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a900 io_done 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'53 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 55 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57819f10) put 0x558c57819f10 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:head# is 1507 (486 bytes onode + 2 bytes spanning blobs + 1019 bytes inline extents) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819f28 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e14000 space 0x558c57819f28 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57819d50) put 0x558c57819d50 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57819d68 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore.extentmap(0x558c57f8e608) update 2#1:6336c6f2:::smithi13140796-16:17#46 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbba70 space 0x558c57819d68 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'53 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574d38f0) put 0x558c574d38f0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3908 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c57982480 space 0x558c574d3908 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6336c6f2:::smithi13140796-16:17#46 is 637 (412 bytes onode + 2 bytes spanning blobs + 223 bytes inline extents) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574d3880) put 0x558c574d3880 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c5500 allocated 0x[] released 0x[] 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3898 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbb950 space 0x558c574d3898 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574d3730) put 0x558c574d3730 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3748 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c579823f0 space 0x558c574d3748 0x0~1000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c574d32d0) put 0x558c574d32d0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'54, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d32e8 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5500 prepare 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e145a0 space 0x558c574d32e8 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c5500 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c573963f0) put 0x558c573963f0 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5500 io_done 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396408 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c579822d0 space 0x558c57396408 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c57396150) put 0x558c57396150 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57396168 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c57982360 space 0x558c57396168 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d75340) put 0x558c54d75340 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75358 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c57982240 space 0x558c54d75358 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d75180) put 0x558c54d75180 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75198 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54dbbe60 space 0x558c54d75198 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54d3df80) put 0x558c54d3df80 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d3df98 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.621+0000 7f8f199ac700 20 _rm buffer(0x558c54e14240 space 0x558c54d3df98 0x0~2000 clean) 2019-12-07T19:34:34.621+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=53'69 crt=53'69 lcod 53'69 active mbc={} ps=[e~1,1a~1]] op_applied version 57'71 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore.sharedblob(0x558c54cf8700) put 0x558c54cf8700 removing self from set 0x558c54d380d0 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54cf8718 in 0x558c53a59ce0) _clear 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 _rm buffer(0x558c579821b0 space 0x558c54cf8718 0x0~2000 clean) 2019-12-07T19:34:34.622+0000 7f8f05984700 10 osd.7 57 dequeue_op 0x558c581cc840 finish 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2c00 kv_submitted 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd2c00 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2c00 finishing 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd2c00 onodes 2019-12-07T19:34:34.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2c00 done 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd2c00 [] 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7200 kv_submitted 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a7200 2019-12-07T19:34:34.622+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7200 finishing 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a7200 onodes 0x558c5840c500,0x558c5840c780 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a7200 done 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- OBJ_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a7200 [] 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2000 kv_submitted 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a2000 2019-12-07T19:34:34.622+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2000 finishing 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a2000 onodes 0x558c57573680,0x558c57e8ec80 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a2000 done 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.2_0000000000000001.D35811E6.1b.smithi13140796-30..' <- SNA_1_0000000000000015_.2_0000000000000001.D35811E6.1b.smithi13140796-30.. 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a2000 [] 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c578a7500 seq 51 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.622+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000057.00000000000000000029' <- 0000000057.00000000000000000029 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) get_onode oid 5#1:131421e8:::smithi13140796-7:15#36 key 0x858000000000000001131421e821736d'ithi13140796-7!='0x00000000000000150000000000000036'o' 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000057.00000000000000000030' <- 0000000057.00000000000000000030 2019-12-07T19:34:34.622+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 57/41 ECSubWriteReply(tid=110, last_complete=57'41, committed=1, applied=1)) v2 -- 0x558c5763ca00 con 0x558c54c5b400 2019-12-07T19:34:34.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.8s5_head 0x558c54df6960) r -2 v.len 0 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:34.622+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# -> 5#1:131421e8:::smithi13140796-7:15#36 2019-12-07T19:34:34.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58541800 cost 8479 (1 ios * 4000 + 4479 bytes) 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58541800 onodes 0x558c54d96780,0x558c585cf180 shared_blobs 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54d96888) update 2#1:bca18876:::smithi13140796-30:1b# 2019-12-07T19:34:34.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:15# -> 5#1:131421e8:::smithi13140796-7:15#36 = 0 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:1b# is 826 (412 bytes onode + 2 bytes spanning blobs + 412 bytes inline extents) 2019-12-07T19:34:34.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) queued 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore.extentmap(0x558c585cf288) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:34.622+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:34.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 628 (626 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:34.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) pg 0x558c57676000 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58541800 allocated 0x[] released 0x[] 2019-12-07T19:34:34.622+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541800 prepare 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58541800 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541800 io_done 2019-12-07T19:34:34.622+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.622+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c57334f20 prio 196 cost 0 latency 0.000197 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- OBJ_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=55'28 crt=55'28 lcod 55'28 active mbc={} ps=[e~1,1a~1]] op_applied version 57'30 2019-12-07T19:34:34.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.622+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c57334f20 finish 2019-12-07T19:34:34.622+0000 7f8f0217d700 10 osd.7 57 dequeue_op 0x558c57335e40 finish 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.622+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.8C824871.15.smithi13140796-7..' <- SNA_1_0000000000000015_.5_0000000000000001.8C824871.15.smithi13140796-7.. 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.622+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000057.00000000000000000054' <- 0000000057.00000000000000000054 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000057.00000000000000000055' <- 0000000057.00000000000000000055 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:34.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a7500 cost 7524 (1 ios * 4000 + 3524 bytes) 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a7500 onodes 0x558c57598000,0x558c575f9b80 shared_blobs 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore.extentmap(0x558c57598108) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 922 (484 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore.extentmap(0x558c575f9c88) update 5#1:131421e8:::smithi13140796-7:15#36 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:131421e8:::smithi13140796-7:15#36 is 849 (411 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a7500 allocated 0x[] released 0x[] 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7500 prepare 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a7500 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7500 io_done 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'53 crt=54'51 lcod 54'51 active mbc={} ps=[e~1,1a~1]] op_applied version 57'55 2019-12-07T19:34:34.622+0000 7f8f0898a700 10 osd.7 57 dequeue_op 0x558c5511fe40 finish 2019-12-07T19:34:34.622+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 57/41 ECSubWriteReply(tid=280, last_complete=57'53, committed=1, applied=1)) v2 -- 0x558c5795dc00 con 0x558c54c64000 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5765 < min_flush_size 524288 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3900 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e500 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 3 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58093680 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10110 < min_flush_size 524288 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d96780 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585cf180 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14320 < min_flush_size 524288 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57598000 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9b80 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840ca00 had 1 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14343 < min_flush_size 524288 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14343 < min_flush_size 524288 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3fbf50~3807 to file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3fbf50 0x3fbf50~3807 to file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3d0000 x_off 0x3cbf50 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf50 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x757 and padding block with 0x8a9 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x3ff757 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000205040s (0.000000119s flush + 0.000204921s kv commit) 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853a900,0x558c575c5500,0x558c58541800,0x558c578a7500> 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a900 kv_submitted 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853a900 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853a900 finishing 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853a900 onodes 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853a900 done 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853a900 [] 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5500 kv_submitted 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c5500 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5500 finishing 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c5500 onodes 0x558c54ce3900,0x558c57f8e500 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c5500 done 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c5500 [] 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.622+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541800 kv_submitted 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58541800 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58541800 finishing 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58541800 onodes 0x558c54d96780,0x558c585cf180 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58541800 done 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58541800 [] 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7500 kv_submitted 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a7500 2019-12-07T19:34:34.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7500 finishing 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a7500 onodes 0x558c57598000,0x558c575f9b80 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a7500 done 2019-12-07T19:34:34.622+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 57/28 ECSubWriteReply(tid=153, last_complete=57'71, committed=1, applied=1)) v2 -- 0x558c57951600 con 0x558c54c5bc00 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a7500 [] 2019-12-07T19:34:34.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.622+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 57/41 ECSubWriteReply(tid=77, last_complete=57'30, committed=1, applied=1)) v2 -- 0x558c54c59200 con 0x558c54c19800 2019-12-07T19:34:34.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.622+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.622+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 57/41 ECSubWriteReply(tid=282, last_complete=57'55, committed=1, applied=1)) v2 -- 0x558c5795d800 con 0x558c54c64000 2019-12-07T19:34:34.622+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 471 ==== MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d800 con 0x558c54c64000 2019-12-07T19:34:34.623+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511e840 prio 196 cost 0 latency 0.000011 epoch 57 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.623+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) 2019-12-07T19:34:34.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) queued 2019-12-07T19:34:34.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2) prio 196 cost 0 e57) pg 0x558c57676000 2019-12-07T19:34:34.623+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c5511e840 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:34.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 57/28 ECSubWriteReply(tid=48, last_complete=57'42, committed=1, applied=1)) v2 2019-12-07T19:34:34.623+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:19 v=0'0 tt=0'0 tid=48 reqid=osd.7.0:47 roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:34.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:34.623+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:34.623+0000 7f8f0297e700 10 osd.7 57 dequeue_op 0x558c5511e840 finish 2019-12-07T19:34:34.623+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:34.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 277 ==== MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=154, reqid=osd.3.0:154, at_version=57'17, trim_to=0'0, roll_forward_to=54'14)) v2 ==== 3362+0+0 (crc 0 0 0) 0x558c575bf000 con 0x558c54c5bc00 2019-12-07T19:34:34.623+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581cc9a0 prio 127 cost 0 latency 0.000029 epoch 57 MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=154, reqid=osd.3.0:154, at_version=57'17, trim_to=0'0, roll_forward_to=54'14)) v2 2019-12-07T19:34:34.623+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=154, reqid=osd.3.0:154, at_version=57'17, trim_to=0'0, roll_forward_to=54'14)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=154, reqid=osd.3.0:154, at_version=57'17, trim_to=0'0, roll_forward_to=54'14)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=154, reqid=osd.3.0:154, at_version=57'17, trim_to=0'0, roll_forward_to=54'14)) v2) prio 127 cost 0 e57) pg 0x558c57383000 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 57 dequeue_op 0x558c581cc9a0 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=154, reqid=osd.3.0:154, at_version=57'17, trim_to=0'0, roll_forward_to=54'14)) v2 pg pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=154, reqid=osd.3.0:154, at_version=57'17, trim_to=0'0, roll_forward_to=54'14)) v2 2019-12-07T19:34:34.623+0000 7f8f05183700 20 snap_mapper.update_snaps 1:fc3a86a2:::smithi13140796-33:1b 1b was 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 20 snap_mapper.get_snaps 1:fc3a86a2:::smithi13140796-33:1b 15,1b 2019-12-07T19:34:34.623+0000 7f8f05183700 20 snap_mapper.set_snaps 1:fc3a86a2:::smithi13140796-33:1b 1b 2019-12-07T19:34:34.623+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:34.623+0000 7f8f05183700 20 snap_mapper.update_snaps rm SNA_1_0000000000000015_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 55'15 (0'0,55'15] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,55'15], crt=54'14) [57'16 (54'13) modify 1:fc3a86a2:::smithi13140796-33:1b by unknown.0.0:0 2019-12-07T19:34:34.624151+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'17 (54'14) modify 1:fc3a86a2:::smithi13140796-33:head by unknown.0.0:0 2019-12-07T19:34:34.624151+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'16 (0'0,57'16] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=55'15 crt=54'14 lcod 55'15 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'16 (54'13) modify 1:fc3a86a2:::smithi13140796-33:1b by unknown.0.0:0 2019-12-07T19:34:34.624151+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=55'15 crt=54'14 lcod 55'15 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'17 (54'14) modify 1:fc3a86a2:::smithi13140796-33:head by unknown.0.0:0 2019-12-07T19:34:34.624151+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=55'15 crt=54'14 lcod 55'15 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 17 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=55'15 crt=54'14 lcod 55'15 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.623+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.623+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c583b6600 seq 29 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# 1 keys 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# 1 keys = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# hinfo_key (42 bytes) 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2 keys 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2 keys = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.F3C51654.1b.smithi13140796-33..' <- OBJ_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.F3C51654.1b.smithi13140796-33..' <- SNA_1_0000000000000015_.5_0000000000000001.F3C51654.1b.smithi13140796-33.. 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000057.00000000000000000016' <- 0000000057.00000000000000000016 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.0000000057.00000000000000000017' <- 0000000057.00000000000000000017 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b6600 cost 8335 (1 ios * 4000 + 4335 bytes) 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b6600 onodes 0x558c5795a280,0x558c5795a500 shared_blobs 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore.extentmap(0x558c5795a388) update 5#1:fc3a86a2:::smithi13140796-33:1b# 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:1b# is 877 (412 bytes onode + 2 bytes spanning blobs + 463 bytes inline extents) 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore.extentmap(0x558c5795a608) update 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:head# is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b6600 allocated 0x[] released 0x[] 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6600 prepare 2019-12-07T19:34:34.623+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b6600 2019-12-07T19:34:34.623+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6600 io_done 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 lua=55'15 crt=54'14 lcod 55'15 active mbc={} ps=[e~1,1a~1]] op_applied version 57'17 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2217 < min_flush_size 524288 2019-12-07T19:34:34.623+0000 7f8f05183700 10 osd.7 57 dequeue_op 0x558c581cc9a0 finish 2019-12-07T19:34:34.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4182 < min_flush_size 524288 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795a280 had 1 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795a500 had 1 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4205 < min_flush_size 524288 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4205 < min_flush_size 524288 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x3ff757~106d to file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x3ff757 0x3ff757~106d to file(ino 16 size 0x400000 mtime 2019-12-07T19:34:34.620788+0000 allocated 400000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3d0000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _allocate len 0x7c4 from 0 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _allocate len 0x7c4 from 1 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd530000 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd530000~10000/10000,10000,acd530000 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x410000 to include allocated 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 71 (was clean) 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3cf757 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x757 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7c4 and padding block with 0x83c 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x410000 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4007c4 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (71) on file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]), flushing log 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 71 len 0x4a crc 0xce00edb1) 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x46000~1000 to file(ino 1 size 0x46000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x46000 0x46000~1000 to file(ino 1 size 0x46000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x47000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x46000 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x47000 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:34.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000020 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 71 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000447865s (0.000000096s flush + 0.000447769s kv commit) 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583b6600> 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6600 kv_submitted 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b6600 2019-12-07T19:34:34.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6600 finishing 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b6600 onodes 0x558c5795a280,0x558c5795a500 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b6600 done 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:34.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b6600 [] 2019-12-07T19:34:34.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.623+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 57/41 ECSubWriteReply(tid=154, last_complete=57'17, committed=1, applied=1)) v2 -- 0x558c58513600 con 0x558c54c5bc00 2019-12-07T19:34:34.624+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 195 ==== MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=78, reqid=osd.2.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=57'30)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c58155800 con 0x558c54c19800 2019-12-07T19:34:34.624+0000 7f8f24c58700 15 osd.7 57 enqueue_op 0x558c57335ce0 prio 127 cost 0 latency 0.000022 epoch 57 MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=78, reqid=osd.2.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=57'30)) v2 2019-12-07T19:34:34.624+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=78, reqid=osd.2.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=57'30)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=78, reqid=osd.2.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=57'30)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=78, reqid=osd.2.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=57'30)) v2) prio 127 cost 0 e57) pg 0x558c55141000 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 osd.7 57 dequeue_op 0x558c57335ce0 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=78, reqid=osd.2.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=57'30)) v2 pg pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.ds2 57/41 ECSubWrite(tid=78, reqid=osd.2.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=57'30)) v2 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=55'28 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'30], crt=55'28) [] 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'29 (54'23) modify 1:bca18876:::smithi13140796-30:1b by unknown.0.0:0 2019-12-07T19:34:34.620843+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'30 (55'28) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:34.620843+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 30 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.624+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58756000 seq 65 2019-12-07T19:34:34.624+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58756000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58756000 onodes shared_blobs 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58756000 allocated 0x[] released 0x[] 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756000 prepare 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58756000 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756000 io_done 2019-12-07T19:34:34.624+0000 7f8f0217d700 10 osd.7 57 dequeue_op 0x558c57335ce0 finish 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4007c4~471 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4007c4 0x4007c4~471 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d07c4 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7c4 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc35 and padding block with 0x3cb 2019-12-07T19:34:34.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 196 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c58537500 con 0x558c54c19800 2019-12-07T19:34:34.624+0000 7f8f24c58700 15 osd.7 57 enqueue_peering_evt 1.ds2 epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:34.624+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s)) prio 255 cost 10 e57) pg 0x558c55141000 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x400c35 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.624+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.082631240s ub 67.082631240s int 16s) 2019-12-07T19:34:34.624+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127394s (0.000000105s flush + 0.000127289s kv commit) 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.624+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:34.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:34.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5853ac00 seq 66 2019-12-07T19:34:34.624+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:34.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:34.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:34.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:34.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853ac00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853ac00 onodes shared_blobs 2019-12-07T19:34:34.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:34.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ac00 prepare 2019-12-07T19:34:34.624+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853ac00 2019-12-07T19:34:34.624+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ac00 io_done 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58756000> 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756000 kv_submitted 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58756000 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756000 finishing 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58756000 onodes 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58756000 done 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853ac00 kv_queued 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58756000 [] 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.624+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 57/41 ECSubWriteReply(tid=78, last_complete=57'30, committed=1, applied=1)) v2 -- 0x558c54c58400 con 0x558c54c19800 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x400c35~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x400c35 0x400c35~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d0c35 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc35 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb2 and padding block with 0xf4e 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4010b2 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000134548s (0.000000124s flush + 0.000134424s kv commit) 2019-12-07T19:34:34.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853ac00> 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ac00 kv_submitted 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853ac00 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ac00 finishing 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853ac00 onodes 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853ac00 done 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853ac00 [] 2019-12-07T19:34:34.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.624+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.624+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 472 ==== MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=283, reqid=osd.1.0:282, at_version=0'0, trim_to=0'0, roll_forward_to=57'55)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c58534000 con 0x558c54c64000 2019-12-07T19:34:34.627+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511e9a0 prio 127 cost 0 latency 0.000020 epoch 57 MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=283, reqid=osd.1.0:282, at_version=0'0, trim_to=0'0, roll_forward_to=57'55)) v2 2019-12-07T19:34:34.627+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=283, reqid=osd.1.0:282, at_version=0'0, trim_to=0'0, roll_forward_to=57'55)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 278 ==== MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=156, reqid=osd.3.0:153, at_version=0'0, trim_to=0'0, roll_forward_to=57'71)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e3f000 con 0x558c54c5bc00 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581cd8c0 prio 127 cost 0 latency 0.000018 epoch 57 MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=156, reqid=osd.3.0:153, at_version=0'0, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=283, reqid=osd.1.0:282, at_version=0'0, trim_to=0'0, roll_forward_to=57'55)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.627+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=156, reqid=osd.3.0:153, at_version=0'0, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=283, reqid=osd.1.0:282, at_version=0'0, trim_to=0'0, roll_forward_to=57'55)) v2) prio 127 cost 0 e57) pg 0x558c57380000 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=156, reqid=osd.3.0:153, at_version=0'0, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 279 ==== MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=155, reqid=osd.3.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=57'17)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c54e41000 con 0x558c54c5bc00 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 57 dequeue_op 0x558c5511e9a0 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=283, reqid=osd.1.0:282, at_version=0'0, trim_to=0'0, roll_forward_to=57'55)) v2 pg pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581cc580 prio 127 cost 0 latency 0.000012 epoch 57 MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=155, reqid=osd.3.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=57'17)) v2 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=156, reqid=osd.3.0:153, at_version=0'0, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e57) pg 0x558c54d5c000 2019-12-07T19:34:34.627+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=155, reqid=osd.3.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=57'17)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.8s5 57/41 ECSubWrite(tid=283, reqid=osd.1.0:282, at_version=0'0, trim_to=0'0, roll_forward_to=57'55)) v2 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'51 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'55], crt=54'51) [] 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 osd.7 57 dequeue_op 0x558c581cd8c0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=156, reqid=osd.3.0:153, at_version=0'0, trim_to=0'0, roll_forward_to=57'71)) v2 pg pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.6s2 57/28 ECSubWrite(tid=156, reqid=osd.3.0:153, at_version=0'0, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'52 (54'50) modify 1:10779340:::smithi13140796-32:1b by unknown.0.0:0 2019-12-07T19:34:34.618826+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=53'69 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'71], crt=53'69) [] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'53 (54'51) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:34.618826+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'54 (53'48) delete 1:131421e8:::smithi13140796-7:15 by unknown.0.0:0 2019-12-07T19:34:34.621300+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'70 (53'68) delete 1:6336c6f2:::smithi13140796-16:17 by unknown.0.0:0 2019-12-07T19:34:34.619778+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'55 (53'49) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:34.621300+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 55 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.627+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'71 (53'69) modify 1:6336c6f2:::smithi13140796-16:head by unknown.0.0:0 2019-12-07T19:34:34.619778+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 71 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.627+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5875a000 seq 52 2019-12-07T19:34:34.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 280 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52700 con 0x558c54c5bc00 2019-12-07T19:34:34.627+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:15#36 onode 0x558c575f9b80 txc 0x558c5875a000 2019-12-07T19:34:34.627+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.8s5_head 5#1:131421e8:::smithi13140796-7:15#36 0x0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db4fc0) put_ref 0x0~2000 Blob(0x558c57db4fc0 blob([0x83a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.fs5 epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:34.627+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db4ee0) put_ref 0x0~2000 Blob(0x558c57db4ee0 blob([0x83c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c550b3340) put_ref 0x0~2000 Blob(0x558c550b3340 blob([0x83e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5853e600 seq 108 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db4e00) put_ref 0x0~2000 Blob(0x558c57db4e00 blob([0x840000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db5030) put_ref 0x0~2000 Blob(0x558c57db5030 blob([0x842000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db50a0) put_ref 0x0~2000 Blob(0x558c57db50a0 blob([0x844000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db44d0) put_ref 0x0~2000 Blob(0x558c57db44d0 blob([0x846000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c580ec540) put_ref 0x0~2000 Blob(0x558c580ec540 blob([0x848000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57ed1b90) put_ref 0x0~2000 Blob(0x558c57ed1b90 blob([0x84a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c580ec5b0) put_ref 0x0~2000 Blob(0x558c580ec5b0 blob([0x84c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17#46 onode 0x558c57f8e500 txc 0x558c5853e600 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db43f0) put_ref 0x0~2000 Blob(0x558c57db43f0 blob([0x84e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17#46 0x0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db4540) put_ref 0x0~2000 Blob(0x558c57db4540 blob([0x850000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db45b0) put_ref 0x0~2000 Blob(0x558c57db45b0 blob([0x852000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db4620) put_ref 0x0~2000 Blob(0x558c57db4620 blob([0x854000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c57654f50) put_ref 0x0~2000 Blob(0x558c57654f50 blob([0x63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db4690) put_ref 0x0~2000 Blob(0x558c57db4690 blob([0x856000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.blob(0x558c57db4700) put_ref 0x0~1000 Blob(0x558c57db4700 blob([0x858000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore.extentmap(0x558c575f9c88) dirty_range mark inline shard dirty 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c55060690) put_ref 0x0~2000 Blob(0x558c55060690 blob([0x65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57db4fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x83a000~2000] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c550b2700) put_ref 0x0~2000 Blob(0x558c550b2700 blob([0x67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c55060770) put_ref 0x0~2000 Blob(0x558c55060770 blob([0x69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57db4ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x83c000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c550607e0) put_ref 0x0~2000 Blob(0x558c550607e0 blob([0x6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c550b3340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x83e000~2000] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c55278a10) put_ref 0x0~2000 Blob(0x558c55278a10 blob([0x6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c550605b0) put_ref 0x0~2000 Blob(0x558c550605b0 blob([0x6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57db4e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x840000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.blob(0x558c580a2e00) put_ref 0x0~2000 Blob(0x558c580a2e00 blob([0x71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8e608) dirty_range mark inline shard dirty 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57db5030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x842000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57654f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x63000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57db50a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x844000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57db44d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x846000~2000] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c55060690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x65000~2000] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c580ec540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x848000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c550b2700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x67000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57ed1b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84a000~2000] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c55060770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c580ec5b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x69000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84c000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57db43f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x84e000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c550607e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57db4540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x6b000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x850000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57db45b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x852000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c55278a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x6d000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57db4620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x854000~2000] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57db4690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x856000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c550605b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x6f000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~1000: 0x0~1000 Blob(0x558c57db4700 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=2)))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x858000~1000] 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c580a2e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=2)))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x71000~2000] 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c577f2000,0x558c583ea690,0x558c583ea770,0x558c583ea850,0x558c583ea930,0x558c583eaa10,0x558c583eaaf0,0x558c583eabd0,0x558c583eacb0,0x558c583ead90,0x558c583eb9d0,0x558c583ebab0,0x558c583ebb90,0x558c583ebc70,0x558c583ebd50,0x558c583ebe30 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:15#36 = 0 2019-12-07T19:34:34.627+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf8af0,0x558c54cf8d90,0x558c54cf8ee0,0x558c54cf8fc0,0x558c54cf90a0,0x558c54cf9180,0x558c54d3cc40,0x558c54d75ea0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6336c6f2:::smithi13140796-16:17#46 = 0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.627+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875a000 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875a000 onodes shared_blobs 0x558c577f2000,0x558c583ea690,0x558c583ea770,0x558c583ea850,0x558c583ea930,0x558c583eaa10,0x558c583eaaf0,0x558c583eabd0,0x558c583eacb0,0x558c583ead90,0x558c583eb9d0,0x558c583ebab0,0x558c583ebb90,0x558c583ebc70,0x558c583ebd50,0x558c583ebe30 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2929 is 11 SharedBlob(0x558c577f2000 loaded (sbid 0x2929 ref_map(0x83a000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292a is 11 SharedBlob(0x558c583ea690 loaded (sbid 0x292a ref_map(0x83c000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853e600 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292b is 11 SharedBlob(0x558c583ea770 loaded (sbid 0x292b ref_map(0x83e000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853e600 onodes shared_blobs 0x558c54cf8af0,0x558c54cf8d90,0x558c54cf8ee0,0x558c54cf8fc0,0x558c54cf90a0,0x558c54cf9180,0x558c54d3cc40,0x558c54d75ea0 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292c is 11 SharedBlob(0x558c583ea850 loaded (sbid 0x292c ref_map(0x840000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2941 is 11 SharedBlob(0x558c54cf8af0 loaded (sbid 0x2941 ref_map(0x69000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292d is 11 SharedBlob(0x558c583ea930 loaded (sbid 0x292d ref_map(0x842000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292e is 11 SharedBlob(0x558c583eaa10 loaded (sbid 0x292e ref_map(0x844000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2940 is 11 SharedBlob(0x558c54cf8d90 loaded (sbid 0x2940 ref_map(0x67000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x292f is 11 SharedBlob(0x558c583eaaf0 loaded (sbid 0x292f ref_map(0x846000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2942 is 11 SharedBlob(0x558c54cf8ee0 loaded (sbid 0x2942 ref_map(0x6b000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2930 is 11 SharedBlob(0x558c583eabd0 loaded (sbid 0x2930 ref_map(0x848000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2943 is 11 SharedBlob(0x558c54cf8fc0 loaded (sbid 0x2943 ref_map(0x6d000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2931 is 11 SharedBlob(0x558c583eacb0 loaded (sbid 0x2931 ref_map(0x84a000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2944 is 11 SharedBlob(0x558c54cf90a0 loaded (sbid 0x2944 ref_map(0x6f000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2932 is 11 SharedBlob(0x558c583ead90 loaded (sbid 0x2932 ref_map(0x84c000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2945 is 11 SharedBlob(0x558c54cf9180 loaded (sbid 0x2945 ref_map(0x71000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2933 is 11 SharedBlob(0x558c583eb9d0 loaded (sbid 0x2933 ref_map(0x84e000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293f is 11 SharedBlob(0x558c54d3cc40 loaded (sbid 0x293f ref_map(0x65000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2934 is 11 SharedBlob(0x558c583ebab0 loaded (sbid 0x2934 ref_map(0x850000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293e is 11 SharedBlob(0x558c54d75ea0 loaded (sbid 0x293e ref_map(0x63000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2935 is 11 SharedBlob(0x558c583ebb90 loaded (sbid 0x2935 ref_map(0x852000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853e600 allocated 0x[] released 0x[] 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2936 is 11 SharedBlob(0x558c583ebc70 loaded (sbid 0x2936 ref_map(0x854000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2937 is 11 SharedBlob(0x558c583ebd50 loaded (sbid 0x2937 ref_map(0x856000~2000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2938 is 11 SharedBlob(0x558c583ebe30 loaded (sbid 0x2938 ref_map(0x858000~1000=1))) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875a000 allocated 0x[] released 0x[] 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a000 prepare 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875a000 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a000 io_done 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e600 prepare 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853e600 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e600 io_done 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 57 dequeue_op 0x558c5511e9a0 finish 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f0197c700 10 osd.7 57 dequeue_op 0x558c581cd8c0 finish 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 281 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52700 con 0x558c54c5bc00 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.6s2 epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s)) prio 255 cost 10 e57) pg 0x558c57383000 2019-12-07T19:34:34.627+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064394266s ub 67.064394266s int 16s) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c5875c000 seq 30 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9b80 had 1 2019-12-07T19:34:34.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3017 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:34.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:34.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875c000 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875c000 onodes shared_blobs 2019-12-07T19:34:34.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e57) pg 0x558c54d5c000 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875c000 allocated 0x[] released 0x[] 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8e500 had 1 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3040 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3040 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c000 prepare 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875c000 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c000 io_done 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4010b2~be0 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4010b2 0x4010b2~be0 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=155, reqid=osd.3.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=57'17)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d10b2 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb2 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.627+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc92 and padding block with 0x36e 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=155, reqid=osd.3.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=57'17)) v2) prio 127 cost 0 e57) pg 0x558c57383000 2019-12-07T19:34:34.627+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 57 dequeue_op 0x558c581cc580 prio 127 cost 0 latency 0.000439 MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=155, reqid=osd.3.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=57'17)) v2 pg pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:34.627+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:34.627+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c575c5800 seq 109 2019-12-07T19:34:34.627+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.fs5 57/41 ECSubWrite(tid=155, reqid=osd.3.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=57'17)) v2 2019-12-07T19:34:34.627+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:34.627+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:34.627+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:34.627+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c5800 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x401c92 2019-12-07T19:34:34.627+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c5800 onodes shared_blobs 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=54'14 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,57'17], crt=54'14) [] 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.627+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c5800 allocated 0x[] released 0x[] 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000122691s (0.000000034s flush + 0.000122657s kv commit) 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=55'15 (0'0) error 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:417 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875a000,0x558c5853e600> 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a000 kv_submitted 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875a000 2019-12-07T19:34:34.627+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5800 prepare 2019-12-07T19:34:34.627+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c5800 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a000 finishing 2019-12-07T19:34:34.627+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5800 io_done 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875a000 onodes 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875a000 done 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875a000 [] 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=57'16 (54'13) modify 1:fc3a86a2:::smithi13140796-33:1b by unknown.0.0:0 2019-12-07T19:34:34.624151+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e600 kv_submitted 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853e600 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e600 finishing 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853e600 onodes 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853e600 done 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c5800 kv_queued 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853e600 [] 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=57'17 (54'14) modify 1:fc3a86a2:::smithi13140796-33:head by unknown.0.0:0 2019-12-07T19:34:34.624151+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 57/41 ECSubWriteReply(tid=283, last_complete=57'55, committed=1, applied=1)) v2 -- 0x558c58114a00 con 0x558c54c64000 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 17 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x401c92~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x401c92 0x401c92~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.627+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d1c92 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc92 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x10f and padding block with 0xef1 2019-12-07T19:34:34.627+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c5875c300 seq 31 2019-12-07T19:34:34.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875c300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875c300 onodes shared_blobs 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875c300 allocated 0x[] released 0x[] 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40210f 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c300 prepare 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875c300 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c300 io_done 2019-12-07T19:34:34.627+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 282 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a52700 con 0x558c54c5bc00 2019-12-07T19:34:34.627+0000 7f8f0117b700 10 osd.7 57 dequeue_op 0x558c581cc580 finish 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000099503s (0.000000090s flush + 0.000099413s kv commit) 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:34.627+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.1s1 epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:34.627+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 57/28 ECSubWriteReply(tid=156, last_complete=57'71, committed=1, applied=1)) v2 -- 0x558c57951800 con 0x558c54c5bc00 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2240 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.627+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s)) prio 255 cost 10 e57) pg 0x558c5736a000 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2263 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2263 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40210f~8d7 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.064402266s ub 67.064402266s int 16s) 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40210f 0x40210f~8d7 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d210f 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x10f 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x9e6 and padding block with 0x61a 2019-12-07T19:34:34.627+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.627+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:34.627+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:34.627+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c579d3200 seq 24 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875c000> 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c000 kv_submitted 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875c000 2019-12-07T19:34:34.627+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c000 finishing 2019-12-07T19:34:34.627+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875c000 onodes 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875c000 done 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875c300 kv_submitted 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875c000 [] 2019-12-07T19:34:34.627+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.627+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.627+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:34.627+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c579d3200 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:34.627+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c579d3200 onodes shared_blobs 2019-12-07T19:34:34.627+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c579d3200 allocated 0x[] released 0x[] 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3200 prepare 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4029e6 2019-12-07T19:34:34.627+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c579d3200 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.627+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3200 io_done 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.627+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000105196s (0.000000089s flush + 0.000105107s kv commit) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4029e6~495 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4029e6 0x4029e6~495 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d29e6 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x9e6 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe7b and padding block with 0x185 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c575c5800,0x558c5875c300> 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5800 kv_submitted 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c5800 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5800 finishing 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c5800 onodes 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c5800 done 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c5800 [] 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c300 kv_submitted 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875c300 2019-12-07T19:34:34.627+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c300 finishing 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875c300 onodes 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875c300 done 2019-12-07T19:34:34.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875c300 [] 2019-12-07T19:34:34.627+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.627+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 473 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:34.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.8s5 epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:34.627+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.627+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 57/41 ECSubWriteReply(tid=155, last_complete=57'17, committed=1, applied=1)) v2 -- 0x558c58513400 con 0x558c54c5bc00 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.627+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s)) prio 255 cost 10 e57) pg 0x558c57380000 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068518371s ub 67.068518371s int 16s) 2019-12-07T19:34:34.627+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.627+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5875a300 seq 53 2019-12-07T19:34:34.627+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875a300 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875a300 onodes shared_blobs 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875a300 allocated 0x[] released 0x[] 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a300 prepare 2019-12-07T19:34:34.627+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875a300 2019-12-07T19:34:34.627+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a300 io_done 2019-12-07T19:34:34.627+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x402e7b 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000189767s (0.000000082s flush + 0.000189685s kv commit) 2019-12-07T19:34:34.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x402e7b~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x402e7b 0x402e7b~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d2e7b 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe7b 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x2f8 and padding block with 0xd08 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4032f8 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000134819s (0.000000071s flush + 0.000134748s kv commit) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 474 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:34.628+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.es4 epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:34.628+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c579d3200,0x558c5875a300> 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3200 kv_submitted 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c579d3200 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s)) prio 255 cost 10 e57) pg 0x558c54d98000 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3200 finishing 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c579d3200 onodes 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c579d3200 done 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c579d3200 [] 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a300 kv_submitted 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875a300 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a300 finishing 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875a300 onodes 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875a300 done 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068340184s ub 67.068340184s int 16s) 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875a300 [] 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.628+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.628+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5853e900 seq 22 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853e900 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853e900 onodes shared_blobs 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853e900 allocated 0x[] released 0x[] 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e900 prepare 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853e900 2019-12-07T19:34:34.628+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e900 io_done 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 475 ==== MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=284, reqid=osd.1.0:281, at_version=0'0, trim_to=0'0, roll_forward_to=57'44)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c5852d000 con 0x558c54c64000 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511e580 prio 127 cost 0 latency 0.000018 epoch 57 MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=284, reqid=osd.1.0:281, at_version=0'0, trim_to=0'0, roll_forward_to=57'44)) v2 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4032f8~495 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=284, reqid=osd.1.0:281, at_version=0'0, trim_to=0'0, roll_forward_to=57'44)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4032f8 0x4032f8~495 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d32f8 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x2f8 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x78d and padding block with 0x873 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=284, reqid=osd.1.0:281, at_version=0'0, trim_to=0'0, roll_forward_to=57'44)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=284, reqid=osd.1.0:281, at_version=0'0, trim_to=0'0, roll_forward_to=57'44)) v2) prio 127 cost 0 e57) pg 0x558c5500a000 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40378d 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.628+0000 7f8f04181700 10 osd.7 57 dequeue_op 0x558c5511e580 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=284, reqid=osd.1.0:281, at_version=0'0, trim_to=0'0, roll_forward_to=57'44)) v2 pg pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000085468s (0.000000106s flush + 0.000085362s kv commit) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.628+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.9s4 57/41 ECSubWrite(tid=284, reqid=osd.1.0:281, at_version=0'0, trim_to=0'0, roll_forward_to=57'44)) v2 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853e900> 2019-12-07T19:34:34.628+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=55'42 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'44], crt=55'42) [] 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e900 kv_submitted 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853e900 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853e900 finishing 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853e900 onodes 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853e900 done 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853e900 [] 2019-12-07T19:34:34.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'43 (55'39) modify 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:34.618831+0000 0 snaps [1b,20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'44 (55'42) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:34.618831+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.628+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 44 2019-12-07T19:34:34.628+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.628+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.628+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:34.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57956f00 seq 84 2019-12-07T19:34:34.628+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:34.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:34.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:34.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57956f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:34.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57956f00 onodes shared_blobs 2019-12-07T19:34:34.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57956f00 allocated 0x[] released 0x[] 2019-12-07T19:34:34.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956f00 prepare 2019-12-07T19:34:34.628+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57956f00 2019-12-07T19:34:34.628+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956f00 io_done 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f04181700 10 osd.7 57 dequeue_op 0x558c5511e580 finish 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40378d~471 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40378d 0x40378d~471 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d378d 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x78d 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbfe and padding block with 0x402 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x403bfe 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000102125s (0.000000069s flush + 0.000102056s kv commit) 2019-12-07T19:34:34.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57956f00> 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956f00 kv_submitted 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57956f00 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57956f00 finishing 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57956f00 onodes 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57956f00 done 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57956f00 [] 2019-12-07T19:34:34.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.628+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 57/41 ECSubWriteReply(tid=284, last_complete=57'44, committed=1, applied=1)) v2 -- 0x558c57950800 con 0x558c54c64000 2019-12-07T19:34:34.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 295 ==== MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=111, reqid=osd.4.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54e40800 con 0x558c54c5b400 2019-12-07T19:34:34.628+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c581cdce0 prio 127 cost 0 latency 0.000016 epoch 57 MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=111, reqid=osd.4.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:34.628+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=111, reqid=osd.4.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=111, reqid=osd.4.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=111, reqid=osd.4.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e57) pg 0x558c575ef000 2019-12-07T19:34:34.628+0000 7f8f0317f700 10 osd.7 57 dequeue_op 0x558c581cdce0 prio 127 cost 0 latency 0.000042 MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=111, reqid=osd.4.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 pg pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 57/41 ECSubWrite(tid=111, reqid=osd.4.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:34.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=53'37 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'41], crt=53'37) [] 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'38 (53'34) delete 1:dce63cc3:::smithi13140796-50:15 by unknown.0.0:0 2019-12-07T19:34:34.618977+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'39 (53'35) modify 1:dce63cc3:::smithi13140796-50:head by unknown.0.0:0 2019-12-07T19:34:34.618977+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'40 (53'36) delete 1:db85c185:::smithi13140796-18:15 by unknown.0.0:0 2019-12-07T19:34:34.619978+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'41 (53'37) modify 1:db85c185:::smithi13140796-18:head by unknown.0.0:0 2019-12-07T19:34:34.619978+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 41 2019-12-07T19:34:34.628+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.628+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.628+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.628+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a2300 seq 50 2019-12-07T19:34:34.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 476 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:34.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15#26 onode 0x558c574a0a00 txc 0x558c586a2300 2019-12-07T19:34:34.628+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15#26 0x0 2019-12-07T19:34:34.628+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.9s4 epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.9s0( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c574f1e30) put_ref 0x0~2000 Blob(0x558c574f1e30 blob([0x652000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.9s0( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c574f1180) put_ref 0x0~2000 Blob(0x558c574f1180 blob([0x654000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c550b2d90) put_ref 0x0~2000 Blob(0x558c550b2d90 blob([0x656000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c55136c40) put_ref 0x0~2000 Blob(0x558c55136c40 blob([0x658000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3b90) put_ref 0x0~2000 Blob(0x558c579b3b90 blob([0x65a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c57970770) put_ref 0x0~2000 Blob(0x558c57970770 blob([0x65c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad8f0) put_ref 0x0~2000 Blob(0x558c576ad8f0 blob([0x65e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3ab0) put_ref 0x0~2000 Blob(0x558c579b3ab0 blob([0x660000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3b20) put_ref 0x0~2000 Blob(0x558c579b3b20 blob([0x662000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c57970700) put_ref 0x0~2000 Blob(0x558c57970700 blob([0x664000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad880) put_ref 0x0~2000 Blob(0x558c576ad880 blob([0x666000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b39d0) put_ref 0x0~2000 Blob(0x558c579b39d0 blob([0x668000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3a40) put_ref 0x0~2000 Blob(0x558c579b3a40 blob([0x66a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c57970690) put_ref 0x0~2000 Blob(0x558c57970690 blob([0x66c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c57818c40) put_ref 0x0~2000 Blob(0x558c57818c40 blob([0x66e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b38f0) put_ref 0x0~2000 Blob(0x558c579b38f0 blob([0x670000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3960) put_ref 0x0~2000 Blob(0x558c579b3960 blob([0x672000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c57970620) put_ref 0x0~2000 Blob(0x558c57970620 blob([0x674000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b2000) put_ref 0x0~2000 Blob(0x558c579b2000 blob([0x676000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3810) put_ref 0x0~2000 Blob(0x558c579b3810 blob([0x678000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3880) put_ref 0x0~2000 Blob(0x558c579b3880 blob([0x67a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579705b0) put_ref 0x0~2000 Blob(0x558c579705b0 blob([0x67c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c576ad9d0) put_ref 0x0~2000 Blob(0x558c576ad9d0 blob([0x67e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b3730) put_ref 0x0~2000 Blob(0x558c579b3730 blob([0x680000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c579b37a0) put_ref 0x0~2000 Blob(0x558c579b37a0 blob([0x682000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.blob(0x558c57970540) put_ref 0x0~1000 Blob(0x558c57970540 blob([0x684000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore.extentmap(0x558c574a0b08) dirty_range mark inline shard dirty 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574f1e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=2)))) 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x652000~2000] 2019-12-07T19:34:34.628+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c574f1180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x654000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c550b2d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x656000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c55136c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x658000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c579b3b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x65a000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57970770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x65c000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c576ad8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x65e000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c579b3ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x660000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c579b3b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x662000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57970700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x664000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c576ad880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x666000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c579b39d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x668000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c579b3a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x66a000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57970690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x66c000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57818c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x66e000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c579b38f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x670000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c579b3960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x672000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c57970620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x674000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c579b2000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x676000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c579b3810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x678000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c579b3880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x67a000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c579705b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x67c000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c576ad9d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x67e000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c579b3730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x680000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c579b37a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x682000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~1000: 0x0~1000 Blob(0x558c57970540 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x684000~1000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c55279810,0x558c5742af50,0x558c5742bb20,0x558c574fd500,0x558c574fd6c0,0x558c577f3880,0x558c57819dc0,0x558c57881810,0x558c57a073b0,0x558c57b97f10,0x558c57e63340,0x558c57efc070,0x558c581ea070,0x558c581ea150,0x558c581ea230,0x558c581ea310,0x558c581ea3f0,0x558c581ea4d0,0x558c581ea5b0,0x558c581ea690,0x558c581ea770,0x558c581ea850,0x558c581ea930,0x558c581eaa10,0x558c581eaaf0,0x558c581eabd0 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:15#26 = 0 2019-12-07T19:34:34.629+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs1_head 1#1:db85c185:::smithi13140796-18:15#28 onode 0x558c57573680 txc 0x558c586a2300 2019-12-07T19:34:34.629+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:db85c185:::smithi13140796-18:15#28 0x0 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c55136f50) put_ref 0x0~2000 Blob(0x558c55136f50 blob([0x611000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2930) put_ref 0x0~2000 Blob(0x558c574e2930 blob([0x613000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3110) put_ref 0x0~2000 Blob(0x558c550b3110 blob([0x615000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c55137030) put_ref 0x0~2000 Blob(0x558c55137030 blob([0x617000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c551370a0) put_ref 0x0~2000 Blob(0x558c551370a0 blob([0x619000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c55137180) put_ref 0x0~2000 Blob(0x558c55137180 blob([0x61b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c551371f0) put_ref 0x0~2000 Blob(0x558c551371f0 blob([0x61d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c55137260) put_ref 0x0~2000 Blob(0x558c55137260 blob([0x61f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.blob(0x558c551372d0) put_ref 0x0~2000 Blob(0x558c551372d0 blob([0x621000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57573788) dirty_range mark inline shard dirty 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c55136f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x611000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c574e2930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x613000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c550b3110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x615000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c55137030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x617000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c551370a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x619000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c55137180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x61b000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c551371f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x61d000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c55137260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x61f000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c551372d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=2)))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x621000~2000] 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c57971ea0,0x558c57a075e0,0x558c57a07e30,0x558c57c38a10,0x558c57f49880,0x558c57f49960,0x558c57f49a40,0x558c57f49b20,0x558c5809b2d0 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.bs1_head 1#1:db85c185:::smithi13140796-18:15#28 = 0 2019-12-07T19:34:34.629+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a2300 cost 5587 (1 ios * 4000 + 1587 bytes) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a2300 onodes shared_blobs 0x558c55279810,0x558c5742af50,0x558c5742bb20,0x558c574fd500,0x558c574fd6c0,0x558c577f3880,0x558c57819dc0,0x558c57881810,0x558c57971ea0,0x558c57a073b0,0x558c57a075e0,0x558c57a07e30,0x558c57b97f10,0x558c57c38a10,0x558c57e63340,0x558c57efc070,0x558c57f49880,0x558c57f49960,0x558c57f49a40,0x558c57f49b20,0x558c5809b2d0,0x558c581ea070,0x558c581ea150,0x558c581ea230,0x558c581ea310,0x558c581ea3f0,0x558c581ea4d0,0x558c581ea5b0,0x558c581ea690,0x558c581ea770,0x558c581ea850,0x558c581ea930,0x558c581eaa10,0x558c581eaaf0,0x558c581eabd0 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2907 is 11 SharedBlob(0x558c55279810 loaded (sbid 0x2907 ref_map(0x654000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290e is 11 SharedBlob(0x558c5742af50 loaded (sbid 0x290e ref_map(0x662000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290f is 11 SharedBlob(0x558c5742bb20 loaded (sbid 0x290f ref_map(0x664000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2909 is 11 SharedBlob(0x558c574fd500 loaded (sbid 0x2909 ref_map(0x658000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290d is 11 SharedBlob(0x558c574fd6c0 loaded (sbid 0x290d ref_map(0x660000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2910 is 11 SharedBlob(0x558c577f3880 loaded (sbid 0x2910 ref_map(0x666000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2906 is 11 SharedBlob(0x558c57819dc0 loaded (sbid 0x2906 ref_map(0x652000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290a is 11 SharedBlob(0x558c57881810 loaded (sbid 0x290a ref_map(0x65a000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2927 is 11 SharedBlob(0x558c57971ea0 loaded (sbid 0x2927 ref_map(0x61f000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2908 is 11 SharedBlob(0x558c57a073b0 loaded (sbid 0x2908 ref_map(0x656000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2921 is 11 SharedBlob(0x558c57a075e0 loaded (sbid 0x2921 ref_map(0x613000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2920 is 11 SharedBlob(0x558c57a07e30 loaded (sbid 0x2920 ref_map(0x611000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2911 is 11 SharedBlob(0x558c57b97f10 loaded (sbid 0x2911 ref_map(0x668000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2926 is 11 SharedBlob(0x558c57c38a10 loaded (sbid 0x2926 ref_map(0x61d000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290c is 11 SharedBlob(0x558c57e63340 loaded (sbid 0x290c ref_map(0x65e000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x290b is 11 SharedBlob(0x558c57efc070 loaded (sbid 0x290b ref_map(0x65c000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2925 is 11 SharedBlob(0x558c57f49880 loaded (sbid 0x2925 ref_map(0x61b000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2924 is 11 SharedBlob(0x558c57f49960 loaded (sbid 0x2924 ref_map(0x619000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2923 is 11 SharedBlob(0x558c57f49a40 loaded (sbid 0x2923 ref_map(0x617000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2922 is 11 SharedBlob(0x558c57f49b20 loaded (sbid 0x2922 ref_map(0x615000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2928 is 11 SharedBlob(0x558c5809b2d0 loaded (sbid 0x2928 ref_map(0x621000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2912 is 11 SharedBlob(0x558c581ea070 loaded (sbid 0x2912 ref_map(0x66a000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2913 is 11 SharedBlob(0x558c581ea150 loaded (sbid 0x2913 ref_map(0x66c000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2914 is 11 SharedBlob(0x558c581ea230 loaded (sbid 0x2914 ref_map(0x66e000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2915 is 11 SharedBlob(0x558c581ea310 loaded (sbid 0x2915 ref_map(0x670000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2916 is 11 SharedBlob(0x558c581ea3f0 loaded (sbid 0x2916 ref_map(0x672000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2917 is 11 SharedBlob(0x558c581ea4d0 loaded (sbid 0x2917 ref_map(0x674000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2918 is 11 SharedBlob(0x558c581ea5b0 loaded (sbid 0x2918 ref_map(0x676000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2919 is 11 SharedBlob(0x558c581ea690 loaded (sbid 0x2919 ref_map(0x678000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291a is 11 SharedBlob(0x558c581ea770 loaded (sbid 0x291a ref_map(0x67a000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291b is 11 SharedBlob(0x558c581ea850 loaded (sbid 0x291b ref_map(0x67c000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291c is 11 SharedBlob(0x558c581ea930 loaded (sbid 0x291c ref_map(0x67e000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291d is 11 SharedBlob(0x558c581eaa10 loaded (sbid 0x291d ref_map(0x680000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291e is 11 SharedBlob(0x558c581eaaf0 loaded (sbid 0x291e ref_map(0x682000~2000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x291f is 11 SharedBlob(0x558c581eabd0 loaded (sbid 0x291f ref_map(0x684000~1000=1))) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a2300 allocated 0x[] released 0x[] 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2300 prepare 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a2300 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2300 io_done 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 osd.7 57 dequeue_op 0x558c581cdce0 finish 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.9s0( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2115 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.9s0( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s)) prio 255 cost 10 e57) pg 0x558c5500a000 2019-12-07T19:34:34.629+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.9s0( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068600079s ub 67.068600079s int 16s) 2019-12-07T19:34:34.629+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.629+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:34.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:34.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c579d3500 seq 85 2019-12-07T19:34:34.629+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:34.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:34.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c574a0a00 had 1 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573680 had 1 2019-12-07T19:34:34.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2138 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c579d3500 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2138 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c579d3500 onodes shared_blobs 2019-12-07T19:34:34.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c579d3500 allocated 0x[] released 0x[] 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3500 prepare 2019-12-07T19:34:34.629+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c579d3500 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x403bfe~85a to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3500 io_done 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x403bfe 0x403bfe~85a to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d3bfe 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbfe 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x458 and padding block with 0xba8 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x404458 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000171677s (0.000000070s flush + 0.000171607s kv commit) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586a2300> 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2300 kv_submitted 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a2300 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2300 finishing 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a2300 onodes 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a2300 done 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a2300 [] 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x404458~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 57/41 ECSubWriteReply(tid=111, last_complete=57'41, committed=1, applied=1)) v2 -- 0x558c5763c600 con 0x558c54c5b400 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x404458 0x404458~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d4458 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x458 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8d5 and padding block with 0x72b 2019-12-07T19:34:34.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4048d5 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116710s (0.000000090s flush + 0.000116620s kv commit) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c579d3500> 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3500 kv_submitted 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c579d3500 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3500 finishing 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c579d3500 onodes 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c579d3500 done 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c579d3500 [] 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 296 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558c57a52700 con 0x558c54c5b400 2019-12-07T19:34:34.629+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.bs1 epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.bs0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:34.629+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.bs0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.bs0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.bs0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s)) prio 255 cost 10 e57) pg 0x558c575ef000 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 4(0) info: 1.bs0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.057987122s ub 67.057987122s int 16s) 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a2600 seq 51 2019-12-07T19:34:34.629+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a2600 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a2600 onodes shared_blobs 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a2600 allocated 0x[] released 0x[] 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2600 prepare 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a2600 2019-12-07T19:34:34.629+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2600 io_done 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4048d5~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4048d5 0x4048d5~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d48d5 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8d5 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd52 and padding block with 0x2ae 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x404d52 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122619s (0.000000104s flush + 0.000122515s kv commit) 2019-12-07T19:34:34.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 477 ==== MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=285, reqid=osd.1.0:285, at_version=57'9, trim_to=0'0, roll_forward_to=54'7)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c5852c000 con 0x558c54c64000 2019-12-07T19:34:34.629+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511f080 prio 127 cost 0 latency 0.000029 epoch 57 MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=285, reqid=osd.1.0:285, at_version=57'9, trim_to=0'0, roll_forward_to=54'7)) v2 2019-12-07T19:34:34.629+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=285, reqid=osd.1.0:285, at_version=57'9, trim_to=0'0, roll_forward_to=54'7)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 478 ==== MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=286, reqid=osd.1.0:286, at_version=57'39, trim_to=0'0, roll_forward_to=55'37)) v2 ==== 2768+0+0 (crc 0 0 0) 0x558c5852c800 con 0x558c54c64000 2019-12-07T19:34:34.629+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511e6e0 prio 127 cost 0 latency 0.000022 epoch 57 MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=286, reqid=osd.1.0:286, at_version=57'39, trim_to=0'0, roll_forward_to=55'37)) v2 2019-12-07T19:34:34.629+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=286, reqid=osd.1.0:286, at_version=57'39, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=285, reqid=osd.1.0:285, at_version=57'9, trim_to=0'0, roll_forward_to=54'7)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586a2600> 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2600 kv_submitted 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a2600 2019-12-07T19:34:34.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2600 finishing 2019-12-07T19:34:34.629+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=285, reqid=osd.1.0:285, at_version=57'9, trim_to=0'0, roll_forward_to=54'7)) v2) prio 127 cost 0 e57) pg 0x558c57e8b000 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a2600 onodes 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a2600 done 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a2600 [] 2019-12-07T19:34:34.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.629+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f03980700 10 osd.7 57 dequeue_op 0x558c5511f080 prio 127 cost 0 latency 0.000105 MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=285, reqid=osd.1.0:285, at_version=57'9, trim_to=0'0, roll_forward_to=54'7)) v2 pg pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.629+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=285, reqid=osd.1.0:285, at_version=57'9, trim_to=0'0, roll_forward_to=54'7)) v2 2019-12-07T19:34:34.629+0000 7f8f03980700 20 snap_mapper.update_snaps 1:51ccf92f:::smithi13140796-15:1b 1b was 2019-12-07T19:34:34.629+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.629+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:34.629+0000 7f8f03980700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.629+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.629+0000 7f8f03980700 20 snap_mapper.get_snaps 1:51ccf92f:::smithi13140796-15:1b 15,1b 2019-12-07T19:34:34.629+0000 7f8f03980700 20 snap_mapper.set_snaps 1:51ccf92f:::smithi13140796-15:1b 1b 2019-12-07T19:34:34.629+0000 7f8f03980700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:34.629+0000 7f8f03980700 20 snap_mapper.update_snaps rm SNA_1_0000000000000015_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:34.629+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 54'7 (0'0,54'7] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,54'7], crt=54'7) [57'8 (54'3) modify 1:51ccf92f:::smithi13140796-15:1b by unknown.0.0:0 2019-12-07T19:34:34.629757+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'9 (54'7) modify 1:51ccf92f:::smithi13140796-15:head by unknown.0.0:0 2019-12-07T19:34:34.629757+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.630+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'8 (0'0,57'8] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'7 crt=54'7 lcod 54'7 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'8 (54'3) modify 1:51ccf92f:::smithi13140796-15:1b by unknown.0.0:0 2019-12-07T19:34:34.629757+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.630+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'7 crt=54'7 lcod 54'7 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'9 (54'7) modify 1:51ccf92f:::smithi13140796-15:head by unknown.0.0:0 2019-12-07T19:34:34.629757+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.630+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'7 crt=54'7 lcod 54'7 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 9 2019-12-07T19:34:34.630+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'7 crt=54'7 lcod 54'7 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.630+0000 7f8f03980700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.630+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58260600 seq 26 2019-12-07T19:34:34.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# 1 keys 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# 1 keys = 0 2019-12-07T19:34:34.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# hinfo_key (42 bytes) 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:1b# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# 2 keys = 0 2019-12-07T19:34:34.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.as5_head 5#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.A833F94F.1b.smithi13140796-15..' <- OBJ_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.5_0000000000000001.A833F94F.1b.smithi13140796-15..' <- SNA_1_0000000000000015_.5_0000000000000001.A833F94F.1b.smithi13140796-15.. 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.630+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000057.00000000000000000008' <- 0000000057.00000000000000000008 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.0000000057.00000000000000000009' <- 0000000057.00000000000000000009 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58260600 cost 8367 (1 ios * 4000 + 4367 bytes) 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58260600 onodes 0x558c54e13180,0x558c575f9180 shared_blobs 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore.extentmap(0x558c54e13288) update 5#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:head# is 1638 (558 bytes onode + 2 bytes spanning blobs + 1078 bytes inline extents) 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore.extentmap(0x558c575f9288) update 5#1:51ccf92f:::smithi13140796-15:1b# 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:51ccf92f:::smithi13140796-15:1b# is 634 (412 bytes onode + 2 bytes spanning blobs + 220 bytes inline extents) 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58260600 allocated 0x[] released 0x[] 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260600 prepare 2019-12-07T19:34:34.630+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58260600 2019-12-07T19:34:34.630+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260600 io_done 2019-12-07T19:34:34.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.630+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=54'7 crt=54'7 lcod 54'7 active mbc={} ps=[e~1,1a~1]] op_applied version 57'9 2019-12-07T19:34:34.630+0000 7f8f03980700 10 osd.7 57 dequeue_op 0x558c5511f080 finish 2019-12-07T19:34:34.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=286, reqid=osd.1.0:286, at_version=57'39, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=286, reqid=osd.1.0:286, at_version=57'39, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e57) pg 0x558c54d5f000 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 57 dequeue_op 0x558c5511e6e0 prio 127 cost 0 latency 0.000379 MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=286, reqid=osd.1.0:286, at_version=57'39, trim_to=0'0, roll_forward_to=55'37)) v2 pg pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=286, reqid=osd.1.0:286, at_version=57'39, trim_to=0'0, roll_forward_to=55'37)) v2 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 snap_mapper.remove_oid 1:ecc72bdb:::smithi13140796-10:15 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 snap_mapper._remove_oid 1:ecc72bdb:::smithi13140796-10:15 2019-12-07T19:34:34.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5107 < min_flush_size 524288 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e13180 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f9180 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5130 < min_flush_size 524288 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5130 < min_flush_size 524288 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x404d52~140a to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x404d52 0x404d52~140a to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d4d52 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd52 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x15c and padding block with 0xea4 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40615c 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000113991s (0.000000095s flush + 0.000113896s kv commit) 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:15 15 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 snap_mapper.clear_snaps 1:ecc72bdb:::smithi13140796-10:15 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58260600> 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 snap_mapper.clear_snaps rm OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:34.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260600 kv_submitted 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58260600 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 snap_mapper._remove_oid rm SNA_1_0000000000000015_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:34.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260600 finishing 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58260600 onodes 0x558c54e13180,0x558c575f9180 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58260600 done 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:34.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58260600 [] 2019-12-07T19:34:34.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.630+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 57/41 ECSubWriteReply(tid=285, last_complete=57'9, committed=1, applied=1)) v2 -- 0x558c580b8c00 con 0x558c54c64000 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 55'37 (0'0,55'37] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,55'37], crt=55'37) [57'38 (53'30) delete 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:34.630359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'39 (54'35) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:34.630359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.630+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'38 (0'0,57'38] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=55'37 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'38 (53'30) delete 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:34.630359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.630+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=55'37 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'39 (54'35) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:34.630359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=55'37 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 39 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=55'37 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.630+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'38, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5875c600 seq 85 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ecc72bdb:::smithi13140796-10:15#26 key 0x848000000000000001ecc72bdb21736d'ithi13140796-10!='0x00000000000000150000000000000026'o' 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# -> 4#1:ecc72bdb:::smithi13140796-10:15#26 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15# -> 4#1:ecc72bdb:::smithi13140796-10:15#26 = 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.4_0000000000000001.733E4DBD.15.smithi13140796-10..' <- SNA_1_0000000000000015_.4_0000000000000001.733E4DBD.15.smithi13140796-10.. 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000057.00000000000000000038' <- 0000000057.00000000000000000038 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000057.00000000000000000039' <- 0000000057.00000000000000000039 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875c600 cost 7663 (1 ios * 4000 + 3663 bytes) 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875c600 onodes 0x558c575a2280,0x558c575a2a00 shared_blobs 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2388) update 4#1:ecc72bdb:::smithi13140796-10:15#26 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:15#26 is 556 (412 bytes onode + 2 bytes spanning blobs + 142 bytes inline extents) 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 1041 (541 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875c600 allocated 0x[] released 0x[] 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c600 prepare 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875c600 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c600 io_done 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4124 < min_flush_size 524288 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=55'37 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] op_applied version 57'39 2019-12-07T19:34:34.630+0000 7f8f0117b700 10 osd.7 57 dequeue_op 0x558c5511e6e0 finish 2019-12-07T19:34:34.630+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2280 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795a780 had 1 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4147 < min_flush_size 524288 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4147 < min_flush_size 524288 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40615c~1033 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40615c 0x40615c~1033 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d615c 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x15c 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x18f and padding block with 0xe71 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40718f 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 479 ==== MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=287, reqid=osd.1.0:287, at_version=57'41, trim_to=0'0, roll_forward_to=55'37)) v2 ==== 3394+0+0 (crc 0 0 0) 0x558c5852b000 con 0x558c54c64000 2019-12-07T19:34:34.630+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511f1e0 prio 127 cost 0 latency 0.000025 epoch 57 MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=287, reqid=osd.1.0:287, at_version=57'41, trim_to=0'0, roll_forward_to=55'37)) v2 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000141567s (0.000000104s flush + 0.000141463s kv commit) 2019-12-07T19:34:34.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.630+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=287, reqid=osd.1.0:287, at_version=57'41, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=287, reqid=osd.1.0:287, at_version=57'41, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.630+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=287, reqid=osd.1.0:287, at_version=57'41, trim_to=0'0, roll_forward_to=55'37)) v2) prio 127 cost 0 e57) pg 0x558c54d5f000 2019-12-07T19:34:34.630+0000 7f8f05183700 10 osd.7 57 dequeue_op 0x558c5511f1e0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=287, reqid=osd.1.0:287, at_version=57'41, trim_to=0'0, roll_forward_to=55'37)) v2 pg pg[1.7s4( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.630+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=287, reqid=osd.1.0:287, at_version=57'41, trim_to=0'0, roll_forward_to=55'37)) v2 2019-12-07T19:34:34.630+0000 7f8f05183700 20 snap_mapper.update_snaps 1:ec1285ab:::smithi13140796-19:20 1b,20 was 2019-12-07T19:34:34.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.630+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:34.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.630+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ec1285ab:::smithi13140796-19:20 15,1b,20 2019-12-07T19:34:34.630+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ec1285ab:::smithi13140796-19:20 1b,20 2019-12-07T19:34:34.630+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:34.630+0000 7f8f05183700 20 snap_mapper.update_snaps rm SNA_1_0000000000000015_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:34.630+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'39 (0'0,57'39] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'39], crt=55'37) [57'40 (55'36) modify 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:34.631357+0000 0 snaps [1b,20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,57'41 (55'37) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:34.631357+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:34.630+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'40 (0'0,57'40] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'39 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'40 (55'36) modify 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:34.631357+0000 0 snaps [1b,20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.630+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'39 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] add_log_entry 57'41 (55'37) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:34.631357+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.630+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'39 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 41 2019-12-07T19:34:34.630+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'39 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.630+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.630+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 57'40, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:34.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c583b6900 seq 86 2019-12-07T19:34:34.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# 1 keys 2019-12-07T19:34:34.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# 1 keys = 0 2019-12-07T19:34:34.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# hinfo_key (42 bytes) 2019-12-07T19:34:34.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:34.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:34.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:34.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:34.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.630+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:34.630+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.630+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000015_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- SNA_1_0000000000000015_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:34.631+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:34.631+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000057.00000000000000000040' <- 0000000057.00000000000000000040 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000057.00000000000000000041' <- 0000000057.00000000000000000041 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875c600> 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.631+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c600 kv_submitted 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875c600 2019-12-07T19:34:34.631+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b6900 cost 7567 (1 ios * 4000 + 3567 bytes) 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c600 finishing 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b6900 onodes 0x558c585ce000,0x558c585ce280 shared_blobs 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875c600 onodes 0x558c575a2280,0x558c575a2a00 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875c600 done 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b6900 prepare 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update 4#1:ec1285ab:::smithi13140796-19:20# 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875c600 [] 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:20# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce388) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 483 (475 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b6900 allocated 0x[] released 0x[] 2019-12-07T19:34:34.631+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6900 prepare 2019-12-07T19:34:34.631+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b6900 2019-12-07T19:34:34.631+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6900 io_done 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2947 < min_flush_size 524288 2019-12-07T19:34:34.631+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'39 crt=55'37 lcod 55'37 active mbc={} ps=[e~1,1a~1]] op_applied version 57'41 2019-12-07T19:34:34.631+0000 7f8f05183700 10 osd.7 57 dequeue_op 0x558c5511f1e0 finish 2019-12-07T19:34:34.631+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 57/41 ECSubWriteReply(tid=286, last_complete=57'39, committed=1, applied=1)) v2 -- 0x558c58512400 con 0x558c54c64000 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce000 had 1 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce280 had 1 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2970 < min_flush_size 524288 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2970 < min_flush_size 524288 2019-12-07T19:34:34.631+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40718f~b9a to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40718f 0x40718f~b9a to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d718f 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x18f 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd29 and padding block with 0x2d7 2019-12-07T19:34:34.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x407d29 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000136521s (0.000000065s flush + 0.000136456s kv commit) 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583b6900> 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6900 kv_submitted 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b6900 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6900 finishing 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b6900 onodes 0x558c585ce000,0x558c585ce280 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b6900 done 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b6900 [] 2019-12-07T19:34:34.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.631+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.631+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 57/41 ECSubWriteReply(tid=287, last_complete=57'41, committed=1, applied=1)) v2 -- 0x558c55044800 con 0x558c54c64000 2019-12-07T19:34:34.631+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.631+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 480 ==== MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=288, reqid=osd.1.0:285, at_version=0'0, trim_to=0'0, roll_forward_to=57'9)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5852a000 con 0x558c54c64000 2019-12-07T19:34:34.631+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511f4a0 prio 127 cost 0 latency 0.000017 epoch 57 MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=288, reqid=osd.1.0:285, at_version=0'0, trim_to=0'0, roll_forward_to=57'9)) v2 2019-12-07T19:34:34.631+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=288, reqid=osd.1.0:285, at_version=0'0, trim_to=0'0, roll_forward_to=57'9)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.631+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.631+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=288, reqid=osd.1.0:285, at_version=0'0, trim_to=0'0, roll_forward_to=57'9)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.631+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.631+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGOpItem(op=MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=288, reqid=osd.1.0:285, at_version=0'0, trim_to=0'0, roll_forward_to=57'9)) v2) prio 127 cost 0 e57) pg 0x558c57e8b000 2019-12-07T19:34:34.631+0000 7f8f07988700 10 osd.7 57 dequeue_op 0x558c5511f4a0 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=288, reqid=osd.1.0:285, at_version=0'0, trim_to=0'0, roll_forward_to=57'9)) v2 pg pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.631+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.as5 57/41 ECSubWrite(tid=288, reqid=osd.1.0:285, at_version=0'0, trim_to=0'0, roll_forward_to=57'9)) v2 2019-12-07T19:34:34.631+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=54'7 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'9], crt=54'7) [] 2019-12-07T19:34:34.631+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'8 (54'3) modify 1:51ccf92f:::smithi13140796-15:1b by unknown.0.0:0 2019-12-07T19:34:34.629757+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.631+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'9 (54'7) modify 1:51ccf92f:::smithi13140796-15:head by unknown.0.0:0 2019-12-07T19:34:34.629757+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.631+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 9 2019-12-07T19:34:34.631+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.631+0000 7f8f07988700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.631+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.631+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:34.631+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58111200 seq 27 2019-12-07T19:34:34.631+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:34.631+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:34.631+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.631+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.631+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:34.631+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58111200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:34.631+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58111200 onodes shared_blobs 2019-12-07T19:34:34.631+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58111200 allocated 0x[] released 0x[] 2019-12-07T19:34:34.631+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111200 prepare 2019-12-07T19:34:34.631+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58111200 2019-12-07T19:34:34.631+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111200 io_done 2019-12-07T19:34:34.631+0000 7f8f07988700 10 osd.7 57 dequeue_op 0x558c5511f4a0 finish 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.631+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 727 < min_flush_size 524288 2019-12-07T19:34:34.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:34:34.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x407d29~47c to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x407d29 0x407d29~47c to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d7d29 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd29 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1a5 and padding block with 0xe5b 2019-12-07T19:34:34.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 481 ==== MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=289, reqid=osd.1.0:287, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5852a800 con 0x558c54c64000 2019-12-07T19:34:34.632+0000 7f8f24457700 15 osd.7 57 enqueue_op 0x558c5511f760 prio 127 cost 0 latency 0.000022 epoch 57 MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=289, reqid=osd.1.0:287, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:34.632+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=289, reqid=osd.1.0:287, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e57) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=289, reqid=osd.1.0:287, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e57) queued 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=289, reqid=osd.1.0:287, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e57) pg 0x558c54d5f000 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 57 dequeue_op 0x558c5511f760 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=289, reqid=osd.1.0:287, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 pg pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.7s4 57/41 ECSubWrite(tid=289, reqid=osd.1.0:287, at_version=0'0, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4081a5 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 482 ==== pg_info2(1.as5 1.as5 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=55'37 active mbc={} ps=[e~1,1a~1]] append_log log((0'0,57'41], crt=55'37) [] 2019-12-07T19:34:34.632+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.as5 epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:34.632+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000194792s (0.000000101s flush + 0.000194691s kv commit) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'38 (53'30) delete 1:ecc72bdb:::smithi13140796-10:15 by unknown.0.0:0 2019-12-07T19:34:34.630359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.632+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.632+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'39 (54'35) modify 1:ecc72bdb:::smithi13140796-10:head by unknown.0.0:0 2019-12-07T19:34:34.630359+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.632+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s)) prio 255 cost 10 e57) pg 0x558c57e8b000 2019-12-07T19:34:34.632+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.069633703s ub 67.069633703s int 16s) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'40 (55'36) modify 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:34.631357+0000 0 snaps [1b,20] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.632+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] rollforward: entry=57'41 (55'37) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:34.631357+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] append_log approx pg log length = 41 2019-12-07T19:34:34.632+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:34.632+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:34.632+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58260900 seq 28 2019-12-07T19:34:34.632+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:34.632+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:34.632+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:34.632+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:34.632+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:34.632+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58260900 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.632+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58260900 onodes shared_blobs 2019-12-07T19:34:34.632+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58260900 allocated 0x[] released 0x[] 2019-12-07T19:34:34.632+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260900 prepare 2019-12-07T19:34:34.632+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58260900 2019-12-07T19:34:34.632+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260900 io_done 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5875c900 seq 87 2019-12-07T19:34:34.632+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15#26 onode 0x558c575a2280 txc 0x558c5875c900 2019-12-07T19:34:34.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15#26 0x0 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore.blob(0x558c583eb5e0) put_ref 0x0~2000 Blob(0x558c583eb5e0 blob([0x3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore.blob(0x558c5794dc70) put_ref 0x0~2000 Blob(0x558c5794dc70 blob([0x40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58111200> 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111200 kv_submitted 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore.blob(0x558c550b3880) put_ref 0x0~2000 Blob(0x558c550b3880 blob([0x42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58111200 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore.blob(0x558c5800d490) put_ref 0x0~2000 Blob(0x558c5800d490 blob([0x44000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111200 finishing 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58111200 onodes 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58111200 done 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58260900 kv_queued 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore.blob(0x558c583eb650) put_ref 0x0~2000 Blob(0x558c583eb650 blob([0x46000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58111200 [] 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2388) dirty_range mark inline shard dirty 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c583eb5e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x3e000~2000] 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c5794dc70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x40000~2000] 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c550b3880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x42000~2000] 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c5800d490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x44000~2000] 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c583eb650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=2)))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x46000~2000] 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54d3ce70,0x558c54d74930,0x558c54d74a10,0x558c54d74af0,0x558c54d74c40 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:15#26 = 0 2019-12-07T19:34:34.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:34.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 483 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875c900 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:34.632+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 57/41 ECSubWriteReply(tid=288, last_complete=57'9, committed=1, applied=1)) v2 -- 0x558c580b8e00 con 0x558c54c64000 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875c900 onodes shared_blobs 0x558c54d3ce70,0x558c54d74930,0x558c54d74a10,0x558c54d74af0,0x558c54d74c40 2019-12-07T19:34:34.632+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.7s4 epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293c is 11 SharedBlob(0x558c54d3ce70 loaded (sbid 0x293c ref_map(0x44000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293d is 11 SharedBlob(0x558c54d74930 loaded (sbid 0x293d ref_map(0x46000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2939 is 11 SharedBlob(0x558c54d74a10 loaded (sbid 0x2939 ref_map(0x3e000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293a is 11 SharedBlob(0x558c54d74af0 loaded (sbid 0x293a ref_map(0x40000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x293b is 11 SharedBlob(0x558c54d74c40 loaded (sbid 0x293b ref_map(0x42000~2000=1))) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875c900 allocated 0x[] released 0x[] 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c900 prepare 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875c900 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c900 io_done 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 57 dequeue_op 0x558c5511f760 finish 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s)) prio 255 cost 10 e57) pg 0x558c54d5f000 2019-12-07T19:34:34.632+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 67.068728225s ub 67.068728225s int 16s) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2467 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5875cc00 seq 88 2019-12-07T19:34:34.632+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2280 had 1 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875cc00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2490 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875cc00 onodes shared_blobs 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2490 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875cc00 allocated 0x[] released 0x[] 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cc00 prepare 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875cc00 2019-12-07T19:34:34.632+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cc00 io_done 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4081a5~9ba to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4081a5 0x4081a5~9ba to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d81a5 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1a5 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb5f and padding block with 0x4a1 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x408b5f 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000111349s (0.000000067s flush + 0.000111282s kv commit) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x408b5f~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x408b5f 0x408b5f~47d to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d8b5f 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb5f 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfdc and padding block with 0x24 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58260900,0x558c5875c900> 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260900 kv_submitted 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58260900 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260900 finishing 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58260900 onodes 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58260900 done 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58260900 [] 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c900 kv_submitted 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875c900 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875c900 finishing 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875c900 onodes 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875c900 done 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875cc00 kv_submitted 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875c900 [] 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.632+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x408fdc 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.632+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 57/41 ECSubWriteReply(tid=289, last_complete=57'41, committed=1, applied=1)) v2 -- 0x558c5768b200 con 0x558c54c64000 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000006 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000109202s (0.000000059s flush + 0.000109143s kv commit) 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875cc00> 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cc00 kv_submitted 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875cc00 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cc00 finishing 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875cc00 onodes 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875cc00 done 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875cc00 [] 2019-12-07T19:34:34.632+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 484 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e57/57) v1 ==== 1092+0+0 (crc 0 0 0) 0x558c57a54a00 con 0x558c54c64000 2019-12-07T19:34:34.632+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.0s3 epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:34.632+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s)) prio 255 cost 10 e57) pg 0x558c5500d000 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 68.076978842s ub 68.076978842s int 16s) 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1] 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c578a7800 seq 49 2019-12-07T19:34:34.632+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a7800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a7800 onodes shared_blobs 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a7800 allocated 0x[] released 0x[] 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7800 prepare 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a7800 2019-12-07T19:34:34.632+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7800 io_done 2019-12-07T19:34:34.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x408fdc~495 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x408fdc 0x408fdc~495 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d8fdc 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfdc 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:34.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x471 and padding block with 0xb8f 2019-12-07T19:34:34.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x409471 2019-12-07T19:34:34.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:34.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:34.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:34.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:34.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000136618s (0.000000091s flush + 0.000136527s kv commit) 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:34.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c578a7800> 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:34.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7800 kv_submitted 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a7800 2019-12-07T19:34:34.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7800 finishing 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a7800 onodes 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a7800 done 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:34.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a7800 [] 2019-12-07T19:34:34.633+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:34.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:34.633+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.633+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:34.641+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.645+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 14 ==== osd_ping(ping e57 up_from 13 ping_stamp 2019-12-07T19:34:34.646448+0000/58.112771402s send_stamp 58.112771402s delta_ub 0.031057686s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c58400 con 0x558c54c5a400 2019-12-07T19:34:34.645+0000 7f8f24c58700 20 osd.7 57 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030748430s,0.031057686s]) 2019-12-07T19:34:34.645+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 14 ==== osd_ping(ping e57 up_from 13 ping_stamp 2019-12-07T19:34:34.646448+0000/58.112771402s send_stamp 58.112771402s delta_ub 0.031057686s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5763ca00 con 0x558c54c5ac00 2019-12-07T19:34:34.645+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e57 up_from 13 ping_stamp 2019-12-07T19:34:34.646448+0000/58.112771402s send_stamp 58.082022972s delta_ub 18446744073.678803186s) v5 -- 0x558c580b9c00 con 0x558c54c5a400 2019-12-07T19:34:34.645+0000 7f8f23c56700 20 osd.7 57 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030725103s,0.031057686s]) 2019-12-07T19:34:34.645+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e57 up_from 13 ping_stamp 2019-12-07T19:34:34.646448+0000/58.112771402s send_stamp 58.082046299s delta_ub 18446744073.678826513s) v5 -- 0x558c58513400 con 0x558c54c5ac00 2019-12-07T19:34:34.691+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.741+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 10 osd.7 57 tick_without_osd_lock 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f498000/0x40000000/0x1680000000, data 0xbc4ddd/0xb66000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 25 osd.7 57 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:32.576231+0000 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 25 osd.7 57 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576218+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 25 osd.7 57 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 25 osd.7 57 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576154+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 25 osd.7 57 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 20 osd.7 57 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 10 osd.7 57 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 20 osd.7 57 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:34.741+0000 7f8f1fbbe700 10 osd.7 57 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:34.791+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.841+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.857+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 17 ==== osd_ping(ping e57 up_from 13 ping_stamp 2019-12-07T19:34:34.857300+0000/58.314223348s send_stamp 58.314223348s delta_ub 0.021980811s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5763c600 con 0x558c54c19000 2019-12-07T19:34:34.857+0000 7f8f23c56700 20 osd.7 57 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021304429s,0.021980811s]) 2019-12-07T19:34:34.857+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e57 up_from 13 ping_stamp 2019-12-07T19:34:34.857300+0000/58.314223348s send_stamp 58.292918919s delta_ub 18446744073.688247187s) v5 -- 0x558c58513400 con 0x558c54c19000 2019-12-07T19:34:34.857+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 17 ==== osd_ping(ping e57 up_from 13 ping_stamp 2019-12-07T19:34:34.857300+0000/58.314223348s send_stamp 58.314223348s delta_ub 0.021980811s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5763c600 con 0x558c54c18800 2019-12-07T19:34:34.857+0000 7f8f23c56700 20 osd.7 57 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021261392s,0.021980811s]) 2019-12-07T19:34:34.857+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e57 up_from 13 ping_stamp 2019-12-07T19:34:34.857300+0000/58.314223348s send_stamp 58.292961956s delta_ub 18446744073.688290224s) v5 -- 0x558c57951800 con 0x558c54c18800 2019-12-07T19:34:34.892+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.942+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:34.992+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.042+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.092+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.142+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.192+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.242+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.255+0000 7f8f21451700 10 osd.7 57 tick 2019-12-07T19:34:35.255+0000 7f8f21451700 10 osd.7 57 do_waiters -- start 2019-12-07T19:34:35.255+0000 7f8f21451700 10 osd.7 57 do_waiters -- finish 2019-12-07T19:34:35.255+0000 7f8f21451700 20 osd.7 57 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:35.293+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.342+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.392+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.443+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.493+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.524+0000 7f8f1219d700 20 osd.7 57 reports for 0 queries 2019-12-07T19:34:35.524+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c579b5500 con 0x558c5476d800 2019-12-07T19:34:35.524+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:35.524+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0xb7f000/0xac5000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:35.524+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(3 pgs tid 0 v 0) v2 -- 0x558c579b5c00 con 0x558c5476d800 2019-12-07T19:34:35.544+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.594+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1350370 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.606+0000 7f8f0998c700 15 osd.7 57 enqueue_peering_evt 1.4s0 epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:35.606+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) 2019-12-07T19:34:35.606+0000 7f8f0998c700 15 osd.7 57 enqueue_peering_evt 1.2s0 epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:35.606+0000 7f8f0998c700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) queued 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) queued 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) pg 0x558c57e88000 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 41 epoch_requested: 41 RenewLease) prio 255 cost 10 e41) pg 0x558c548ab000 2019-12-07T19:34:35.606+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:35.606+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 41 epoch_requested: 41 RenewLease 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041802321s (sent 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041757061s (sent 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.2s2 pg_lease(ru 67.041802321s ub 75.042150448s int 16s) e57/57) v1 -- 0x558c580b9000 con 0x558c54c64000 2019-12-07T19:34:35.606+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.4s2 pg_lease(ru 67.041757061s ub 75.042151890s int 16s) e57/57) v1 -- 0x558c57568800 con 0x558c54c64000 2019-12-07T19:34:35.606+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.2s4 pg_lease(ru 67.041802321s ub 75.042150448s int 16s) e57/57) v1 -- 0x558c580b9200 con 0x558c54c19800 2019-12-07T19:34:35.606+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.4s4 pg_lease(ru 67.041757061s ub 75.042151890s int 16s) e57/57) v1 -- 0x558c5841f400 con 0x558c54c19800 2019-12-07T19:34:35.606+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.2s5 pg_lease(ru 67.041802321s ub 75.042150448s int 16s) e57/57) v1 -- 0x558c57612000 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.4s1 pg_lease(ru 67.041757061s ub 75.042151890s int 16s) e57/57) v1 -- 0x558c580b8000 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.2s1 pg_lease(ru 67.041802321s ub 75.042150448s int 16s) e57/57) v1 -- 0x558c57612600 con 0x558c54c5b400 2019-12-07T19:34:35.606+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.2s3 pg_lease(ru 67.041802321s ub 75.042150448s int 16s) e57/57) v1 -- 0x558c5841e200 con 0x558c54c19c00 2019-12-07T19:34:35.606+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.4s5 pg_lease(ru 67.041757061s ub 75.042151890s int 16s) e57/57) v1 -- 0x558c576b5400 con 0x558c54c5b400 2019-12-07T19:34:35.606+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.4s3 pg_lease(ru 67.041757061s ub 75.042151890s int 16s) e57/57) v1 -- 0x558c5841f600 con 0x558c54c19c00 2019-12-07T19:34:35.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 109 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 75.042151890s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c575cc960 con 0x558c54c19c00 2019-12-07T19:34:35.606+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.4s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 297 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 75.042151890s) e57/57) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c576dcb40 con 0x558c54c5b400 2019-12-07T19:34:35.606+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.4s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 110 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 75.042150448s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c575cc960 con 0x558c54c19c00 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.2s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) pg 0x558c548ab000 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) pg 0x558c57e88000 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.5 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041757061s (sent 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 283 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 75.042150448s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c576dcb40 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.2s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) pg 0x558c548ab000 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041802321s (sent 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 284 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 75.042151890s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c576dcb40 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.4s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 197 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 75.042150448s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5852a000 con 0x558c54c19800 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) pg 0x558c57e88000 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f24c58700 15 osd.7 57 enqueue_peering_evt 1.2s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 198 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 75.042151890s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c5852a000 con 0x558c54c19800 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 298 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 75.042150448s) e57/57) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c576dcb40 con 0x558c54c5b400 2019-12-07T19:34:35.606+0000 7f8f24c58700 15 osd.7 57 enqueue_peering_evt 1.4s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.2s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041757061s (sent 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041802321s (sent 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) pg 0x558c57e88000 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) pg 0x558c548ab000 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.3 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 285 ==== pg_lease(1.fs5 pg_lease(ru 67.064394266s ub 75.064626080s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5763ca00 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.fs5 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064394266s ub 75.064626080s int 16s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064394266s ub 75.064626080s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064394266s ub 75.064626080s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064394266s ub 75.064626080s int 16s)) prio 255 cost 10 e57) pg 0x558c57383000 2019-12-07T19:34:35.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 286 ==== pg_lease(1.1s1 pg_lease(ru 67.064402266s ub 75.064655423s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c5763ca00 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041757061s (sent 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041802321s (sent 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.1s1 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064402266s ub 75.064655423s int 16s) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064402266s ub 75.064655423s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064394266s ub 75.064626080s int 16s) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.064394266s ub 75.064626080s int 16s) 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064402266s ub 75.064655423s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) pg 0x558c57e88000 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064402266s ub 75.064655423s int 16s)) prio 255 cost 10 e57) pg 0x558c5736a000 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.022384101s -> ru 67.042010165s 2019-12-07T19:34:35.606+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.4 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.042010165s 2019-12-07T19:34:35.606+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.3 pg_lease(ru 67.064402266s ub 75.064655423s int 16s) 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.022038874s -> ruub 75.042587206s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.064402266s ub 75.064655423s int 16s) 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.042587206s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041802321s 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.022384101s -> ru 67.042018165s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) pg 0x558c548ab000 2019-12-07T19:34:35.606+0000 7f8f0117b700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.fs0 pg_lease_ack(ruub 75.064626080s) e57/57) v1 -- 0x558c5852a960 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.042018165s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.022038874s -> ruub 75.042616549s 2019-12-07T19:34:35.606+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.2 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.042616549s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041802321s (sent 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f04181700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.1s0 pg_lease_ack(ruub 75.064655423s) e57/57) v1 -- 0x558c57edfc20 con 0x558c54c5bc00 2019-12-07T19:34:35.606+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 67.041757061s 2019-12-07T19:34:35.606+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 67.041757061s (sent 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 485 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 75.042150448s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c575cc960 con 0x558c54c64000 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.2s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042150448s)) prio 255 cost 10 e57) pg 0x558c57e88000 2019-12-07T19:34:35.606+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 486 ==== pg_lease_ack(1.4s0 pg_lease_ack(ruub 75.042151890s) e57/57) v1 ==== 42+0+0 (crc 0 0 0) 0x558c575cc960 con 0x558c54c64000 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.4s0 epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 75.042150448s 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 75.042150448s (sent 75.042150448s) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042151890s)) prio 255 cost 10 e57) pg 0x558c548ab000 2019-12-07T19:34:35.606+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f06986700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLeaseAck epoch 57 from osd.1 pg_lease_ack(ruub 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.3 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.1 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.5 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.2 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until peer osd.4 ruub 75.042151890s 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recalc_readable_until readable_until[_ub] 75.042151890s (sent 75.042151890s) 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:35.606+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.606+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 299 ==== pg_lease(1.3s4 pg_lease(ru 67.056879179s ub 75.057037678s int 16s) e57/57) v1 ==== 58+0+0 (unknown 0 0 0) 0x558c5763ca00 con 0x558c54c5b400 2019-12-07T19:34:35.606+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.3s4 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.056879179s ub 75.057037678s int 16s) 2019-12-07T19:34:35.606+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.056879179s ub 75.057037678s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.056879179s ub 75.057037678s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.056879179s ub 75.057037678s int 16s)) prio 255 cost 10 e57) pg 0x558c5513e000 2019-12-07T19:34:35.606+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.056879179s ub 75.057037678s int 16s) 2019-12-07T19:34:35.606+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.056879179s ub 75.057037678s int 16s) 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.013845196s -> ru 67.043033983s 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.043033983s 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.013595194s -> ruub 75.043442484s 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.043442484s 2019-12-07T19:34:35.606+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.3s0 pg_lease_ack(ruub 75.057037678s) e57/57) v1 -- 0x558c576dd4a0 con 0x558c54c5b400 2019-12-07T19:34:35.606+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:35.606+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.607+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 300 ==== pg_lease(1.bs1 pg_lease(ru 67.057987122s ub 75.058137057s int 16s) e57/57) v1 ==== 58+0+0 (unknown 0 0 0) 0x558c5763ca00 con 0x558c54c5b400 2019-12-07T19:34:35.607+0000 7f8f23c56700 15 osd.7 57 enqueue_peering_evt 1.bs1 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.057987122s ub 75.058137057s int 16s) 2019-12-07T19:34:35.607+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.057987122s ub 75.058137057s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.057987122s ub 75.058137057s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.057987122s ub 75.058137057s int 16s)) prio 255 cost 10 e57) pg 0x558c575ef000 2019-12-07T19:34:35.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.4 pg_lease(ru 67.057987122s ub 75.058137057s int 16s) 2019-12-07T19:34:35.607+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.057987122s ub 75.058137057s int 16s) 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.013845196s -> ru 67.044141926s 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.044141926s 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.013595194s -> ruub 75.044541863s 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.044541863s 2019-12-07T19:34:35.607+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.bs0 pg_lease_ack(ruub 75.058137057s) e57/57) v1 -- 0x558c581e94a0 con 0x558c54c5b400 2019-12-07T19:34:35.607+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:35.607+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.609+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 199 ==== pg_lease(1.5s4 pg_lease(ru 67.081869114s ub 75.082067284s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c54c59200 con 0x558c54c19800 2019-12-07T19:34:35.609+0000 7f8f24c58700 15 osd.7 57 enqueue_peering_evt 1.5s4 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.081869114s ub 75.082067284s int 16s) 2019-12-07T19:34:35.609+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.081869114s ub 75.082067284s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.081869114s ub 75.082067284s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.081869114s ub 75.082067284s int 16s)) prio 255 cost 10 e57) pg 0x558c57522000 2019-12-07T19:34:35.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.081869114s ub 75.082067284s int 16s) 2019-12-07T19:34:35.609+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.081869114s ub 75.082067284s int 16s) 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.036592113s -> ru 67.045277001s 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.045277001s 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.036212722s -> ruub 75.045854562s 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.045854562s 2019-12-07T19:34:35.609+0000 7f8f0217d700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.5s0 pg_lease_ack(ruub 75.082067284s) e57/57) v1 -- 0x558c575ccb40 con 0x558c54c19800 2019-12-07T19:34:35.609+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:35.609+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:35.609+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 200 ==== pg_lease(1.ds2 pg_lease(ru 67.082631240s ub 75.082846331s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c54c59200 con 0x558c54c19800 2019-12-07T19:34:35.609+0000 7f8f24c58700 15 osd.7 57 enqueue_peering_evt 1.ds2 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.082631240s ub 75.082846331s int 16s) 2019-12-07T19:34:35.609+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.082631240s ub 75.082846331s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.082631240s ub 75.082846331s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.609+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.082631240s ub 75.082846331s int 16s)) prio 255 cost 10 e57) pg 0x558c55141000 2019-12-07T19:34:35.609+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.2 pg_lease(ru 67.082631240s ub 75.082846331s int 16s) 2019-12-07T19:34:35.609+0000 7f8f06185700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.082631240s ub 75.082846331s int 16s) 2019-12-07T19:34:35.610+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.036592113s -> ru 67.046039127s 2019-12-07T19:34:35.610+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.046039127s 2019-12-07T19:34:35.610+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.036212722s -> ruub 75.046633609s 2019-12-07T19:34:35.610+0000 7f8f06185700 20 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.046633609s 2019-12-07T19:34:35.610+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.ds0 pg_lease_ack(ruub 75.082846331s) e57/57) v1 -- 0x558c54d38d20 con 0x558c54c19800 2019-12-07T19:34:35.610+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:35.610+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:35.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 487 ==== pg_lease(1.es4 pg_lease(ru 67.068340184s ub 75.068542267s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c57568800 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.es4 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068340184s ub 75.068542267s int 16s) 2019-12-07T19:34:35.610+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068340184s ub 75.068542267s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068340184s ub 75.068542267s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068340184s ub 75.068542267s int 16s)) prio 255 cost 10 e57) pg 0x558c54d98000 2019-12-07T19:34:35.610+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068340184s ub 75.068542267s int 16s) 2019-12-07T19:34:35.610+0000 7f8f05984700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.068340184s ub 75.068542267s int 16s) 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.021980811s -> ru 67.046359373s 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.046359373s 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.021261392s -> ruub 75.047280875s 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.047280875s 2019-12-07T19:34:35.610+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.es0 pg_lease_ack(ruub 75.068542267s) e57/57) v1 -- 0x558c585f94a0 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:35.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 488 ==== pg_lease(1.8s5 pg_lease(ru 67.068518371s ub 75.068731979s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c57568800 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.8s5 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068518371s ub 75.068731979s int 16s) 2019-12-07T19:34:35.610+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068518371s ub 75.068731979s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068518371s ub 75.068731979s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068518371s ub 75.068731979s int 16s)) prio 255 cost 10 e57) pg 0x558c57380000 2019-12-07T19:34:35.610+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068518371s ub 75.068731979s int 16s) 2019-12-07T19:34:35.610+0000 7f8f04982700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.068518371s ub 75.068731979s int 16s) 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.021980811s -> ru 67.046537560s 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.046537560s 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.021261392s -> ruub 75.047470587s 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.047470587s 2019-12-07T19:34:35.610+0000 7f8f04982700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.8s0 pg_lease_ack(ruub 75.068731979s) e57/57) v1 -- 0x558c54df7c20 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:35.610+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:35.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 489 ==== pg_lease(1.9s4 pg_lease(ru 67.068600079s ub 75.068844211s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8e00 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.9s4 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068600079s ub 75.068844211s int 16s) 2019-12-07T19:34:35.610+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068600079s ub 75.068844211s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.610+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 490 ==== pg_lease(1.7s4 pg_lease(ru 67.068728225s ub 75.068898077s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8e00 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.7s4 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068728225s ub 75.068898077s int 16s) 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068728225s ub 75.068898077s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068600079s ub 75.068844211s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068600079s ub 75.068844211s int 16s)) prio 255 cost 10 e57) pg 0x558c5500a000 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068728225s ub 75.068898077s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068728225s ub 75.068898077s int 16s)) prio 255 cost 10 e57) pg 0x558c54d5f000 2019-12-07T19:34:35.610+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068600079s ub 75.068844211s int 16s) 2019-12-07T19:34:35.610+0000 7f8f08189700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.068600079s ub 75.068844211s int 16s) 2019-12-07T19:34:35.610+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.068728225s ub 75.068898077s int 16s) 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.021980811s -> ru 67.046619268s 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.046619268s 2019-12-07T19:34:35.610+0000 7f8f05183700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.068728225s ub 75.068898077s int 16s) 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.021261392s -> ruub 75.047582819s 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.021980811s -> ru 67.046747414s 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.047582819s 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.046747414s 2019-12-07T19:34:35.610+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.9s0 pg_lease_ack(ruub 75.068844211s) e57/57) v1 -- 0x558c5734af00 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.021261392s -> ruub 75.047636685s 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.047636685s 2019-12-07T19:34:35.610+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:35.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.610+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.7s0 pg_lease_ack(ruub 75.068898077s) e57/57) v1 -- 0x558c575ccd20 con 0x558c54c64000 2019-12-07T19:34:35.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:35.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:35.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 491 ==== pg_lease(1.as5 pg_lease(ru 67.069633703s ub 75.069826990s int 16s) e57/57) v1 ==== 58+0+0 (crc 0 0 0) 0x558c580b8e00 con 0x558c54c64000 2019-12-07T19:34:35.611+0000 7f8f24457700 15 osd.7 57 enqueue_peering_evt 1.as5 epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.069633703s ub 75.069826990s int 16s) 2019-12-07T19:34:35.611+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.069633703s ub 75.069826990s int 16s)) prio 255 cost 10 e57) 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.069633703s ub 75.069826990s int 16s)) prio 255 cost 10 e57) queued 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.069633703s ub 75.069826990s int 16s)) prio 255 cost 10 e57) pg 0x558c57e8b000 2019-12-07T19:34:35.611+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 57 epoch_requested: 57 MLease epoch 57 from osd.1 pg_lease(ru 67.069633703s ub 75.069826990s int 16s) 2019-12-07T19:34:35.611+0000 7f8f03980700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 67.069633703s ub 75.069826990s int 16s) 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.021980811s -> ru 67.047652892s 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 67.047652892s 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.021261392s -> ruub 75.048565598s 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 75.048565598s 2019-12-07T19:34:35.611+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.as0 pg_lease_ack(ruub 75.069826990s) e57/57) v1 -- 0x558c57edef00 con 0x558c54c64000 2019-12-07T19:34:35.611+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:35.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:35.631+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 301 ==== osd_map(58..58 src has 1..58) v4 ==== 633+0+0 (unknown 0 0 0) 0x558c579d9d40 con 0x558c54c5b400 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 osd.7 57 OSD::ms_dispatch: osd_map(58..58 src has 1..58) v4 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 osd.7 57 do_waiters -- start 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 osd.7 57 do_waiters -- finish 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 osd.7 57 _dispatch 0x558c579d9d40 osd_map(58..58 src has 1..58) v4 2019-12-07T19:34:35.631+0000 7f8f1199c700 3 osd.7 57 handle_osd_map epochs [58,58], i have 57, src has [1,58] 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 osd.7 57 handle_osd_map got inc map for epoch 58 2019-12-07T19:34:35.631+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3034e826:::osdmap.57:0# 0x0~0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d7340 blob([0xb9d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579d7570 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580f6e70 blob([!~1000,0xb9e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c580f6ee0 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3034e826:::osdmap.57:0# 0x0~1464 = 5220 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 osd.7 57 add_map_bl 57 5220 bytes 2019-12-07T19:34:35.631+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 302 ==== MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=112, reqid=client.4540.0:432, at_version=58'43, trim_to=0'0, roll_forward_to=57'41)) v2 ==== 179474+0+0 (unknown 0 0 0) 0x558c54e3e800 con 0x558c54c5b400 2019-12-07T19:34:35.631+0000 7f8f23c56700 15 osd.7 57 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000040 epoch 58 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=112, reqid=client.4540.0:432, at_version=58'43, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.631+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=112, reqid=client.4540.0:432, at_version=58'43, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 osd.7 57 got_full_map 58, nothing requested 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 osd.7 57 handle_osd_map pg_num_history pg_num_history(e58 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {58={}} 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 osd.7 57 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e58 [1,58] lci=[13,58]) 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c578aaf00 seq 49 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c327972f:::inc_osdmap.58:0# key 0x7f7fffffffffffffffc32797'/!inc_osdmap.58!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:35.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=112, reqid=client.4540.0:432, at_version=58'43, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:35.631+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=112, reqid=client.4540.0:432, at_version=58'43, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) pg 0x558c575ef000 2019-12-07T19:34:35.631+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c327972f:::inc_osdmap.58:0# 0x0~24d 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1270 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c327972f:::inc_osdmap.58:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aaf00 1 blobs 2019-12-07T19:34:35.631+0000 7f8f07187700 10 osd.7 57 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000163 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=112, reqid=client.4540.0:432, at_version=58'43, trim_to=0'0, roll_forward_to=57'41)) v2 pg pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2a6000~1000/1000,1000,0 2019-12-07T19:34:35.631+0000 7f8f07187700 10 osd.7 57 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 57 -> 58 (as per caller) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2a6000~1000] 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d75e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b38f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d75e0 blob([0x2a6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b38f0 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c580d75e0) get_ref 0x0~24d Blob(0x558c580d75e0 blob([0x2a6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b38f0 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c580d75e0) get_ref init 0x1000, 1000 2019-12-07T19:34:35.631+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=112, reqid=client.4540.0:432, at_version=58'43, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c580d75e0 blob([0x2a6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c550b38f0 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c550b3908 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.add_oid 1:d07269f7:::smithi13140796-28:23 1b,20,21,23 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851ed88) dirty_range mark inline shard dirty 2019-12-07T19:34:35.631+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c327972f:::inc_osdmap.58:0# 0x0~24d = 0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3bb4e826:::osdmap.58:0# key 0x7f7fffffffffffffff3bb4e8'&!osdmap.58!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:35.631+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3bb4e826:::osdmap.58:0# 0x0~1448 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1271 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3bb4e826:::osdmap.58:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~448 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aaf00 2 blobs 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2a7000~2000/1000,2000,0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2a7000~2000] 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5898b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55106000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5898b0a0 blob([0x2a7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106000 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c5898b0a0) get_ref 0x0~1000 Blob(0x558c5898b0a0 blob([0x2a7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106000 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c5898b0a0) get_ref init 0x1000, 1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5898b0a0 blob([0x2a7000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55106000 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55106018 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574d2b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574d2b60 blob([!~1000,0x2a8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90f50 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c574d2b60) get_ref 0x1000~448 Blob(0x558c574d2b60 blob([!~1000,0x2a8000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57f90f50 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c574d2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558c574d2b60 blob([!~1000,0x2a8000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57f90f50 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57f90f68 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1448 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f008) dirty_range mark inline shard dirty 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3bb4e826:::osdmap.58:0# 0x0~1448 = 0 2019-12-07T19:34:35.631+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aaf00 0 blobs 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:35.631+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:35.631+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:35.631+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aaf00 0 blobs 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578aaf00 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578aaf00 onodes 0x558c53a91900,0x558c55276c80,0x558c5851ec80,0x558c5851ef00 shared_blobs 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851ed88) update #-1:c327972f:::inc_osdmap.58:0# 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851ed88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c327972f:::inc_osdmap.58:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f008) update #-1:3bb4e826:::osdmap.58:0# 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f008) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3bb4e826:::osdmap.58:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578aaf00 allocated 0x[2a6000~3000] released 0x[] 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 freelist allocate 0x2a6000~3000 2019-12-07T19:34:35.631+0000 7f8f1199c700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aaf00 prepare 2019-12-07T19:34:35.631+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c578aaf00 2019-12-07T19:34:35.631+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:35.631+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.get_snaps 1:d07269f7:::smithi13140796-28:23 got.empty() 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.set_snaps 1:d07269f7:::smithi13140796-28:23 1b,20,21,23 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_0000000000000021_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.631+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_0000000000000023_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.631+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,57'41], crt=57'41) [58'42 (45'29) clone 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:22.635389+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,58'43 (45'29) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:432 2019-12-07T19:34:35.629323+0000 0 ObjectCleanRegions clean_offsets: [0~1343488,2048000~18446744073707503615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.631+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 58'42 (0'0,58'42] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'42 (45'29) clone 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:22.635389+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.631+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'43 (45'29) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:432 2019-12-07T19:34:35.629323+0000 0 ObjectCleanRegions clean_offsets: [0~1343488,2048000~18446744073707503615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.632+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 43 2019-12-07T19:34:35.632+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.632+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.632+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'42, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586e0900 seq 52 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:d07269f7:::smithi13140796-28:23# key 0x818000000000000001d07269f721736d'ithi13140796-28!='0x0000000000000023ffffffffffffffff'o' 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:23# 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1272 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.onode(0x558c54e44780).flush flush done 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# 0x0 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:23# 0x0~52000 -> 0x0~52000 2019-12-07T19:34:35.632+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aaf00 aio_wait 2019-12-07T19:34:35.632+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578aaf00 2019-12-07T19:34:35.632+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aaf00 io_done 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x0~2000: 0x0~2000 Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10766 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c5840f2d0 blob([0x999000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c5840f2d0 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6150 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6150) get_ref 0x0~2000 Blob(0x558c580f6150 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6150) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x0~2000: 0x0~2000 Blob(0x558c580f6150 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2000~2000: 0x0~2000 Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10767 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c583eac40 blob([0x99b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c583eac40 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c579d7650 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c579d7650) get_ref 0x0~2000 Blob(0x558c579d7650 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c579d7650) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c579d7650 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4000~2000: 0x0~2000 Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10768 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c5840f3b0 blob([0x99d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c5840f3b0 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c578809a0 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851ec80 had 1 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851ef00 had 1 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c578809a0) get_ref 0x0~2000 Blob(0x558c578809a0 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c578809a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c578809a0 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x6000~2000: 0x0~2000 Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10769 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3856596992 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c578f1570 blob([0x99f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c578f1570 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6230 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997694) bluestore 86 GiB free (0.959876), bluefs_ratio 0.0399168 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6230) get_ref 0x0~2000 Blob(0x558c580f6230 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6230) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c580f6230 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x8000~2000: 0x0~2000 Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10770 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c550b2e00 blob([0x9a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c550b2e00 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f62a0 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f62a0) get_ref 0x0~2000 Blob(0x558c580f62a0 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f62a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c580f62a0 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x409471~226c to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xa000~2000: 0x0~2000 Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10771 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x409471 0x409471~226c to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c583eb960 blob([0x9a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c583eb960 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3d9471 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x471 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6380 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6dd and padding block with 0x923 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6380) get_ref 0x0~2000 Blob(0x558c580f6380 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6380) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c580f6380 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10772 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970fc0 blob([0x9a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970fc0 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f63f0 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f63f0) get_ref 0x0~2000 Blob(0x558c580f63f0 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f63f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c580f63f0 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10773 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970ee0 blob([0x9a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970ee0 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6460 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6460) get_ref 0x0~2000 Blob(0x558c580f6460 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6460) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c580f6460 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x10000~2000: 0x0~2000 Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10774 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c578199d0 blob([0x9a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c578199d0 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f64d0 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f64d0) get_ref 0x0~2000 Blob(0x558c580f64d0 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f64d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c580f64d0 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10775 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970e00 blob([0x9ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970e00 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6540 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6540) get_ref 0x0~2000 Blob(0x558c580f6540 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6540) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c580f6540 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10776 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57818d20 blob([0x9ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57818d20 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f65b0 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f65b0) get_ref 0x0~2000 Blob(0x558c580f65b0 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f65b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c580f65b0 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40b6dd 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10777 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970d20 blob([0x9af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970d20 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6620 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6620) get_ref 0x0~2000 Blob(0x558c580f6620 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000204370s (0.000004339s flush + 0.000200031s kv commit) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6620) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c580f6620 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x18000~2000: 0x0~2000 Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10778 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c578aaf00> 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57818cb0 blob([0x9b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aaf00 kv_submitted 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578aaf00 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57818cb0 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c578aaf00 osr 0x558c54ea4b40 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6690 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 46 0x1b000~1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6690) get_ref 0x0~2000 Blob(0x558c580f6690 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6690) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 47 0x1b000~1000 crc c6d7813e 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c580f6690 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 46 0x2000~1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 47 0x2000~1000 crc 75a333d4 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10779 2019-12-07T19:34:35.632+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970c40 blob([0x9b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970c40 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6700 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6700) get_ref 0x0~2000 Blob(0x558c580f6700 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6700) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c580f6700 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10780 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6770 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6770) get_ref 0x0~2000 Blob(0x558c580f6770 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6770) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c580f6770 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10781 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970b60 blob([0x9b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970b60 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f67e0 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f67e0) get_ref 0x0~2000 Blob(0x558c580f67e0 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f67e0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c580f67e0 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x20000~2000: 0x0~2000 Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10782 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57819a40 blob([0x9b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57819a40 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6850 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6850) get_ref 0x0~2000 Blob(0x558c580f6850 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6850) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c580f6850 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x22000~1000: 0x0~1000 Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10783 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970a80 blob([0x9bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970a80 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f68c0 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f68c0) get_ref 0x0~1000 Blob(0x558c580f68c0 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f68c0) get_ref init 0x1000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x22000~1000: 0x0~1000 Blob(0x558c580f68c0 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x23000~2000: 0x0~2000 Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10784 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c550b36c0 blob([0x9bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c550b36c0 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6930 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6930) get_ref 0x0~2000 Blob(0x558c580f6930 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6930) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c580f6930 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x25000~2000: 0x0~2000 Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10785 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c579702a0 blob([0x9be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c579702a0 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f69a0 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f69a0) get_ref 0x0~2000 Blob(0x558c580f69a0 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7 57 _committed_osd_maps 58..58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f69a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7 57 advance to epoch 58 (<= last 58 <= newest_map 58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c580f69a0 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x27000~2000: 0x0~2000 Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10786 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57970230 blob([0x9c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 7 osd.7 58 consume_map version 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57970230 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.8s5 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6a10 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.0s3 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.1s1 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6a10) get_ref 0x0~2000 Blob(0x558c580f6a10 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6a10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.9s4 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.as5 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c580f6a10 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.2s0 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.bs1 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x29000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.3s4 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10787 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.cs0 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.4s0 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.5s4 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.ds2 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.es4 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6a80 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.6s2 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.fs5 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6a80) get_ref 0x0~2000 Blob(0x558c580f6a80 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6a80) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 identify_splits_and_merges 1.7s4 e57 to e58 pg_nums {18=16} 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 58 prune_pg_created 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c580f6a80 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:0.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10788 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:1.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c578f1810 blob([0x9c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:2.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c578f1810 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:3.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:4.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6af0 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:5.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6af0) get_ref 0x0~2000 Blob(0x558c580f6af0 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6af0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:6.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7:7.consume_map 58 (was 57) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c580f6af0 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10789 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574fd570 blob([0x9c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 10 osd.7 58 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574fd570 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6b60 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.8s5 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6b60) get_ref 0x0~2000 Blob(0x558c580f6b60 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6b60) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c580f6b60 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2f000~2000: 0x0~2000 Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10790 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.0s3 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55106e00 blob([0x9c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.1s1 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55106e00 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6bd0 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6bd0) get_ref 0x0~2000 Blob(0x558c580f6bd0 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.9s4 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2f000~2000: 0x0~2000 Blob(0x558c580f6bd0 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.as5 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x31000~2000: 0x0~2000 Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10791 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55106ee0 blob([0x9ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.2s0 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55106ee0 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.bs1 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6c40 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6c40) get_ref 0x0~2000 Blob(0x558c580f6c40 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6c40) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.3s4 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x31000~2000: 0x0~2000 Blob(0x558c580f6c40 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.cs0 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x33000~2000: 0x0~2000 Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10792 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55106fc0 blob([0x9cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55106fc0 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.4s0 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6cb0 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.5s4 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6cb0) get_ref 0x0~2000 Blob(0x558c580f6cb0 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x33000~2000: 0x0~2000 Blob(0x558c580f6cb0 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.ds2 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x35000~2000: 0x0~2000 Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10793 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c551070a0 blob([0x9ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.es4 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c551070a0 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.6s2 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6d20 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.fs5 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6d20) get_ref 0x0~2000 Blob(0x558c580f6d20 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6d20) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x35000~2000: 0x0~2000 Blob(0x558c580f6d20 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x37000~2000: 0x0~2000 Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10794 2019-12-07T19:34:35.632+0000 7f8f141a1700 15 osd.7 58 enqueue_peering_evt 1.7s4 epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107180 blob([0x9d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107180 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f141a1700 7 osd.7 58 activate_map version 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c580f6d90 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6d90) get_ref 0x0~2000 Blob(0x558c580f6d90 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c580f6d90) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x37000~2000: 0x0~2000 Blob(0x558c580f6d90 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c57522000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x39000~2000: 0x0~2000 Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10795 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107260 blob([0x9d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107260 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764000 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764000) get_ref 0x0~2000 Blob(0x558c58764000 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764000) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x39000~2000: 0x0~2000 Blob(0x558c58764000 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x3b000~2000: 0x0~2000 Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 57 -> 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10796 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107340 blob([0x9d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c5736a000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107340 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764070 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764070) get_ref 0x0~2000 Blob(0x558c58764070 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764070) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x3b000~2000: 0x0~2000 Blob(0x558c58764070 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x3d000~2000: 0x0~2000 Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10797 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57e633b0 blob([0x9d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57e633b0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587640e0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587640e0) get_ref 0x0~2000 Blob(0x558c587640e0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587640e0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.632+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x3d000~2000: 0x0~2000 Blob(0x558c587640e0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 57 -> 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x3f000~2000: 0x0~2000 Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10798 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c551079d0 blob([0x9d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c551079d0 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764150 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764150) get_ref 0x0~2000 Blob(0x558c58764150 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764150) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x3f000~2000: 0x0~2000 Blob(0x558c58764150 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x41000~2000: 0x0~2000 Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10799 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57ed1e30 blob([0x9da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57ed1e30 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587641c0 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587641c0) get_ref 0x0~2000 Blob(0x558c587641c0 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587641c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x41000~2000: 0x0~2000 Blob(0x558c587641c0 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x43000~2000: 0x0~2000 Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10800 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107ab0 blob([0x9dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107ab0 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764230 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764230) get_ref 0x0~2000 Blob(0x558c58764230 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764230) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x43000~2000: 0x0~2000 Blob(0x558c58764230 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x45000~2000: 0x0~2000 Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10801 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107b90 blob([0x9de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107b90 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587642a0 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587642a0) get_ref 0x0~2000 Blob(0x558c587642a0 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587642a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x45000~2000: 0x0~2000 Blob(0x558c587642a0 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x47000~2000: 0x0~2000 Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10802 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107c70 blob([0x9e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107c70 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764310 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764310) get_ref 0x0~2000 Blob(0x558c58764310 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764310) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x47000~2000: 0x0~2000 Blob(0x558c58764310 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x49000~2000: 0x0~2000 Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10803 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107d50 blob([0x9e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107d50 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764380 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c55141000 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764380) get_ref 0x0~2000 Blob(0x558c58764380 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764380) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x49000~2000: 0x0~2000 Blob(0x558c58764380 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4b000~2000: 0x0~2000 Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10804 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107e30 blob([0x9e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107e30 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587643f0 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587643f0) get_ref 0x0~2000 Blob(0x558c587643f0 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587643f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4b000~2000: 0x0~2000 Blob(0x558c587643f0 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4d000~2000: 0x0~2000 Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10805 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 57 -> 58 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55107f10 blob([0x9e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55107f10 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764460 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764460) get_ref 0x0~2000 Blob(0x558c58764460 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764460) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 57 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:35.632+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4d000~2000: 0x0~2000 Blob(0x558c58764460 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4f000~2000: 0x0~2000 Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10806 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c580a2000 blob([0x9e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c580a2000 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=1)))) 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587644d0 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587644d0) get_ref 0x0~2000 Blob(0x558c587644d0 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587644d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c5500a000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4f000~2000: 0x0~2000 Blob(0x558c587644d0 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x51000~1000: 0x1000~1000 Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10807 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=1)))) 2019-12-07T19:34:35.632+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58764540 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764540) get_ref 0x1000~1000 Blob(0x558c58764540 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764540) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x51000~1000: 0x1000~1000 Blob(0x558c58764540 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=2)))) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark inline shard dirty 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f35508) dirty_range mark inline shard dirty 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:23# = 0 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# snapset 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# snapset = 0 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 57 -> 58 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# 1 keys 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# 1 keys = 0 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# hinfo_key (42 bytes) 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 57 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:35.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x52000~2b000 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0x52000~2b000 - have 0x52000 (335872) bytes fadvise_flags 0x0 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:35.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x52000~2b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.632+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c586e0900 22 blobs 2019-12-07T19:34:35.632+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c57380000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.632+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.632+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:35.632+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 57 -> 58 2019-12-07T19:34:35.632+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.632+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2b000/1000,2b000,0 2019-12-07T19:34:35.632+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x2a9000~d000/1000,2b000,0 2019-12-07T19:34:35.632+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x350000~1e000/1000,2b000,0 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x2a9000~d000,0x350000~1e000] 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c587645b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c587645b0 blob([0x2a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764620 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587645b0) get_ref 0x0~2000 Blob(0x558c587645b0 blob([0x2a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764620 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c57383000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c587645b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c587645b0 blob([0x2a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764620 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764638 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764690 blob([0x2ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764700 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764690) get_ref 0x0~2000 Blob(0x558c58764690 blob([0x2ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764700 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 57 -> 58 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.blob(0x558c58764690) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c58764690 blob([0x2ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764700 sbid 0x0)) 2019-12-07T19:34:35.632+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764718 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.632+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587647e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764770 blob([0x2ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587647e0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764770) get_ref 0x0~2000 Blob(0x558c58764770 blob([0x2ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587647e0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764770) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c58764770 blob([0x2ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587647e0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587647f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587648c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764850 blob([0x2af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587648c0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c54d98000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764850) get_ref 0x0~2000 Blob(0x558c58764850 blob([0x2af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587648c0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764850) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c58764850 blob([0x2af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587648c0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587648d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587649a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764930 blob([0x2b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587649a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764930) get_ref 0x0~2000 Blob(0x558c58764930 blob([0x2b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587649a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764930) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c58764930 blob([0x2b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587649a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587649b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764a10 blob([0x2b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764a80 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764a10) get_ref 0x0~2000 Blob(0x558c58764a10 blob([0x2b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764a80 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764a10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0918b700 20 osd.7 58 agent_entry empty queue 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~2000: 0x0~2000 Blob(0x558c58764a10 blob([0x2b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764a80 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764a98 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764af0 blob([0x2b5000~1000,0x350000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764b60 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764af0) get_ref 0x0~2000 Blob(0x558c58764af0 blob([0x2b5000~1000,0x350000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764b60 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764af0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c57676000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5e000~2000: 0x0~2000 Blob(0x558c58764af0 blob([0x2b5000~1000,0x350000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764b60 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764bd0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764c40 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764bd0) get_ref 0x0~2000 Blob(0x558c58764bd0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764c40 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x60000~2000: 0x0~2000 Blob(0x558c58764bd0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764c40 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] state: Active advmap 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764cb0 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764d20 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764cb0) get_ref 0x0~2000 Blob(0x558c58764cb0 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764d20 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.632+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x62000~2000: 0x0~2000 Blob(0x558c58764cb0 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764d20 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764d90 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764e00 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764d90) get_ref 0x0~2000 Blob(0x558c58764d90 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764e00 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764d90) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x64000~2000: 0x0~2000 Blob(0x558c58764d90 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764e00 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764e70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764ee0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764e70) get_ref 0x0~2000 Blob(0x558c58764e70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764ee0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764e70) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c57e8b000 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x66000~2000: 0x0~2000 Blob(0x558c58764e70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764ee0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764ef8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c5500d000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58764f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58764fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 7 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58764f50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764fc0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764f50) get_ref 0x0~2000 Blob(0x558c58764f50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764fc0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58764f50) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x68000~2000: 0x0~2000 Blob(0x558c58764f50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764fc0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58764fd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58765030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587650a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58765030 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587650a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:192 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765030) get_ref 0x0~2000 Blob(0x558c58765030 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587650a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 57 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765030) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6a000~2000: 0x0~2000 Blob(0x558c58765030 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587650a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587650b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58765110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58765180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58765110 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765180 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765110) get_ref 0x0~2000 Blob(0x558c58765110 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765180 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765110) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6c000~2000: 0x0~2000 Blob(0x558c58765110 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765180 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58765198 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c587651f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58765260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c548ab000 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c587651f0 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765260 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c587651f0) get_ref 0x0~2000 Blob(0x558c587651f0 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765260 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c587651f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6e000~2000: 0x0~2000 Blob(0x558c587651f0 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765260 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58765278 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c587652d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58765340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c587652d0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765340 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c587652d0) get_ref 0x0~2000 Blob(0x558c587652d0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765340 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c587652d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x70000~2000: 0x0~2000 Blob(0x558c587652d0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765340 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58765358 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 57 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c587653b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58765420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c587653b0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765420 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] state: Active advmap 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c587653b0) get_ref 0x0~2000 Blob(0x558c587653b0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765420 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c587653b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x72000~2000: 0x0~2000 Blob(0x558c587653b0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765420 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58765438 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58765490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58765500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58765490 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765500 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765490) get_ref 0x0~2000 Blob(0x558c58765490 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765500 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c57e88000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765490) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x74000~2000: 0x0~2000 Blob(0x558c58765490 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765500 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58765518 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58765570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587655e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58765570 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587655e0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765570) get_ref 0x0~2000 Blob(0x558c58765570 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587655e0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s0 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c54d5f000 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 57 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] state: Active advmap 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765570) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x76000~2000: 0x0~2000 Blob(0x558c58765570 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587655e0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587655f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 57 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58765650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587656c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f0297e700 7 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58765650 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587656c0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765650) get_ref 0x0~2000 Blob(0x558c58765650 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587656c0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765650) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x78000~2000: 0x0~2000 Blob(0x558c58765650 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587656c0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587656d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:35.633+0000 7f8f0297e700 15 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:154 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58765730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587657a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58765730 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587657a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765730) get_ref 0x0~2000 Blob(0x558c58765730 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587657a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765730) get_ref init 0x2000, 1000 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x7a000~2000: 0x0~2000 Blob(0x558c58765730 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587657a0 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587657b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07988700 7 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:35.633+0000 7f8f07988700 15 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:166 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07988700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c54d5c000 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 57 -> 58 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 57 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.633+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.633+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:35.633+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58765810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58765880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58765810 blob([0x36d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765880 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765810) get_ref 0x0~1000 Blob(0x558c58765810 blob([0x36d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765880 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.blob(0x558c58765810) get_ref init 0x1000, 1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x7c000~1000: 0x0~1000 Blob(0x558c58765810 blob([0x36d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58765880 sbid 0x0)) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58765898 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x7d000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark inline shard dirty 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x52000~2b000 = 0 2019-12-07T19:34:35.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- SNA_1_000000000000001B_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- SNA_1_0000000000000020_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- SNA_1_0000000000000021_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- SNA_1_0000000000000023_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000058.00000000000000000042' <- 0000000058.00000000000000000042 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000058.00000000000000000043' <- 0000000058.00000000000000000043 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e0900 cost 276983 (24 ios * 4000 + 180983 bytes) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e0900 onodes 0x558c54e44780,0x558c57f35400 shared_blobs 0x558c54677810,0x558c54cf96c0,0x558c550b2ee0,0x558c550b3650,0x558c55106e70,0x558c55106f50,0x558c55107030,0x558c55107110,0x558c551071f0,0x558c551072d0,0x558c551073b0,0x558c55107a40,0x558c55107b20,0x558c55107c00,0x558c55107ce0,0x558c55107dc0,0x558c55107ea0,0x558c55107f80,0x558c552790a0,0x558c574fd960,0x558c57567110,0x558c576ad650,0x558c576adb90,0x558c57897c70,0x558c57970af0,0x558c57970bd0,0x558c57970cb0,0x558c57970d90,0x558c57970e70,0x558c579718f0,0x558c57971960,0x558c579719d0,0x558c57971a40,0x558c57971ab0,0x558c57971b20,0x558c57971b90,0x558c579b22a0,0x558c580a2070,0x558c580a2150,0x558c580d79d0,0x558c583ead20,0x558c5840f420 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update inline shard 1569 bytes from 64 extents 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore.extentmap(0x558c54e44888) reshard 0x[0,ffffffff) of 0 shards on 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard extent_avg 24, target 500, slop 100 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x0 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x27000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x4f000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x76000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard new [0x0(0x0 bytes),0x27000(0x0 bytes),0x4f000(0x0 bytes),0x76000(0x0 bytes)] 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard old [] 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard fin [0x0(0x0 bytes),0x27000(0x0 bytes),0x4f000(0x0 bytes),0x76000(0x0 bytes)] 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# force 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update shard 0x0 is 541 bytes (was 0) from 20 extents 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update shard 0x27000 is 545 bytes (was 0) from 20 extents 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update shard 0x4f000 is 420 bytes (was 0) from 20 extents 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update shard 0x76000 is 78 bytes (was 0) from 4 extents 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 601 (599 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f35508) update 1#1:d07269f7:::smithi13140796-28:23# 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f35508) update inline shard 1148 bytes from 42 extents 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:23# is 1567 (413 bytes onode + 2 bytes spanning blobs + 1152 bytes inline extents) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a23 is 11 SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2f is 11 SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a25 is 11 SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a20 is 11 SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a26 is 11 SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a27 is 11 SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a28 is 11 SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a29 is 11 SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2a is 11 SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2b is 11 SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2c is 11 SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2e is 11 SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a30 is 11 SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a31 is 11 SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a32 is 11 SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a33 is 11 SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a34 is 11 SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a35 is 11 SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a22 is 11 SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a24 is 11 SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2d is 11 SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a13 is 11 SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a21 is 11 SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0f is 11 SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1c is 11 SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1a is 11 SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a18 is 11 SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a16 is 11 SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a14 is 11 SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1f is 11 SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1d is 11 SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1b is 11 SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a19 is 11 SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a17 is 11 SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a15 is 11 SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a10 is 11 SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1e is 11 SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a36 is 11 SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a37 is 11 SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0e is 11 SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a11 is 11 SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a12 is 11 SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=2))) 2019-12-07T19:34:35.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e0900 allocated 0x[2a9000~d000,350000~1e000] released 0x[] 2019-12-07T19:34:35.633+0000 7f8f07187700 10 freelist allocate 0x2a9000~d000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:35.633+0000 7f8f07187700 10 freelist allocate 0x350000~1e000 2019-12-07T19:34:35.633+0000 7f8f07187700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0900 prepare 2019-12-07T19:34:35.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c586e0900 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'43 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 58 dequeue_op 0x558c54e78840 finish 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) queued 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c5513e000 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 NullEvt) prio 255 cost 10 e58) pg 0x558c575ef000 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 57 -> 58 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 58 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 57 -> 58 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 57 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 57 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:35.634+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:35.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 58 advance_pg new pool opts old pool opts 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 57'41 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.634+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 58 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 58 2019-12-07T19:34:35.634+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:35.634+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 NullEvt 2019-12-07T19:34:35.634+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.634+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0900 aio_wait 2019-12-07T19:34:35.634+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e0900 2019-12-07T19:34:35.634+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0900 io_done 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8143 < min_flush_size 524288 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35400 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8166 < min_flush_size 524288 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8166 < min_flush_size 524288 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40b6dd~1fe6 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40b6dd 0x40b6dd~1fe6 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3db6dd 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6dd 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6c3 and padding block with 0x93d 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40d6c3 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.634+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 492 ==== osd_map(58..58 src has 1..58) v4 ==== 633+0+0 (crc 0 0 0) 0x558c58763680 con 0x558c54c64000 2019-12-07T19:34:35.634+0000 7f8f1199c700 20 osd.7 58 OSD::ms_dispatch: osd_map(58..58 src has 1..58) v4 2019-12-07T19:34:35.634+0000 7f8f1199c700 10 osd.7 58 do_waiters -- start 2019-12-07T19:34:35.634+0000 7f8f1199c700 10 osd.7 58 do_waiters -- finish 2019-12-07T19:34:35.634+0000 7f8f1199c700 20 osd.7 58 _dispatch 0x558c58763680 osd_map(58..58 src has 1..58) v4 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138267s (0.000003875s flush + 0.000134392s kv commit) 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.634+0000 7f8f1199c700 3 osd.7 58 handle_osd_map epochs [58,58], i have 58, src has [1,58] 2019-12-07T19:34:35.634+0000 7f8f1199c700 10 osd.7 58 no new maps here, dropping 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586e0900> 2019-12-07T19:34:35.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 493 ==== MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=290, reqid=client.4540.0:437, at_version=58'6, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 3175+0+0 (crc 0 0 0) 0x558c57a45800 con 0x558c54c64000 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0900 kv_submitted 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e0900 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0900 finishing 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e0900 onodes 0x558c54e44780,0x558c57f35400 2019-12-07T19:34:35.634+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511fa20 prio 127 cost 0 latency 0.000031 epoch 58 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=290, reqid=client.4540.0:437, at_version=58'6, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764638 in 0x558c53a59c00) _finish_write discard buffer(0x558c57b8e1b0 space 0x558c58764638 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.634+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=290, reqid=client.4540.0:437, at_version=58'6, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764718 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c16480 space 0x558c58764718 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587647f8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f4e870 space 0x558c587647f8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587648d8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f4e7e0 space 0x558c587648d8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587649b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57f4e6c0 space 0x558c587649b8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764a98 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c16d80 space 0x558c58764a98 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764b78 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c16cf0 space 0x558c58764b78 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764c58 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c16c60 space 0x558c58764c58 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764d38 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c16bd0 space 0x558c58764d38 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764e18 in 0x558c53a59c00) _finish_write discard buffer(0x558c579822d0 space 0x558c58764e18 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=290, reqid=client.4540.0:437, at_version=58'6, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764ef8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e145a0 space 0x558c58764ef8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58764fd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c579823f0 space 0x558c58764fd8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587650b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbb950 space 0x558c587650b8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=290, reqid=client.4540.0:437, at_version=58'6, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) pg 0x558c54d98000 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58765198 in 0x558c53a59c00) _finish_write discard buffer(0x558c57982480 space 0x558c58765198 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58765278 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbba70 space 0x558c58765278 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58765358 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e14000 space 0x558c58765358 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58765438 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbbb00 space 0x558c58765438 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58765518 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbbb90 space 0x558c58765518 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587655f8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbbc20 space 0x558c587655f8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587656d8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbbcb0 space 0x558c587656d8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587657b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbbd40 space 0x558c587657b8 0x0~2000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58765898 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbbdd0 space 0x558c58765898 0x0~1000 writing nocache) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0900 done 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 58 dequeue_op 0x558c5511fa20 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=290, reqid=client.4540.0:437, at_version=58'6, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e0900 [] 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.634+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 58/41 ECSubWriteReply(tid=112, last_complete=58'43, committed=1, applied=1)) v2 -- 0x558c576b5c00 con 0x558c54c5b400 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 58 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 57 -> 58 (as per caller) 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=290, reqid=client.4540.0:437, at_version=58'6, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.add_oid 1:733d7474:::smithi13140796-5:23 1b,20,21,23 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.get_snaps 1:733d7474:::smithi13140796-5:23 got.empty() 2019-12-07T19:34:35.634+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.set_snaps 1:733d7474:::smithi13140796-5:23 1b,20,21,23 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_0000000000000021_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_0000000000000023_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,21'4], crt=21'4) [58'5 (21'2) clone 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:33:58.627337+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,58'6 (21'2) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:437 2019-12-07T19:34:35.633816+0000 0 ObjectCleanRegions clean_offsets: [0~81920,81920~18446744073709469695], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'5 (0'0,58'5] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'5 (21'2) clone 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:33:58.627337+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'6 (0'0,58'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'6 (21'2) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:437 2019-12-07T19:34:35.633816+0000 0 ObjectCleanRegions clean_offsets: [0~81920,81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'6 (0'0,58'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 6 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'6 (0'0,58'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.634+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.634+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c575c5b00 seq 23 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) get_onode oid 4#1:733d7474:::smithi13140796-5:23# key 0x848000000000000001733d74't!smithi13140796-5!='0x0000000000000023ffffffffffffffff'o' 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) r -2 v.len 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.es4_head 4#1:733d7474:::smithi13140796-5:head# -> 4#1:733d7474:::smithi13140796-5:23# 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1273 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.onode(0x558c54ce3180).flush flush done 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.es4_head 4#1:733d7474:::smithi13140796-5:23# 0x0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.es4_head 4#1:733d7474:::smithi13140796-5:head# -> 4#1:733d7474:::smithi13140796-5:23# 0x0~5000 -> 0x0~5000 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup src 0x0~2000: 0x0~2000 Blob(0x558c57896380 blob([0x6b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 sbid 0x0)) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10808 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) make_blob_shared Blob(0x558c57896380 blob([0x6b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 sbid 0x0)) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) make_blob_shared now Blob(0x558c57896380 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=1)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup new Blob(0x558c5809bb90 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.blob(0x558c5809bb90) get_ref 0x0~2000 Blob(0x558c5809bb90 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.blob(0x558c5809bb90) get_ref init 0x2000, 1000 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup dst 0x0~2000: 0x0~2000 Blob(0x558c5809bb90 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup src 0x2000~2000: 0x0~2000 Blob(0x558c578962a0 blob([0x6b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 sbid 0x0)) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10809 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) make_blob_shared Blob(0x558c578962a0 blob([0x6b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 sbid 0x0)) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) make_blob_shared now Blob(0x558c578962a0 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=1)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup new Blob(0x558c550b2a80 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.blob(0x558c550b2a80) get_ref 0x0~2000 Blob(0x558c550b2a80 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.blob(0x558c550b2a80) get_ref init 0x2000, 1000 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c550b2a80 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup src 0x4000~1000: 0x0~1000 Blob(0x558c57896070 blob([0x6bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 sbid 0x0)) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10810 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) make_blob_shared Blob(0x558c57896070 blob([0x6bb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 sbid 0x0)) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) make_blob_shared now Blob(0x558c57896070 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=1)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup new Blob(0x558c58766e00 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.blob(0x558c58766e00) get_ref 0x0~1000 Blob(0x558c58766e00 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.blob(0x558c58766e00) get_ref init 0x1000, 1000 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dup dst 0x4000~1000: 0x0~1000 Blob(0x558c58766e00 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=2)))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) dirty_range mark inline shard dirty 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c579deb08) dirty_range mark inline shard dirty 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.es4_head 4#1:733d7474:::smithi13140796-5:head# -> 4#1:733d7474:::smithi13140796-5:23# = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# snapset 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# snapset = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:23# 1 keys 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:23# 1 keys = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# hinfo_key (42 bytes) 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- SNA_1_000000000000001B_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- SNA_1_0000000000000020_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- SNA_1_0000000000000021_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- SNA_1_0000000000000023_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000058.00000000000000000005' <- 0000000058.00000000000000000005 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000058.00000000000000000006' <- 0000000058.00000000000000000006 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c575c5b00 cost 8693 (1 ios * 4000 + 4693 bytes) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c575c5b00 onodes 0x558c54ce3180,0x558c579dea00 shared_blobs 0x558c54df8150,0x558c57896310,0x558c57896540 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c54ce3288) update inline shard 81 bytes from 3 extents 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 666 (579 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c579deb08) update 4#1:733d7474:::smithi13140796-5:23# 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore.extentmap(0x558c579deb08) update inline shard 81 bytes from 3 extents 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:23# is 497 (410 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a39 is 11 SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=2))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a38 is 11 SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=2))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3a is 11 SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=2))) 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c575c5b00 allocated 0x[] released 0x[] 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5b00 prepare 2019-12-07T19:34:35.634+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c575c5b00 2019-12-07T19:34:35.634+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5b00 io_done 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:35.634+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'6 (0'0,58'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'6 2019-12-07T19:34:35.634+0000 7f8f05984700 10 osd.7 58 dequeue_op 0x558c5511fa20 finish 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4311 < min_flush_size 524288 2019-12-07T19:34:35.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3180 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579dea00 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4334 < min_flush_size 524288 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4334 < min_flush_size 524288 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40d6c3~10ee to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40d6c3 0x40d6c3~10ee to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3dd6c3 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6c3 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7b1 and padding block with 0x84f 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40e7b1 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.634+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000138042s (0.000000102s flush + 0.000137940s kv commit) 2019-12-07T19:34:35.634+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c575c5b00> 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5b00 kv_submitted 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c575c5b00 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c575c5b00 finishing 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c575c5b00 onodes 0x558c54ce3180,0x558c579dea00 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c575c5b00 done 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c575c5b00 [] 2019-12-07T19:34:35.634+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.634+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.634+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.634+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 58/41 ECSubWriteReply(tid=290, last_complete=58'6, committed=1, applied=1)) v2 -- 0x558c580b8e00 con 0x558c54c64000 2019-12-07T19:34:35.634+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 23 ==== osd_op(client.4540.0:441 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 ==== 258+0+0 (crc 0 0 0) 0x558c575a1340 con 0x558c54c9f000 2019-12-07T19:34:35.635+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511e000 prio 63 cost 0 latency 0.000017 epoch 58 osd_op(client.4540.0:441 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:441 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 24 ==== osd_op(client.4540.0:442 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 ==== 258+0+0 (crc 0 0 0) 0x558c575a1600 con 0x558c54c9f000 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:441 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) queued 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.635+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5740c6e0 prio 63 cost 0 latency 0.000009 epoch 58 osd_op(client.4540.0:442 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:441 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:442 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511e000 prio 63 cost 0 latency 0.000044 osd_op(client.4540.0:441 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 pg pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 48 -> 58 (as per caller) 2019-12-07T19:34:35.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 25 ==== osd_op(client.4540.0:443 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 ==== 258+0+0 (crc 0 0 0) 0x558c575a18c0 con 0x558c54c9f000 2019-12-07T19:34:35.635+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5740c420 prio 63 cost 0 latency 0.000008 epoch 58 osd_op(client.4540.0:443 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: osd_op(client.4540.0:441 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.635+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:443 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op: op osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.635+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:442 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) queued 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(53'38 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: e=[]:{} 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(53'38 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:35.635+0000 7f8f06986700 25 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(53'38 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=1 w=0)) 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx 0x558c58137800 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] ov 53'38 av 58'43 snapc 23=[23,21,20,1b] snapset e=[]:{} 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op append 0~0 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [write 507904~0] 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op write 507904~0 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] clean_regions modifiedclean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head snapset=e=[]:{} snapc=23=[23,21,20,1b] 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] got greedy write on clone_obc obc(1:3a5a3f5b:::smithi13140796-23:23 rwstate(write n=1 w=0)) 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] cloning v 53'38 to 1:3a5a3f5b:::smithi13140796-23:23 v 58'43 snaps=[23,21,20,1b] snapset=e=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head done, snapset=23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] finish_ctx 1:3a5a3f5b:::smithi13140796-23:head 0x558c58137800 op modify 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] set mtime to 2019-12-07T19:34:35.636323+0000 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] final snapset 23=[]:{23=[23,21,20,1b]} in 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] finish_ctx object 1:3a5a3f5b:::smithi13140796-23:head marks clean_regions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx alloc reply 0x558c54d2c280 result 0 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] calc_trim_to_aggressive limit = 57'42 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op order client.4540 tid 441 last was 286 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] new_repop rep_tid 49 on osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] new_repop: repgather(0x558c576b6fc0 0'0 rep_tid=49 committed?=0 r=0) 2019-12-07T19:34:35.635+0000 7f8f06986700 7 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] issue_repop rep_tid 49 o 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:head v=58'44 tt=0'0 tid=49 reqid=client.4540.0:441 client_op=osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:35.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 2019-12-07T19:34:35.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:23# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0x0000000000000023ffffffffffffffff'o' 2019-12-07T19:34:35.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:35.635+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:35.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:23 projected size 507904 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.635+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 494 ==== MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=291, reqid=client.4540.0:438, at_version=58'7, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2134+0+0 (crc 0 0 0) 0x558c5831d800 con 0x558c54c64000 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:35.635+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000031 epoch 58 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=291, reqid=client.4540.0:438, at_version=58'7, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:35.635+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=291, reqid=client.4540.0:438, at_version=58'7, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'44 tt=0'0 tid=49 reqid=client.4540.0:441 client_op=osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_state_to_reads: invalidating cache after this op 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'44 tt=0'0 tid=49 reqid=client.4540.0:441 client_op=osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=291, reqid=client.4540.0:438, at_version=58'7, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=291, reqid=client.4540.0:438, at_version=58'7, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) pg 0x558c54d98000 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=58'44 tt=0'0 tid=49 reqid=client.4540.0:441 client_op=osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 58 dequeue_op 0x558c5740cc60 prio 127 cost 0 latency 0.000077 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=291, reqid=client.4540.0:438, at_version=58'7, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.es4( v 58'6 (0'0,58'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:23, current size is 0 buffers are {} 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'6 (0'0,58'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=291, reqid=client.4540.0:438, at_version=58'7, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'6 (0'0,58'6] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'6], crt=21'4) [58'7 (58'6) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:438 2019-12-07T19:34:35.633837+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 0, new_object: 0] 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): new_size start 507904 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'7 (0'0,58'7] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'6 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'7 (58'6) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:438 2019-12-07T19:34:35.633837+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 0, new_object: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'7 (0'0,58'7] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'6 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 7 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:23 resetting hinfo to logical size 507904 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'7 (0'0,58'7] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'6 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.635+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5853ec00 seq 24 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): new_size start 507904 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) get_onode oid 4#1:733d7474:::smithi13140796-5:head#7 key 0x848000000000000001733d74't!smithi13140796-5!='0xfffffffffffffffe0000000000000007'o' 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.es4_head 0x558c57f10000) r -2 v.len 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.es4_head 4#1:733d7474:::smithi13140796-5:head# -> 4#1:733d7474:::smithi13140796-5:head#7 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.es4_head 4#1:733d7474:::smithi13140796-5:head# -> 4#1:733d7474:::smithi13140796-5:head#7 = 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:23={},1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'44 tt=0'0 tid=49 reqid=client.4540.0:441 client_op=osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.es4_head 4#1:733d7474:::smithi13140796-5:23# -> 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1274 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.onode(0x558c579dea00).flush flush done 2019-12-07T19:34:35.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x0 2019-12-07T19:34:35.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.es4_head 4#1:733d7474:::smithi13140796-5:23# -> 4#1:733d7474:::smithi13140796-5:head# 0x0~5000 -> 0x0~5000 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup src 0x0~2000: 0x0~2000 Blob(0x558c5809bb90 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=2)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup new Blob(0x558c58766fc0 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.blob(0x558c58766fc0) get_ref 0x0~2000 Blob(0x558c58766fc0 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.blob(0x558c58766fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:23,{} 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c58766fc0 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c550b2a80 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=2)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup new Blob(0x558c57396fc0 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=3)))) 2019-12-07T19:34:35.635+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.blob(0x558c57396fc0) get_ref 0x0~2000 Blob(0x558c57396fc0 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.blob(0x558c57396fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57396fc0 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup src 0x4000~1000: 0x0~1000 Blob(0x558c58766e00 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=2)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup new Blob(0x558c57397030 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.blob(0x558c57397030) get_ref 0x0~1000 Blob(0x558c57397030 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.blob(0x558c57397030) get_ref init 0x1000, 1000 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) dup dst 0x4000~1000: 0x0~1000 Blob(0x558c57397030 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=3)))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579de608) dirty_range mark inline shard dirty 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.es4_head 4#1:733d7474:::smithi13140796-5:23# -> 4#1:733d7474:::smithi13140796-5:head# = 0 2019-12-07T19:34:35.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys = 0 2019-12-07T19:34:35.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 58/28 ECSubWrite(tid=49, reqid=client.4540.0:441, at_version=58'44, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c575f5800 con 0x558c54c64000 2019-12-07T19:34:35.635+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000058.00000000000000000007' <- 0000000058.00000000000000000007 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._fastinfo' <- _fastinfo 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 send_incremental_map 57 -> 58 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853ec00 cost 5785 (1 ios * 4000 + 1785 bytes) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853ec00 onodes 0x558c54ce3180,0x558c579de500 shared_blobs 0x558c54df8150,0x558c57896310,0x558c57896540 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3288) update 4#1:733d7474:::smithi13140796-5:head#7 2019-12-07T19:34:35.635+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c327972f:::inc_osdmap.58:0# 0x0~0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:head#7 is 666 (579 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:35.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579de608) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:35.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d75e0 blob([0x2a6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c550b38f0 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579de608) update inline shard 81 bytes from 3 extents 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 666 (579 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:35.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c327972f:::inc_osdmap.58:0# 0x0~24d = 589 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 add_map_inc_bl 58 589 bytes 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a39 is 11 SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=3))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a38 is 11 SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=3))) 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(58..58 src has 1..58) v4 -- 0x558c58526480 con 0x558c54c19800 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3a is 11 SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=3))) 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853ec00 allocated 0x[] released 0x[] 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 57 -> 58 (shared) 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 58/28 ECSubWrite(tid=49, reqid=client.4540.0:441, at_version=58'44, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c575f3000 con 0x558c54c19800 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ec00 prepare 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853ec00 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ec00 io_done 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 send_incremental_map 57 -> 58 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(58..58 src has 1..58) v4 -- 0x558c579e6fc0 con 0x558c54c5bc00 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 57 -> 58 (shared) 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 58/28 ECSubWrite(tid=49, reqid=client.4540.0:441, at_version=58'44, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c5831b000 con 0x558c54c5bc00 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'7 (0'0,58'7] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'6 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'7 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 58/28 ECSubWrite(tid=49, reqid=client.4540.0:441, at_version=58'44, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c5831b800 con 0x558c54c5b400 2019-12-07T19:34:35.635+0000 7f8f0197c700 10 osd.7 58 dequeue_op 0x558c5740cc60 finish 2019-12-07T19:34:35.635+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 send_incremental_map 57 -> 58 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(58..58 src has 1..58) v4 -- 0x558c579e6d80 con 0x558c54c19c00 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 58 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 57 -> 58 (shared) 2019-12-07T19:34:35.635+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 58/28 ECSubWrite(tid=49, reqid=client.4540.0:441, at_version=58'44, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c5831c000 con 0x558c54c19c00 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.add_oid 1:3a5a3f5b:::smithi13140796-23:23 1b,20,21,23 2019-12-07T19:34:35.635+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.635+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:35.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:23 got.empty() 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.set_snaps 1:3a5a3f5b:::smithi13140796-23:23 1b,20,21,23 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_0000000000000021_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.635+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.635+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 trim 58'43 (53'38) clone 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 20 trim 58'44 (53'38) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:441 2019-12-07T19:34:35.636323+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 57'42 (0'0,57'42] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log log((0'0,57'42], crt=57'42) [58'43 (53'38) clone 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,58'44 (53'38) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:441 2019-12-07T19:34:35.636323+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'43 (0'0,58'43] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] add_log_entry 58'43 (53'38) clone 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] add_log_entry 58'44 (53'38) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:441 2019-12-07T19:34:35.636323+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log approx pg log length = 44 2019-12-07T19:34:35.635+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.635+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:35.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.635+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'43, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2201 < min_flush_size 524288 2019-12-07T19:34:35.635+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 303 ==== MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=113, reqid=client.4540.0:433, at_version=58'44, trim_to=0'0, roll_forward_to=57'41)) v2 ==== 174552+0+0 (unknown 0 0 0) 0x558c575bf000 con 0x558c54c5b400 2019-12-07T19:34:35.635+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000029 epoch 58 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=113, reqid=client.4540.0:433, at_version=58'44, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.635+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=113, reqid=client.4540.0:433, at_version=58'44, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3180 had 1 2019-12-07T19:34:35.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579de500 had 1 2019-12-07T19:34:35.635+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:35.635+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:35.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2224 < min_flush_size 524288 2019-12-07T19:34:35.635+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2224 < min_flush_size 524288 2019-12-07T19:34:35.635+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58748900 seq 61 2019-12-07T19:34:35.635+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40e7b1~8b0 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=113, reqid=client.4540.0:433, at_version=58'44, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:23# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0x0000000000000023ffffffffffffffff'o' 2019-12-07T19:34:35.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40e7b1 0x40e7b1~8b0 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=113, reqid=client.4540.0:433, at_version=58'44, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) pg 0x558c575ef000 2019-12-07T19:34:35.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3e0000 x_off 0x3de7b1 2019-12-07T19:34:35.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7b1 2019-12-07T19:34:35.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:35.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x61 and padding block with 0xf9f 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:23# 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1275 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.onode(0x558c58092000).flush flush done 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 0x0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:23# 0x0~1f000 -> 0x0~1f000 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=113, reqid=client.4540.0:433, at_version=58'44, trim_to=0'0, roll_forward_to=57'41)) v2 pg pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x0~2000: 0x0~2000 Blob(0x558c57ff52d0 blob([0x56e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10811 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff52d0 blob([0x56e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=113, reqid=client.4540.0:433, at_version=58'44, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff52d0 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c58b78af0 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78af0) get_ref 0x0~2000 Blob(0x558c58b78af0 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78af0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'43 (0'0,58'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'43], crt=57'41) [58'44 (58'43) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:433 2019-12-07T19:34:35.633226+0000 0 ObjectCleanRegions clean_offsets: [0~2048000,2736128~18446744073706815487], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x0~2000: 0x0~2000 Blob(0x558c58b78af0 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57ff53b0 blob([0x570000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10812 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff53b0 blob([0x570000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff53b0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c54e572d0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'44 (0'0,58'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'43 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'44 (58'43) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:433 2019-12-07T19:34:35.633226+0000 0 ObjectCleanRegions clean_offsets: [0~2048000,2736128~18446744073706815487], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c54e572d0) get_ref 0x0~2000 Blob(0x558c54e572d0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c54e572d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c54e572d0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'44 (0'0,58'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'43 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 44 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57ff5490 blob([0x572000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10813 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'44 (0'0,58'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'43 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5490 blob([0x572000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'44, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5490 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c58b78b60 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78b60) get_ref 0x0~2000 Blob(0x558c58b78b60 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78b60) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c58b78b60 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57ff5570 blob([0x574000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10814 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5570 blob([0x574000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5570 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c5794cbd0 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794cbd0) get_ref 0x0~2000 Blob(0x558c5794cbd0 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794cbd0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5794cbd0 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57ff58f0 blob([0x576000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10815 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff58f0 blob([0x576000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff58f0 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c5794cc40 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794cc40) get_ref 0x0~2000 Blob(0x558c5794cc40 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a2900 seq 53 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794cc40) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5794cc40 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57ff5810 blob([0x578000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10816 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5810 blob([0x578000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5810 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c580a5260 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c580a5260) get_ref 0x0~2000 Blob(0x558c580a5260 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c580a5260) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c580a5260 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x7d000~2a000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57ff5730 blob([0x57a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10817 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0x7d000~2a000 - have 0x7d000 (512000) bytes fadvise_flags 0x0 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5730 blob([0x57a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5730 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x7d000~2a000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c5794ccb0 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794ccb0) get_ref 0x0~2000 Blob(0x558c5794ccb0 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794ccb0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c5794ccb0 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57ff5650 blob([0x57c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10818 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5650 blob([0x57c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5650 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c5809af50 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5809af50) get_ref 0x0~2000 Blob(0x558c5809af50 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c586a2900 21 blobs 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5809af50) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c5809af50 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x10000~2000: 0x0~2000 Blob(0x558c57ff4e70 blob([0x57e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10819 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff4e70 blob([0x57e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff4e70 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c58b78a10 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78a10) get_ref 0x0~2000 Blob(0x558c58b78a10 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78a10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c58b78a10 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57ff4f50 blob([0x580000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10820 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff4f50 blob([0x580000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff4f50 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c5794cd20 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794cd20) get_ref 0x0~2000 Blob(0x558c5794cd20 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794cd20) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c5794cd20 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57ff5030 blob([0x582000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10821 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5030 blob([0x582000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5030 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c5794c4d0 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794c4d0) get_ref 0x0~2000 Blob(0x558c5794c4d0 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c5794c4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c5794c4d0 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57ff59d0 blob([0x584000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10822 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff59d0 blob([0x584000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff59d0 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c58b789a0 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b789a0) get_ref 0x0~2000 Blob(0x558c58b789a0 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b789a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c58b789a0 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x18000~2000: 0x0~2000 Blob(0x558c57ff5ab0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10823 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5ab0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5ab0 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c574d33b0 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c574d33b0) get_ref 0x0~2000 Blob(0x558c574d33b0 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c574d33b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c574d33b0 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57ff5b90 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10824 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5b90 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5b90 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c578f0620 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c578f0620) get_ref 0x0~2000 Blob(0x558c578f0620 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c578f0620) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c578f0620 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57ff5c70 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10825 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5c70 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5c70 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c58b78930 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78930) get_ref 0x0~2000 Blob(0x558c58b78930 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c58b78930) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c58b78930 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup src 0x1e000~1000: 0x0~1000 Blob(0x558c57ff5d50 blob([0x58c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10826 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c57ff5d50 blob([0x58c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c57ff5d50 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=1)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup new Blob(0x558c574d3420 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c574d3420) get_ref 0x0~1000 Blob(0x558c574d3420 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.blob(0x558c574d3420) get_ref init 0x1000, 1000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dup dst 0x1e000~1000: 0x0~1000 Blob(0x558c574d3420 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=2)))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) dirty_range mark inline shard dirty 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4608) dirty_range mark inline shard dirty 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:23# = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# snapset 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# snapset = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 1 keys 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 1 keys = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key (42 bytes) 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- SNA_1_000000000000001B_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- SNA_1_0000000000000021_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000058.00000000000000000043' <- 0000000058.00000000000000000043 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000058.00000000000000000044' <- 0000000058.00000000000000000044 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58748900 cost 8710 (1 ios * 4000 + 4710 bytes) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58748900 onodes 0x558c57bd4500,0x558c58092000 shared_blobs 0x558c57ff4e00,0x558c57ff4ee0,0x558c57ff4fc0,0x558c57ff5340,0x558c57ff5420,0x558c57ff5500,0x558c57ff55e0,0x558c57ff56c0,0x558c57ff57a0,0x558c57ff5880,0x558c57ff5960,0x558c57ff5a40,0x558c57ff5b20,0x558c57ff5c00,0x558c57ff5ce0,0x558c57ff5dc0 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4608) update 0#1:3a5a3f5b:::smithi13140796-23:23# 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4608) update inline shard 432 bytes from 16 extents 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:23# is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore.extentmap(0x558c58092108) update inline shard 432 bytes from 16 extents 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1019 (581 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a42 is 11 SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a43 is 11 SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a44 is 11 SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3b is 11 SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3c is 11 SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3d is 11 SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3e is 11 SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a41 is 11 SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a40 is 11 SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3f is 11 SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a45 is 11 SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a46 is 11 SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a47 is 11 SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a48 is 11 SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a49 is 11 SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=2))) 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4a is 11 SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=2))) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58748900 allocated 0x[] released 0x[] 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748900 prepare 2019-12-07T19:34:35.636+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58748900 2019-12-07T19:34:35.636+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748900 io_done 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op_applied version 58'44 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b6fc0 58'44 rep_tid=49 committed?=0 r=0) 2019-12-07T19:34:35.636+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511e000 finish 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2a000/1000,2a000,0 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0x36e000~9000/1000,2a000,0 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xb9f000~21000/1000,2a000,0 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0x36e000~9000,0xb9f000~21000] 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579d77a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.636+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:443 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) queued 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579d77a0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3030 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c579d77a0) get_ref 0x0~2000 Blob(0x558c579d77a0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3030 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c579d77a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x7d000~2000: 0x0~2000 Blob(0x558c579d77a0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3030 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e3048 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:442 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e63650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c551372d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e63650 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551372d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57e63650) get_ref 0x0~2000 Blob(0x558c57e63650 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551372d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57e63650) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5740c6e0 prio 63 cost 0 latency 0.001365 osd_op(client.4540.0:442 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 pg pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x7f000~2000: 0x0~2000 Blob(0x558c57e63650 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551372d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c551372e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: osd_op(client.4540.0:442 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579705b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579705b0 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb90 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c579705b0) get_ref 0x0~2000 Blob(0x558c579705b0 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb90 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c579705b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x81000~2000: 0x0~2000 Blob(0x558c579705b0 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb90 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57f3bba8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op: op osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c551371f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c551371f0 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c551371f0) get_ref 0x0~2000 Blob(0x558c551371f0 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c551371f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x83000~2000: 0x0~2000 Blob(0x558c551371f0 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad9e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137180 blob([0x376000~1000,0xb9f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2d20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c55137180) get_ref 0x0~2000 Blob(0x558c55137180 blob([0x376000~1000,0xb9f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2d20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c55137180) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x85000~2000: 0x0~2000 Blob(0x558c55137180 blob([0x376000~1000,0xb9f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2d20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2d38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=1 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(58'44 client.4540.0:441 dirty|data_digest s 507904 uv 44 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c551370a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c551370a0 blob([0xba0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3730 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(58'44 client.4540.0:441 dirty|data_digest s 507904 uv 44 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c551370a0) get_ref 0x0~2000 Blob(0x558c551370a0 blob([0xba0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3730 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c551370a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x87000~2000: 0x0~2000 Blob(0x558c551370a0 blob([0xba0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3730 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 25 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(58'44 client.4540.0:441 dirty|data_digest s 507904 uv 44 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3748 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=2 w=0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55137030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55137030 blob([0xba2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a9a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx 0x558c58122000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c55137030) get_ref 0x0~2000 Blob(0x558c55137030 blob([0xba2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a9a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c55137030) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x89000~2000: 0x0~2000 Blob(0x558c55137030 blob([0xba2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a9a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5809a9b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] ov 58'44 av 58'45 snapc 23=[23,21,20,1b] snapset 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3110 blob([0xba4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op rollback 21 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3110) get_ref 0x0~2000 Blob(0x558c550b3110 blob([0xba4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c550b3110) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8b000~2000: 0x0~2000 Blob(0x558c550b3110 blob([0xba4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b37b8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _rollback_to 1:3a5a3f5b:::smithi13140796-23:head snapid 21 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2930 blob([0xba6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:21 @21 snapset 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2930) get_ref 0x0~2000 Blob(0x558c574e2930 blob([0xba6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2930) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8d000~2000: 0x0~2000 Blob(0x558c574e2930 blob([0xba6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb20 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: found obc in cache: 0x558c585278c0 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57f3bb38 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c55136f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: 0x558c585278c0 1:3a5a3f5b:::smithi13140796-23:23 rwstate(write n=1 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:23(58'43 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c55136f50 blob([0xba8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970540 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c55136f50) get_ref 0x0~2000 Blob(0x558c55136f50 blob([0xba8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970540 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c55136f50) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:23 snapset 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8f000~2000: 0x0~2000 Blob(0x558c55136f50 blob([0xba8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970540 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57970558 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829ccb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829ccb0 blob([0xbaa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:23 clone_snaps [23,21,20,1b] contains 21 -- HIT 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c5829ccb0) get_ref 0x0~2000 Blob(0x558c5829ccb0 blob([0xbaa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c5829ccb0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x91000~2000: 0x0~2000 Blob(0x558c5829ccb0 blob([0xbaa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _rollback_to deleting smithi13140796-23 and rolling back to old snap 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2d90 blob([0xbac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3810 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2d90) get_ref 0x0~2000 Blob(0x558c574e2d90 blob([0xbac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3810 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2d90) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x93000~2000: 0x0~2000 Blob(0x558c574e2d90 blob([0xbac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3810 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _rollback_to clearing omap flag on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3bc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b2000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3bc00 blob([0xbae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2000 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bc00) get_ref 0x0~2000 Blob(0x558c57f3bc00 blob([0xbae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2000 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bc00) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head snapset=23=[]:{23=[23,21,20,1b]} snapc=23=[23,21,20,1b] 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x95000~2000: 0x0~2000 Blob(0x558c57f3bc00 blob([0xbae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2000 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b2018 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a2850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a2850 blob([0xbb0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970620 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head done, snapset=23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2850) get_ref 0x0~2000 Blob(0x558c580a2850 blob([0xbb0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970620 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2850) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x97000~2000: 0x0~2000 Blob(0x558c580a2850 blob([0xbb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970620 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57970638 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] finish_ctx 1:3a5a3f5b:::smithi13140796-23:head 0x558c58122000 op modify 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2e00 blob([0xbb2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2e00) get_ref 0x0~2000 Blob(0x558c574e2e00 blob([0xbb2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2e00) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] set mtime to 2019-12-07T19:34:35.636336+0000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x99000~2000: 0x0~2000 Blob(0x558c574e2e00 blob([0xbb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3bc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b38f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3bc70 blob([0xbb4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b38f0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bc70) get_ref 0x0~2000 Blob(0x558c57f3bc70 blob([0xbb4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b38f0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bc70) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x9b000~2000: 0x0~2000 Blob(0x558c57f3bc70 blob([0xbb4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b38f0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] final snapset 23=[]:{23=[23,21,20,1b]} in 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3908 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a28c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a28c0 blob([0xbb6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c580a28c0) get_ref 0x0~2000 Blob(0x558c580a28c0 blob([0xbb6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c580a28c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x9d000~2000: 0x0~2000 Blob(0x558c580a28c0 blob([0xbb6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57818c58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57970690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2e70 blob([0xbb8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970690 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2e70) get_ref 0x0~2000 Blob(0x558c574e2e70 blob([0xbb8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970690 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2e70) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] finish_ctx object 1:3a5a3f5b:::smithi13140796-23:head marks clean_regions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x9f000~2000: 0x0~2000 Blob(0x558c574e2e70 blob([0xbb8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970690 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579706a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3bce0 blob([0xbba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bce0) get_ref 0x0~2000 Blob(0x558c57f3bce0 blob([0xbba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3bce0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa1000~2000: 0x0~2000 Blob(0x558c57f3bce0 blob([0xbba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b3a58 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx alloc reply 0x558c575a2f00 result 0 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a2bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579b39d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a2bd0 blob([0xbbc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b39d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2bd0) get_ref 0x0~2000 Blob(0x558c580a2bd0 blob([0xbbc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b39d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa3000~2000: 0x0~2000 Blob(0x558c580a2bd0 blob([0xbbc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b39d0 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c579b39e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] calc_trim_to_aggressive limit = 57'42 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ad880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2ee0 blob([0xbbe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2ee0) get_ref 0x0~2000 Blob(0x558c574e2ee0 blob([0xbbe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op order client.4540 tid 442 last was 441 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa5000~2000: 0x0~2000 Blob(0x558c574e2ee0 blob([0xbbe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad880 sbid 0x0)) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ad898 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xa7000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark shard 0x76000 dirty 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x7d000~2a000 = 0 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] new_repop rep_tid 50 on osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] new_repop: repgather(0x558c576b70e0 0'0 rep_tid=50 committed?=0 r=0) 2019-12-07T19:34:35.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000058.00000000000000000044' <- 0000000058.00000000000000000044 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:35.636+0000 7f8f0297e700 7 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] issue_repop rep_tid 50 o 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a2900 cost 266220 (23 ios * 4000 + 174220 bytes) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a2900 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0x76000 is 483 bytes (was 78) from 25 extents 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 602 (600 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a2900 allocated 0x[36e000~9000,b9f000~21000] released 0x[] 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:head v=58'45 tt=0'0 tid=50 reqid=client.4540.0:442 client_op=osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 freelist allocate 0x36e000~9000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 freelist allocate 0xb9f000~21000 2019-12-07T19:34:35.636+0000 7f8f0317f700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2900 prepare 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c586a2900 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'45 tt=0'0 tid=50 reqid=client.4540.0:442 client_op=osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'45 tt=0'0 tid=50 reqid=client.4540.0:442 client_op=osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.636+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=58'45 tt=0'0 tid=50 reqid=client.4540.0:442 client_op=osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): new_size start 507904 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'45 tt=0'0 tid=50 reqid=client.4540.0:442 client_op=osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.636+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:35.637+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'44 (0'0,58'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'43 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'44 2019-12-07T19:34:35.637+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 58/28 ECSubWrite(tid=50, reqid=client.4540.0:442, at_version=58'45, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c5748f800 con 0x558c54c64000 2019-12-07T19:34:35.637+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:35.637+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.637+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 58/28 ECSubWrite(tid=50, reqid=client.4540.0:442, at_version=58'45, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c54d86800 con 0x558c54c19800 2019-12-07T19:34:35.637+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 58/28 ECSubWrite(tid=50, reqid=client.4540.0:442, at_version=58'45, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c57e7a000 con 0x558c54c5bc00 2019-12-07T19:34:35.637+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 58/28 ECSubWrite(tid=50, reqid=client.4540.0:442, at_version=58'45, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c57e7b800 con 0x558c54c5b400 2019-12-07T19:34:35.637+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 58/28 ECSubWrite(tid=50, reqid=client.4540.0:442, at_version=58'45, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c57e7b000 con 0x558c54c19c00 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 trim 58'45 (58'44) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:442 2019-12-07T19:34:35.636336+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'44 (0'0,58'44] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log log((0'0,58'44], crt=57'42) [58'45 (58'44) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:442 2019-12-07T19:34:35.636336+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0] 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'44 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] add_log_entry 58'45 (58'44) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:442 2019-12-07T19:34:35.636336+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0 2019-12-07T19:34:35.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2900 aio_wait 2019-12-07T19:34:35.637+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a2900 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x40f061 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'44 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log approx pg log length = 45 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2900 io_done 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'44 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.637+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'45, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57bd2f00 seq 62 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:head#2d key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0xfffffffffffffffe000000000000002d'o' 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001239149s (0.000000128s flush + 0.001239021s kv commit) 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:head#2d 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3999 < min_flush_size 524288 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5355 < min_flush_size 524288 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# -> 0#1:3a5a3f5b:::smithi13140796-23:head#2d = 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# -> 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1276 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.onode(0x558c57bd4500).flush flush cnt:1 2019-12-07T19:34:35.637+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 495 ==== MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=293, reqid=client.4540.0:439, at_version=58'8, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2352+0+0 (crc 0 0 0) 0x558c5831c000 con 0x558c54c64000 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd4500 had 1 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092000 had 1 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:35.637+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000025 epoch 58 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=293, reqid=client.4540.0:439, at_version=58'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7790 < min_flush_size 524288 2019-12-07T19:34:35.637+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=293, reqid=client.4540.0:439, at_version=58'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7813 < min_flush_size 524288 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7813 < min_flush_size 524288 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x40f061~1e85 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x40f061 0x40f061~1e85 to file(ino 16 size 0x410000 mtime 2019-12-07T19:34:34.624771+0000 allocated 410000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3e0000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _allocate len 0xee6 from 0 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _allocate len 0xee6 from 1 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd540000 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd540000~10000/10000,10000,acd540000 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x420000 to include allocated 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 72 (was clean) 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3f0000 x_off 0x3df061 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x61 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xee6 and padding block with 0x11a 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x420000 2019-12-07T19:34:35.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=293, reqid=client.4540.0:439, at_version=58'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=293, reqid=client.4540.0:439, at_version=58'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e58) pg 0x558c54d98000 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 58 dequeue_op 0x558c5740c9a0 prio 127 cost 0 latency 0.000123 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=293, reqid=client.4540.0:439, at_version=58'8, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.es4( v 58'7 (0'0,58'7] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'7 (0'0,58'7] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=293, reqid=client.4540.0:439, at_version=58'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'7 (0'0,58'7] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'7], crt=21'4) [58'8 (58'7) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:439 2019-12-07T19:34:35.633841+0000 0 ObjectCleanRegions clean_offsets: [0~81920,81920~18446744073709469695], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'7 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'8 (58'7) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:439 2019-12-07T19:34:35.633841+0000 0 ObjectCleanRegions clean_offsets: [0~81920,81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'7 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 8 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'7 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.637+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.637+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x410ee6 2019-12-07T19:34:35.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5876bb00 seq 25 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys 2019-12-07T19:34:35.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys = 0 2019-12-07T19:34:35.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.637+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000058.00000000000000000008' <- 0000000058.00000000000000000008 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._fastinfo' <- _fastinfo 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (72) on file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]), flushing log 2019-12-07T19:34:35.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5876bb00 cost 6003 (1 ios * 4000 + 2003 bytes) 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5876bb00 onodes 0x558c579de500 shared_blobs 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de608) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 72 len 0x4a crc 0x1c41e342) 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 666 (579 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5876bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x47000~1000 to file(ino 1 size 0x47000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x47000 0x47000~1000 to file(ino 1 size 0x47000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x48000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x47000 2019-12-07T19:34:35.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876bb00 prepare 2019-12-07T19:34:35.637+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5876bb00 2019-12-07T19:34:35.637+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876bb00 io_done 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'7 crt=21'4 lcod 58'6 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'8 2019-12-07T19:34:35.637+0000 7f8f05984700 10 osd.7 58 dequeue_op 0x558c5740c9a0 finish 2019-12-07T19:34:35.637+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x48000 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:35.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:35.637+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 496 ==== MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=292, reqid=client.4540.0:440, at_version=58'46, trim_to=0'0, roll_forward_to=57'44)) v2 ==== 91225+0+0 (crc 0 0 0) 0x558c57e7b000 con 0x558c54c64000 2019-12-07T19:34:35.637+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5740d080 prio 127 cost 0 latency 0.000030 epoch 58 MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=292, reqid=client.4540.0:440, at_version=58'46, trim_to=0'0, roll_forward_to=57'44)) v2 2019-12-07T19:34:35.637+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=292, reqid=client.4540.0:440, at_version=58'46, trim_to=0'0, roll_forward_to=57'44)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.onode(0x558c57bd4500).flush flush done 2019-12-07T19:34:35.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=292, reqid=client.4540.0:440, at_version=58'46, trim_to=0'0, roll_forward_to=57'44)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.637+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=292, reqid=client.4540.0:440, at_version=58'46, trim_to=0'0, roll_forward_to=57'44)) v2) prio 127 cost 0 e58) pg 0x558c5500a000 2019-12-07T19:34:35.637+0000 7f8f08189700 10 osd.7 58 dequeue_op 0x558c5740d080 prio 127 cost 0 latency 0.000079 MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=292, reqid=client.4540.0:440, at_version=58'46, trim_to=0'0, roll_forward_to=57'44)) v2 pg pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.637+0000 7f8f08189700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=292, reqid=client.4540.0:440, at_version=58'46, trim_to=0'0, roll_forward_to=57'44)) v2 2019-12-07T19:34:35.637+0000 7f8f08189700 20 snap_mapper.add_oid 1:9b8adabe:::smithi13140796-6:23 21,23 2019-12-07T19:34:35.637+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:35.637+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0 2019-12-07T19:34:35.637+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# -> 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~1f000 -> 0x0~1f000 2019-12-07T19:34:35.637+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.637+0000 7f8f08189700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:23 got.empty() 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x0~2000: 0x0~2000 Blob(0x558c58b78af0 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:23 21,23 2019-12-07T19:34:35.637+0000 7f8f08189700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5792ca80 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792ca80) get_ref 0x0~2000 Blob(0x558c5792ca80 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792ca80) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x0~2000: 0x0~2000 Blob(0x558c5792ca80 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 snap_mapper.add_oid set SNA_1_0000000000000021_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:35.637+0000 7f8f08189700 20 snap_mapper.add_oid set SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54e572d0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c578819d0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c578819d0) get_ref 0x0~2000 Blob(0x558c578819d0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c578819d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c578819d0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x4000~2000: 0x0~2000 Blob(0x558c58b78b60 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5792ca10 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792ca10) get_ref 0x0~2000 Blob(0x558c5792ca10 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792ca10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c5792ca10 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x6000~2000: 0x0~2000 Blob(0x558c5794cbd0 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c57881a40 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 57'44 (0'0,57'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,57'44], crt=57'44) [58'45 (57'44) clone 1:9b8adabe:::smithi13140796-6:23 by unknown.0.0:0 2019-12-07T19:34:32.653894+0000 0 snaps [23,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,58'46 (57'44) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:440 2019-12-07T19:34:35.636266+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881a40) get_ref 0x0~2000 Blob(0x558c57881a40 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881a40) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57881a40 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x8000~2000: 0x0~2000 Blob(0x558c5794cc40 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5792c9a0 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c9a0) get_ref 0x0~2000 Blob(0x558c5792c9a0 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f08189700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=57'44 crt=57'44 lcod 57'44 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'45 (57'44) clone 1:9b8adabe:::smithi13140796-6:23 by unknown.0.0:0 2019-12-07T19:34:32.653894+0000 0 snaps [23,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5792c9a0 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0xa000~2000: 0x0~2000 Blob(0x558c580a5260 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5898af50 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5898af50) get_ref 0x0~2000 Blob(0x558c5898af50 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5898af50) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5898af50 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=57'44 crt=57'44 lcod 57'44 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'46 (57'44) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:440 2019-12-07T19:34:35.636266+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0xc000~2000: 0x0~2000 Blob(0x558c5794ccb0 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c57881ab0 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881ab0) get_ref 0x0~2000 Blob(0x558c57881ab0 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f08189700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=57'44 crt=57'44 lcod 57'44 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 46 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57881ab0 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0xe000~2000: 0x0~2000 Blob(0x558c5809af50 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=57'44 crt=57'44 lcod 57'44 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5792c930 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c930) get_ref 0x0~2000 Blob(0x558c5792c930 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c930) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c5792c930 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x10000~2000: 0x0~2000 Blob(0x558c58b78a10 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'45, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c57a06e00 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57a06e00) get_ref 0x0~2000 Blob(0x558c57a06e00 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57a06e00) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c57a06e00 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x12000~2000: 0x0~2000 Blob(0x558c5794cd20 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c57881b20 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881b20) get_ref 0x0~2000 Blob(0x558c57881b20 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881b20) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57881b20 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x14000~2000: 0x0~2000 Blob(0x558c5794c4d0 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5792c8c0 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c8c0) get_ref 0x0~2000 Blob(0x558c5792c8c0 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c5792c8c0 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x16000~2000: 0x0~2000 Blob(0x558c58b789a0 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5898b030 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5898b030) get_ref 0x0~2000 Blob(0x558c5898b030 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5898b030) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c5898b030 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x18000~2000: 0x0~2000 Blob(0x558c574d33b0 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c57881b90 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c579d3800 seq 86 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881b90) get_ref 0x0~2000 Blob(0x558c57881b90 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881b90) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c57881b90 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c578f0620 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c5792c850 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c850) get_ref 0x0~2000 Blob(0x558c5792c850 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c5792c850) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) get_onode oid 4#1:9b8adabe:::smithi13140796-6:23# key 0x8480000000000000019b8adabe21736d'ithi13140796-6!='0x0000000000000023ffffffffffffffff'o' 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c5792c850 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c58b78930 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c578f1ce0 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) r -2 v.len 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1ce0) get_ref 0x0~2000 Blob(0x558c578f1ce0 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c578f1ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c578f1ce0 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:23# 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup src 0x1e000~1000: 0x0~1000 Blob(0x558c574d3420 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=2)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1277 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.onode(0x558c57a35400).flush flush done 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup new Blob(0x558c57881c00 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# 0x0 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881c00) get_ref 0x0~1000 Blob(0x558c57881c00 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.blob(0x558c57881c00) get_ref init 0x1000, 1000 2019-12-07T19:34:35.637+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:23# 0x0~a5000 -> 0x0~a5000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd4608) dup dst 0x1e000~1000: 0x0~1000 Blob(0x558c57881c00 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c575a3c88) dirty_range mark inline shard dirty 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# -> 0#1:3a5a3f5b:::smithi13140796-23:head# = 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x0~2000: 0x0~2000 Blob(0x558c57dcb7a0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c575f72d0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c575f72d0) get_ref 0x0~2000 Blob(0x558c575f72d0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c575f72d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x0~2000: 0x0~2000 Blob(0x558c575f72d0 blob([0x2e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2000~2000: 0x0~2000 Blob(0x558c57dcaaf0 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc1c0 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000058.00000000000000000045' <- 0000000058.00000000000000000045 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc1c0) get_ref 0x0~2000 Blob(0x558c579dc1c0 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c579dc1c0 blob([0x2e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4000~2000: 0x0~2000 Blob(0x558c577f3ea0 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57880a80 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd2f00 cost 5790 (1 ios * 4000 + 1790 bytes) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c57880a80) get_ref 0x0~2000 Blob(0x558c57880a80 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c57880a80) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd2f00 onodes 0x558c575a3b80,0x558c58092000 shared_blobs 0x558c57ff4e00,0x558c57ff4ee0,0x558c57ff4fc0,0x558c57ff5340,0x558c57ff5420,0x558c57ff5500,0x558c57ff55e0,0x558c57ff56c0,0x558c57ff57a0,0x558c57ff5880,0x558c57ff5960,0x558c57ff5a40,0x558c57ff5b20,0x558c57ff5c00,0x558c57ff5ce0,0x558c57ff5dc0 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57880a80 blob([0x2e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c575a3c88) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x6000~2000: 0x0~2000 Blob(0x558c54e42540 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c575a3c88) update inline shard 432 bytes from 16 extents 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c580d7420 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c580d7420) get_ref 0x0~2000 Blob(0x558c580d7420 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c580d7420) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1019 (581 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c580d7420 blob([0x2e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58092108) update 0#1:3a5a3f5b:::smithi13140796-23:head#2d 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x8000~2000: 0x0~2000 Blob(0x558c578813b0 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head#2d is 1019 (581 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc380 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a42 is 11 SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc380) get_ref 0x0~2000 Blob(0x558c579dc380 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc380) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a43 is 11 SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a44 is 11 SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c579dc380 blob([0x2eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3b is 11 SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3c is 11 SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0xa000~2000: 0x0~2000 Blob(0x558c57881490 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3d is 11 SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3e is 11 SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc3f0 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a41 is 11 SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc3f0) get_ref 0x0~2000 Blob(0x558c579dc3f0 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a40 is 11 SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3f is 11 SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c579dc3f0 blob([0x2ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a45 is 11 SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a46 is 11 SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57880bd0 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a47 is 11 SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc2a0 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a48 is 11 SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a49 is 11 SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=3))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc2a0) get_ref 0x0~2000 Blob(0x558c579dc2a0 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4a is 11 SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=3))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd2f00 allocated 0x[] released 0x[] 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c579dc2a0 blob([0x2ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0xe000~2000: 0x0~2000 Blob(0x558c578de310 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc540 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2f00 prepare 2019-12-07T19:34:35.637+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd2f00 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2f00 io_done 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc540) get_ref 0x0~2000 Blob(0x558c579dc540 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc540) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c579dc540 blob([0x2f1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x10000~2000: 0x0~2000 Blob(0x558c578de770 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=2)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc5b0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'44 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op_applied version 58'45 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc5b0) get_ref 0x0~2000 Blob(0x558c579dc5b0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=3)))) 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.blob(0x558c579dc5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.637+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c579dc5b0 blob([0x2f3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b70e0 58'45 rep_tid=50 committed?=0 r=0) 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5740c6e0 finish 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x12000~2000: 0x0~2000 Blob(0x558c578de5b0 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc620 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc620) get_ref 0x0~2000 Blob(0x558c579dc620 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc620) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c579dc620 blob([0x2f5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x14000~2000: 0x0~2000 Blob(0x558c578debd0 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dcd20 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcd20) get_ref 0x0~2000 Blob(0x558c579dcd20 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcd20) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c579dcd20 blob([0x2f7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x16000~2000: 0x0~2000 Blob(0x558c578dea10 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dcd90 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcd90) get_ref 0x0~2000 Blob(0x558c579dcd90 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcd90) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c579dcd90 blob([0x2f9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x18000~2000: 0x0~2000 Blob(0x558c578decb0 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c580a4c40 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c580a4c40) get_ref 0x0~2000 Blob(0x558c580a4c40 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c580a4c40) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c580a4c40 blob([0x2fb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 304 ==== MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=114, reqid=client.4540.0:434, at_version=58'45, trim_to=0'0, roll_forward_to=57'41)) v2 ==== 23000+0+0 (unknown 0 0 0) 0x558c57e7b800 con 0x558c54c5b400 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c578def50 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7490 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000026 epoch 58 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=114, reqid=client.4540.0:434, at_version=58'45, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7490) get_ref 0x0~2000 Blob(0x558c579d7490 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7490) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=114, reqid=client.4540.0:434, at_version=58'45, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c579d7490 blob([0x2fd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c578df030 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7420 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7420) get_ref 0x0~2000 Blob(0x558c579d7420 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7420) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c579d7420 blob([0x2ff000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c578df110 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d73b0 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d73b0) get_ref 0x0~2000 Blob(0x558c579d73b0 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d73b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c579d73b0 blob([0x301000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x20000~2000: 0x0~2000 Blob(0x558c578df1f0 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7340 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=114, reqid=client.4540.0:434, at_version=58'45, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7340) get_ref 0x0~2000 Blob(0x558c579d7340 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7340) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c579d7340 blob([0x303000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=114, reqid=client.4540.0:434, at_version=58'45, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) pg 0x558c575ef000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x22000~2000: 0x0~2000 Blob(0x558c54e1c070 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d72d0 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d72d0) get_ref 0x0~2000 Blob(0x558c579d72d0 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d72d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c579d72d0 blob([0x305000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x24000~2000: 0x0~2000 Blob(0x558c54e1c150 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 58 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000078 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=114, reqid=client.4540.0:434, at_version=58'45, trim_to=0'0, roll_forward_to=57'41)) v2 pg pg[1.bs1( v 58'44 (0'0,58'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7260 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7260) get_ref 0x0~2000 Blob(0x558c579d7260 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7260) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c579d7260 blob([0x307000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x26000~2000: 0x0~2000 Blob(0x558c54e1c8c0 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'44 (0'0,58'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=114, reqid=client.4540.0:434, at_version=58'45, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d71f0 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d71f0) get_ref 0x0~2000 Blob(0x558c579d71f0 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d71f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c579d71f0 blob([0x489000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x28000~2000: 0x0~2000 Blob(0x558c54e1c7e0 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc690 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc690) get_ref 0x0~2000 Blob(0x558c579dc690 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc690) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'44 (0'0,58'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'44], crt=57'41) [58'45 (58'44) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:434 2019-12-07T19:34:35.633769+0000 0 ObjectCleanRegions clean_offsets: [0~2736128,2818048~18446744073706733567], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c579dc690 blob([0x48b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000615 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c578f1880 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc700 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc700) get_ref 0x0~2000 Blob(0x558c579dc700 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc700) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c579dc700 blob([0x48d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'44 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'45 (58'44) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:434 2019-12-07T19:34:35.633769+0000 0 ObjectCleanRegions clean_offsets: [0~2736128,2818048~18446744073706733567], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c574e0d90 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc770 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'44 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 45 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc770) get_ref 0x0~2000 Blob(0x558c579dc770 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc770) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c579dc770 blob([0x48f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'44 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.638+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c574e0930 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc7e0 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'45, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc7e0) get_ref 0x0~2000 Blob(0x558c579dc7e0 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c579dc7e0 blob([0x491000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 72 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x30000~2000: 0x0~2000 Blob(0x558c578df260 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc850 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc850) get_ref 0x0~2000 Blob(0x558c579dc850 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc850) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c579dc850 blob([0x493000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.001010226s (0.000003416s flush + 0.001006810s kv commit) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x32000~2000: 0x0~2000 Blob(0x558c578ded90 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc8c0 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc8c0) get_ref 0x0~2000 Blob(0x558c579dc8c0 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1802 < min_flush_size 524288 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c579dc8c0 blob([0x495000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586e1200 seq 54 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x34000~2000: 0x0~2000 Blob(0x558c578dee70 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc930 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579de500 had 1 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc930) get_ref 0x0~2000 Blob(0x558c579dc930 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc930) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5025 < min_flush_size 524288 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c579dc930 blob([0x497000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x36000~2000: 0x0~2000 Blob(0x558c54e1c700 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc9a0 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc9a0) get_ref 0x0~2000 Blob(0x558c579dc9a0 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xa7000~5000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c579dc9a0 blob([0x499000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0xa7000~5000 - have 0xa7000 (684032) bytes fadvise_flags 0x0 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x38000~2000: 0x0~2000 Blob(0x558c54e1c620 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0xa7000~5000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dca10 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dca10) get_ref 0x0~2000 Blob(0x558c579dca10 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dca10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c586e1200 3 blobs 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c579dca10 blob([0x49b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c54e1c540 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dca80 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a3b80 had 1 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092000 had 1 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dca80) get_ref 0x0~2000 Blob(0x558c579dca80 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dca80) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5048 < min_flush_size 524288 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c579dca80 blob([0x49d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5048 < min_flush_size 524288 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c54e1c9a0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dcaf0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.638+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x5000/1000,5000,0 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853ec00,0x558c58748900,0x558c586a2900> 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcaf0) get_ref 0x0~2000 Blob(0x558c579dcaf0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbc0000~5000/1000,5000,0 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ec00 kv_submitted 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853ec00 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c579dcaf0 blob([0x49f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x410ee6~13b8 to file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbc0000~5000] 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c54e1ca80 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ec00 finishing 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58766700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58766690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.638+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x410ee6 0x410ee6~13b8 to file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853ec00 onodes 0x558c54ce3180,0x558c579de500 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc150 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853ec00 done 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5876bb00 kv_submitted 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58766700 blob([0xbc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766690 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc150) get_ref 0x0~2000 Blob(0x558c579dc150 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853ec00 [] 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc150) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3f0000 x_off 0x3e0ee6 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xee6 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.blob(0x558c58766700) get_ref 0x0~2000 Blob(0x558c58766700 blob([0xbc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766690 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c579dc150 blob([0x4a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x29e and padding block with 0xd62 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.blob(0x558c58766700) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748900 kv_submitted 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58748900 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x40000~2000: 0x0~2000 Blob(0x558c54e1cb60 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748900 finishing 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa7000~2000: 0x0~2000 Blob(0x558c58766700 blob([0xbc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58766690 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58748900 onodes 0x558c57bd4500,0x558c58092000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dc230 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58748900 done 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587666a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2f00 kv_submitted 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc230) get_ref 0x0~2000 Blob(0x558c579dc230 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58748900 [] 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dc230) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c579dc230 blob([0x4a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58766770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c587667e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2900 kv_submitted 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x42000~2000: 0x0~2000 Blob(0x558c54e1cc40 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58766770 blob([0xbc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587667e0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a2900 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57f51420 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.blob(0x558c58766770) get_ref 0x0~2000 Blob(0x558c58766770 blob([0xbc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587667e0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2900 finishing 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.blob(0x558c58766770) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a2900 onodes 0x558c54e44780 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c57f51420) get_ref 0x0~2000 Blob(0x558c57f51420 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c57f51420) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c551372e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c583cb7a0 space 0x558c551372e8 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa9000~2000: 0x0~2000 Blob(0x558c58766770 blob([0xbc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587667e0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c57f51420 blob([0x4a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c587667f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x44000~2000: 0x0~2000 Blob(0x558c54e1cd20 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dcb60 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcb60) get_ref 0x0~2000 Blob(0x558c579dcb60 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcb60) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c587668c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58766930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c579dcb60 blob([0x4a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c587668c0 blob([!~1000,0xbc4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766930 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x46000~2000: 0x0~2000 Blob(0x558c54e1ce00 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.blob(0x558c587668c0) get_ref 0x1000~1000 Blob(0x558c587668c0 blob([!~1000,0xbc4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766930 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.blob(0x558c587668c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dcbd0 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xab000~1000: 0x1000~1000 Blob(0x558c587668c0 blob([!~1000,0xbc4000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c58766930 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcbd0) get_ref 0x0~2000 Blob(0x558c579dcbd0 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58766948 in 0x558c53a59c00) _discard 0x1000~1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcbd0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c579dcbd0 blob([0x4a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xac000 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x48000~2000: 0x0~2000 Blob(0x558c54e1cee0 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark shard 0x76000 dirty 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dcc40 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xa7000~5000 = 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcc40) get_ref 0x0~2000 Blob(0x558c579dcc40 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dcc40) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c579dcc40 blob([0x4ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c54e1cfc0 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579dccb0 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dccb0) get_ref 0x0~2000 Blob(0x558c579dccb0 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579dccb0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c579dccb0 blob([0x4ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000058.00000000000000000045' <- 0000000058.00000000000000000045 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4c000~1000: 0x0~1000 Blob(0x558c54e1d0a0 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7180 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7180) get_ref 0x0~1000 Blob(0x558c579d7180 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=3)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7180) get_ref init 0x1000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e1200 cost 38650 (4 ios * 4000 + 22650 bytes) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e1200 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4c000~1000: 0x0~1000 Blob(0x558c579d7180 blob([0x4af000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=3)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4d000~2000: 0x0~2000 Blob(0x558c57818000 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc00 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10827 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57818000 blob([0x33d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc00 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update shard 0x76000 is 553 bytes (was 483) from 28 extents 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57818000 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc00 loaded (sbid 0x2a4b ref_map(0x33d000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7110 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc00 loaded (sbid 0x2a4b ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7110) get_ref 0x0~2000 Blob(0x558c579d7110 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc00 loaded (sbid 0x2a4b ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7110) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4d000~2000: 0x0~2000 Blob(0x558c579d7110 blob([0x33d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc00 loaded (sbid 0x2a4b ref_map(0x33d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 602 (600 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x4f000~2000: 0x0~2000 Blob(0x558c57f50cb0 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970000 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10828 2019-12-07T19:34:35.638+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e1200 allocated 0x[bc0000~5000] released 0x[] 2019-12-07T19:34:35.638+0000 7f8f07187700 10 freelist allocate 0xbc0000~5000 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57f50cb0 blob([0x33f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970000 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f07187700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57f50cb0 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970000 loaded (sbid 0x2a4c ref_map(0x33f000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d70a0 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970000 loaded (sbid 0x2a4c ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d70a0) get_ref 0x0~2000 Blob(0x558c579d70a0 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970000 loaded (sbid 0x2a4c ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d70a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1200 prepare 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x4f000~2000: 0x0~2000 Blob(0x558c579d70a0 blob([0x33f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970000 loaded (sbid 0x2a4c ref_map(0x33f000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c586e1200 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x51000~2000: 0x0~2000 Blob(0x558c57819490 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10829 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57819490 blob([0x341000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57819490 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 loaded (sbid 0x2a4d ref_map(0x341000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7030 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 loaded (sbid 0x2a4d ref_map(0x341000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7030) get_ref 0x0~2000 Blob(0x558c579d7030 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d20 loaded (sbid 0x2a4d ref_map(0x341000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c579d7030) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x51000~2000: 0x0~2000 Blob(0x558c579d7030 blob([0x341000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d20 loaded (sbid 0x2a4d ref_map(0x341000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x53000~2000: 0x0~2000 Blob(0x558c57db5490 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2cb0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10830 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57db5490 blob([0x343000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2cb0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57db5490 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2cb0 loaded (sbid 0x2a4e ref_map(0x343000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58766620 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2cb0 loaded (sbid 0x2a4e ref_map(0x343000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766620) get_ref 0x0~2000 Blob(0x558c58766620 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2cb0 loaded (sbid 0x2a4e ref_map(0x343000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766620) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x53000~2000: 0x0~2000 Blob(0x558c58766620 blob([0x343000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2cb0 loaded (sbid 0x2a4e ref_map(0x343000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x55000~2000: 0x0~2000 Blob(0x558c578f16c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10831 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578f16c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578f16c0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 loaded (sbid 0x2a4f ref_map(0x345000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c587665b0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 loaded (sbid 0x2a4f ref_map(0x345000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587665b0) get_ref 0x0~2000 Blob(0x558c587665b0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adc70 loaded (sbid 0x2a4f ref_map(0x345000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587665b0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x55000~2000: 0x0~2000 Blob(0x558c587665b0 blob([0x345000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adc70 loaded (sbid 0x2a4f ref_map(0x345000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x57000~2000: 0x0~2000 Blob(0x558c57db5420 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970070 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10832 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57db5420 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970070 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57db5420 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970070 loaded (sbid 0x2a50 ref_map(0x347000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58766540 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970070 loaded (sbid 0x2a50 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766540) get_ref 0x0~2000 Blob(0x558c58766540 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970070 loaded (sbid 0x2a50 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766540) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x57000~2000: 0x0~2000 Blob(0x558c58766540 blob([0x347000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970070 loaded (sbid 0x2a50 ref_map(0x347000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x59000~2000: 0x0~2000 Blob(0x558c578181c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10833 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578181c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578181c0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 loaded (sbid 0x2a51 ref_map(0x349000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c587664d0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 loaded (sbid 0x2a51 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587664d0) get_ref 0x0~2000 Blob(0x558c587664d0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e00 loaded (sbid 0x2a51 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587664d0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x59000~2000: 0x0~2000 Blob(0x558c587664d0 blob([0x349000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e00 loaded (sbid 0x2a51 ref_map(0x349000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x5b000~2000: 0x0~2000 Blob(0x558c5794e000 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d90 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10834 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e000 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d90 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'44 crt=57'41 lcod 58'43 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'45 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e000 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d90 loaded (sbid 0x2a52 ref_map(0x34b000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f07187700 10 osd.7 58 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58766460 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d90 loaded (sbid 0x2a52 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766460) get_ref 0x0~2000 Blob(0x558c58766460 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2d90 loaded (sbid 0x2a52 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766460) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x5b000~2000: 0x0~2000 Blob(0x558c58766460 blob([0x34b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2d90 loaded (sbid 0x2a52 ref_map(0x34b000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x5d000~2000: 0x0~2000 Blob(0x558c578180e0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2150 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10835 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578180e0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2150 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 58/41 ECSubWriteReply(tid=113, last_complete=58'44, committed=1, applied=1)) v2 -- 0x558c55251e00 con 0x558c54c5b400 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578180e0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2150 loaded (sbid 0x2a53 ref_map(0x34d000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c587663f0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2150 loaded (sbid 0x2a53 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587663f0) get_ref 0x0~2000 Blob(0x558c587663f0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2150 loaded (sbid 0x2a53 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587663f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x5d000~2000: 0x0~2000 Blob(0x558c587663f0 blob([0x34d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2150 loaded (sbid 0x2a53 ref_map(0x34d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x5f000~2000: 0x0~2000 Blob(0x558c5794e070 blob([0x34f000~1000,0x377000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579700e0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10836 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e070 blob([0x34f000~1000,0x377000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579700e0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e070 blob([0x34f000~1000,0x377000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579700e0 loaded (sbid 0x2a54 ref_map(0x34f000~1000=1,0x377000~1000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58766380 blob([0x34f000~1000,0x377000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579700e0 loaded (sbid 0x2a54 ref_map(0x34f000~1000=2,0x377000~1000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766380) get_ref 0x0~2000 Blob(0x558c58766380 blob([0x34f000~1000,0x377000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579700e0 loaded (sbid 0x2a54 ref_map(0x34f000~1000=2,0x377000~1000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766380) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x5f000~2000: 0x0~2000 Blob(0x558c58766380 blob([0x34f000~1000,0x377000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579700e0 loaded (sbid 0x2a54 ref_map(0x34f000~1000=2,0x377000~1000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x61000~2000: 0x0~2000 Blob(0x558c57819420 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10837 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2d38 in 0x558c53a59c00) _finish_write discard buffer(0x558c583cb710 space 0x558c574e2d38 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57819420 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3048 in 0x558c53a59c00) _finish_write discard buffer(0x558c585341b0 space 0x558c574e3048 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57819420 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 loaded (sbid 0x2a55 ref_map(0x378000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad898 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535c20 space 0x558c576ad898 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ad9e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c58b84ea0 space 0x558c576ad9e8 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58766310 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 loaded (sbid 0x2a55 ref_map(0x378000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766310) get_ref 0x0~2000 Blob(0x558c58766310 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2ee0 loaded (sbid 0x2a55 ref_map(0x378000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58766310) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57818c58 in 0x558c53a59c00) _finish_write discard buffer(0x558c585359e0 space 0x558c57818c58 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970558 in 0x558c53a59c00) _finish_write discard buffer(0x558c585355f0 space 0x558c57970558 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x61000~2000: 0x0~2000 Blob(0x558c58766310 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2ee0 loaded (sbid 0x2a55 ref_map(0x378000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57970638 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535830 space 0x558c57970638 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579706a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535a70 space 0x558c579706a8 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x63000~2000: 0x0~2000 Blob(0x558c5794e0e0 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e70 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10838 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b2018 in 0x558c53a59c00) _finish_write discard buffer(0x558c585357a0 space 0x558c579b2018 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3748 in 0x558c53a59c00) _finish_write discard buffer(0x558c58534120 space 0x558c579b3748 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e0e0 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e70 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b37b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c585354d0 space 0x558c579b37b8 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3828 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535710 space 0x558c579b3828 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e0e0 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e70 loaded (sbid 0x2a56 ref_map(0x37a000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3898 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535680 space 0x558c579b3898 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3908 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535950 space 0x558c579b3908 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c587662a0 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e70 loaded (sbid 0x2a56 ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3978 in 0x558c53a59c00) _finish_write discard buffer(0x558c585358c0 space 0x558c579b3978 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b39e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535b90 space 0x558c579b39e8 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587662a0) get_ref 0x0~2000 Blob(0x558c587662a0 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2e70 loaded (sbid 0x2a56 ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c587662a0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579b3a58 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535b00 space 0x558c579b3a58 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3bb38 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535560 space 0x558c57f3bb38 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x63000~2000: 0x0~2000 Blob(0x558c587662a0 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2e70 loaded (sbid 0x2a56 ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3bba8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45950 space 0x558c57f3bba8 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x65000~2000: 0x0~2000 Blob(0x558c57db58f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67a40 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a9b8 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535440 space 0x558c5809a9b8 0x0~2000 writing nocache) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10839 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57db58f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67a40 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a2900 done 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e1200 aio_wait 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a2900 [] 2019-12-07T19:34:35.638+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57db58f0 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67a40 loaded (sbid 0x2a57 ref_map(0x37c000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767c00 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67a40 loaded (sbid 0x2a57 ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767c00) get_ref 0x0~2000 Blob(0x558c58767c00 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67a40 loaded (sbid 0x2a57 ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767c00) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x65000~2000: 0x0~2000 Blob(0x558c58767c00 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67a40 loaded (sbid 0x2a57 ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x67000~2000: 0x0~2000 Blob(0x558c5794e150 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67960 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10840 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e150 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67960 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e150 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67960 loaded (sbid 0x2a58 ref_map(0x37e000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767c70 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67960 loaded (sbid 0x2a58 ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767c70) get_ref 0x0~2000 Blob(0x558c58767c70 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67960 loaded (sbid 0x2a58 ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767c70) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x67000~2000: 0x0~2000 Blob(0x558c58767c70 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67960 loaded (sbid 0x2a58 ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 287 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b8000 con 0x558c54c5bc00 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x69000~2000: 0x0~2000 Blob(0x558c57db5880 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f678f0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10841 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57db5880 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f678f0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c5740c000 prio 196 cost 0 latency 0.000012 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57db5880 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f678f0 loaded (sbid 0x2a59 ref_map(0x380000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767ce0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f678f0 loaded (sbid 0x2a59 ref_map(0x380000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767ce0) get_ref 0x0~2000 Blob(0x558c58767ce0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f678f0 loaded (sbid 0x2a59 ref_map(0x380000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x69000~2000: 0x0~2000 Blob(0x558c58767ce0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f678f0 loaded (sbid 0x2a59 ref_map(0x380000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x6b000~2000: 0x0~2000 Blob(0x558c5794e1c0 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314000 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10842 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e1c0 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314000 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e1c0 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314000 loaded (sbid 0x2a5a ref_map(0x382000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767d50 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314000 loaded (sbid 0x2a5a ref_map(0x382000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767d50) get_ref 0x0~2000 Blob(0x558c58767d50 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314000 loaded (sbid 0x2a5a ref_map(0x382000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767d50) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x6b000~2000: 0x0~2000 Blob(0x558c58767d50 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314000 loaded (sbid 0x2a5a ref_map(0x382000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x6d000~2000: 0x0~2000 Blob(0x558c57db5810 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809ae70 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10843 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57db5810 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809ae70 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57db5810 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809ae70 loaded (sbid 0x2a5b ref_map(0x384000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767dc0 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809ae70 loaded (sbid 0x2a5b ref_map(0x384000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767dc0) get_ref 0x0~2000 Blob(0x558c58767dc0 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809ae70 loaded (sbid 0x2a5b ref_map(0x384000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x6d000~2000: 0x0~2000 Blob(0x558c58767dc0 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809ae70 loaded (sbid 0x2a5b ref_map(0x384000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x6f000~2000: 0x0~2000 Blob(0x558c5794e230 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67880 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10844 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e230 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67880 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e230 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67880 loaded (sbid 0x2a5c ref_map(0x386000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767e30 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67880 loaded (sbid 0x2a5c ref_map(0x386000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767e30) get_ref 0x0~2000 Blob(0x558c58767e30 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f67880 loaded (sbid 0x2a5c ref_map(0x386000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767e30) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x6f000~2000: 0x0~2000 Blob(0x558c58767e30 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f67880 loaded (sbid 0x2a5c ref_map(0x386000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x71000~2000: 0x0~2000 Blob(0x558c57db57a0 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e3f0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10845 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c57db57a0 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e3f0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c57db57a0 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e3f0 loaded (sbid 0x2a5d ref_map(0x388000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767ea0 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e3f0 loaded (sbid 0x2a5d ref_map(0x388000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767ea0) get_ref 0x0~2000 Blob(0x558c58767ea0 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e3f0 loaded (sbid 0x2a5d ref_map(0x388000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x71000~2000: 0x0~2000 Blob(0x558c58767ea0 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e3f0 loaded (sbid 0x2a5d ref_map(0x388000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x73000~2000: 0x0~2000 Blob(0x558c5794e460 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e4d0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10846 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e460 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e4d0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e460 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e4d0 loaded (sbid 0x2a5e ref_map(0x38a000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767f10 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e4d0 loaded (sbid 0x2a5e ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767f10) get_ref 0x0~2000 Blob(0x558c58767f10 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e4d0 loaded (sbid 0x2a5e ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767f10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x73000~2000: 0x0~2000 Blob(0x558c58767f10 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794e4d0 loaded (sbid 0x2a5e ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x75000~1000: 0x1000~1000 Blob(0x558c5794e540 blob([!~1000,0x38c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5794e5b0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10847 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794e540 blob([!~1000,0x38c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5794e5b0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794e540 blob([!~1000,0x38c000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5794e5b0 loaded (sbid 0x2a5f ref_map(0x38c000~1000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c58767f80 blob([!~1000,0x38c000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e5b0 loaded (sbid 0x2a5f ref_map(0x38c000~1000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767f80) get_ref 0x1000~1000 Blob(0x558c58767f80 blob([!~1000,0x38c000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794e5b0 loaded (sbid 0x2a5f ref_map(0x38c000~1000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c58767f80) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x75000~1000: 0x1000~1000 Blob(0x558c58767f80 blob([!~1000,0x38c000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c5794e5b0 loaded (sbid 0x2a5f ref_map(0x38c000~1000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x76000~2000: 0x0~2000 Blob(0x558c583eab60 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970f50 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10848 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c583eab60 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970f50 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c583eab60 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970f50 loaded (sbid 0x2a60 ref_map(0x38d000~2000=1)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c580f6310 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970f50 loaded (sbid 0x2a60 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c580f6310) get_ref 0x0~2000 Blob(0x558c580f6310 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970f50 loaded (sbid 0x2a60 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.blob(0x558c580f6310) get_ref init 0x2000, 1000 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x76000~2000: 0x0~2000 Blob(0x558c580f6310 blob([0x38d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970f50 loaded (sbid 0x2a60 ref_map(0x38d000~2000=2)))) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x78000~2000: 0x0~2000 Blob(0x558c578f0af0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f71f0 sbid 0x0)) 2019-12-07T19:34:35.638+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10849 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.638+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1200 aio_wait 2019-12-07T19:34:35.638+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e1200 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.638+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1200 io_done 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:443 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8) prio 63 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5740c420 prio 63 cost 0 latency 0.003672 osd_op(client.4540.0:443 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 pg pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.638+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 58/41 ECSubWriteReply(tid=291, last_complete=58'7, committed=1, applied=1)) v2 -- 0x558c57951c00 con 0x558c54c64000 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: osd_op(client.4540.0:443 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.638+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 305 ==== MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=115, reqid=client.4540.0:435, at_version=58'46, trim_to=0'0, roll_forward_to=57'41)) v2 ==== 2488+0+0 (unknown 0 0 0) 0x558c57e7a000 con 0x558c54c5b400 2019-12-07T19:34:35.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.638+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c57f3fb80 prio 127 cost 0 latency 0.000025 epoch 58 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=115, reqid=client.4540.0:435, at_version=58'46, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op: op osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.638+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=115, reqid=client.4540.0:435, at_version=58'46, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=2 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(58'45 client.4540.0:442 dirty|data_digest s 507904 uv 45 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(58'45 client.4540.0:442 dirty|data_digest s 507904 uv 45 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:35.638+0000 7f8f0297e700 25 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(58'45 client.4540.0:442 dirty|data_digest s 507904 uv 45 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:35.638+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=3 w=0)) 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx 0x558c57ed3c00 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] ov 58'45 av 58'46 snapc 23=[23,21,20,1b] snapset 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op append 0~0 2019-12-07T19:34:35.638+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [write 507904~0] 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] do_osd_op write 507904~0 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] clean_regions modifiedclean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head snapset=23=[]:{23=[23,21,20,1b]} snapc=23=[23,21,20,1b] 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] make_writeable 1:3a5a3f5b:::smithi13140796-23:head done, snapset=23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] finish_ctx 1:3a5a3f5b:::smithi13140796-23:head 0x558c57ed3c00 op modify 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] set mtime to 2019-12-07T19:34:35.636340+0000 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] final snapset 23=[]:{23=[23,21,20,1b]} in 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] finish_ctx object 1:3a5a3f5b:::smithi13140796-23:head marks clean_regions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] execute_ctx alloc reply 0x558c575a3900 result 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] calc_trim_to_aggressive limit = 57'42 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op order client.4540 tid 443 last was 442 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] new_repop rep_tid 51 on osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] new_repop: repgather(0x558c576b7200 0'0 rep_tid=51 committed?=0 r=0) 2019-12-07T19:34:35.639+0000 7f8f0297e700 7 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] issue_repop rep_tid 51 o 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:head v=58'46 tt=0'0 tid=51 reqid=client.4540.0:443 client_op=osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'46 tt=0'0 tid=51 reqid=client.4540.0:443 client_op=osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'46 tt=0'0 tid=51 reqid=client.4540.0:443 client_op=osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=58'46 tt=0'0 tid=51 reqid=client.4540.0:443 client_op=osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): new_size start 507904 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'46 tt=0'0 tid=51 reqid=client.4540.0:443 client_op=osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:35.639+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 58/28 ECSubWrite(tid=51, reqid=client.4540.0:443, at_version=58'46, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c57e7c000 con 0x558c54c64000 2019-12-07T19:34:35.639+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 58/28 ECSubWrite(tid=51, reqid=client.4540.0:443, at_version=58'46, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c57e7d800 con 0x558c54c19800 2019-12-07T19:34:35.639+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 58/28 ECSubWrite(tid=51, reqid=client.4540.0:443, at_version=58'46, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c57e7c800 con 0x558c54c5bc00 2019-12-07T19:34:35.639+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 58/28 ECSubWrite(tid=51, reqid=client.4540.0:443, at_version=58'46, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c57e7d000 con 0x558c54c5b400 2019-12-07T19:34:35.639+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 58/28 ECSubWrite(tid=51, reqid=client.4540.0:443, at_version=58'46, trim_to=0'0, roll_forward_to=57'42)) v2 -- 0x558c575be000 con 0x558c54c19c00 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 trim 58'46 (58'45) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:443 2019-12-07T19:34:35.636340+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'45 (0'0,58'45] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log log((0'0,58'45], crt=57'42) [58'46 (58'45) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:443 2019-12-07T19:34:35.636340+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'45 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] add_log_entry 58'46 (58'45) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:443 2019-12-07T19:34:35.636340+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'45 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log approx pg log length = 46 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'45 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.639+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'46, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57bd3200 seq 63 2019-12-07T19:34:35.639+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000058.00000000000000000046' <- 0000000058.00000000000000000046 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd3200 cost 6007 (1 ios * 4000 + 2007 bytes) 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd3200 onodes 0x558c575a3b80 shared_blobs 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore.extentmap(0x558c575a3c88) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1019 (581 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd3200 allocated 0x[] released 0x[] 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3200 prepare 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd3200 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3200 io_done 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 lua=58'45 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] op_applied version 58'46 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b7200 58'46 rep_tid=51 committed?=0 r=0) 2019-12-07T19:34:35.639+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5740c420 finish 2019-12-07T19:34:35.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.638+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c578f0af0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f71f0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c578f0af0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f71f0 loaded (sbid 0x2a61 ref_map(0x38f000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c580f61c0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f71f0 loaded (sbid 0x2a61 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c580f61c0) get_ref 0x0~2000 Blob(0x558c580f61c0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f71f0 loaded (sbid 0x2a61 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c580f61c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x78000~2000: 0x0~2000 Blob(0x558c580f61c0 blob([0x38f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f71f0 loaded (sbid 0x2a61 ref_map(0x38f000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x7a000~2000: 0x0~2000 Blob(0x558c574fa4d0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10850 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c574fa4d0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c574fa4d0 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 loaded (sbid 0x2a62 ref_map(0x391000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c580f60e0 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 loaded (sbid 0x2a62 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c580f60e0) get_ref 0x0~2000 Blob(0x558c580f60e0 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c578df650 loaded (sbid 0x2a62 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c580f60e0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x7a000~2000: 0x0~2000 Blob(0x558c580f60e0 blob([0x391000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578df650 loaded (sbid 0x2a62 ref_map(0x391000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x7c000~2000: 0x0~2000 Blob(0x558c5794d030 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7110 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10851 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5794d030 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7110 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5794d030 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7110 loaded (sbid 0x2a63 ref_map(0x393000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7810 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7110 loaded (sbid 0x2a63 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c579d7810) get_ref 0x0~2000 Blob(0x558c579d7810 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7110 loaded (sbid 0x2a63 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c579d7810) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x7c000~2000: 0x0~2000 Blob(0x558c579d7810 blob([0x393000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7110 loaded (sbid 0x2a63 ref_map(0x393000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x7e000~2000: 0x0~2000 Blob(0x558c575f7340 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f73b0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10852 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f7340 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f73b0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f7340 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f73b0 loaded (sbid 0x2a64 ref_map(0x395000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5829c700 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f73b0 loaded (sbid 0x2a64 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5829c700) get_ref 0x0~2000 Blob(0x558c5829c700 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f73b0 loaded (sbid 0x2a64 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5829c700) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x7e000~2000: 0x0~2000 Blob(0x558c5829c700 blob([0x395000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f73b0 loaded (sbid 0x2a64 ref_map(0x395000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x80000~2000: 0x0~2000 Blob(0x558c575f7420 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7490 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10853 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f7420 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7490 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f7420 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7490 loaded (sbid 0x2a65 ref_map(0x397000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57e63ab0 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7490 loaded (sbid 0x2a65 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57e63ab0) get_ref 0x0~2000 Blob(0x558c57e63ab0 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7490 loaded (sbid 0x2a65 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57e63ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x80000~2000: 0x0~2000 Blob(0x558c57e63ab0 blob([0x397000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7490 loaded (sbid 0x2a65 ref_map(0x397000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x82000~2000: 0x0~2000 Blob(0x558c575f7500 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7570 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10854 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f7500 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7570 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f7500 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7570 loaded (sbid 0x2a66 ref_map(0x399000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c573961c0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7570 loaded (sbid 0x2a66 ref_map(0x399000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c573961c0) get_ref 0x0~2000 Blob(0x558c573961c0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7570 loaded (sbid 0x2a66 ref_map(0x399000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c573961c0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x82000~2000: 0x0~2000 Blob(0x558c573961c0 blob([0x399000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7570 loaded (sbid 0x2a66 ref_map(0x399000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x84000~2000: 0x0~2000 Blob(0x558c575f75e0 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7650 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10855 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f75e0 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7650 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f75e0 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7650 loaded (sbid 0x2a67 ref_map(0x39b000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57880e70 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7650 loaded (sbid 0x2a67 ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880e70) get_ref 0x0~2000 Blob(0x558c57880e70 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7650 loaded (sbid 0x2a67 ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880e70) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x84000~2000: 0x0~2000 Blob(0x558c57880e70 blob([0x39b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7650 loaded (sbid 0x2a67 ref_map(0x39b000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x86000~2000: 0x0~2000 Blob(0x558c575f76c0 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7730 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10856 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f76c0 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7730 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f76c0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7730 loaded (sbid 0x2a68 ref_map(0x39d000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57881ea0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7730 loaded (sbid 0x2a68 ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57881ea0) get_ref 0x0~2000 Blob(0x558c57881ea0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7730 loaded (sbid 0x2a68 ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57881ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x86000~2000: 0x0~2000 Blob(0x558c57881ea0 blob([0x39d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7730 loaded (sbid 0x2a68 ref_map(0x39d000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x88000~2000: 0x0~2000 Blob(0x558c575f77a0 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7810 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10857 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f77a0 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7810 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f77a0 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7810 loaded (sbid 0x2a69 ref_map(0x39f000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c579d7a40 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7810 loaded (sbid 0x2a69 ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c579d7a40) get_ref 0x0~2000 Blob(0x558c579d7a40 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7810 loaded (sbid 0x2a69 ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c579d7a40) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x88000~2000: 0x0~2000 Blob(0x558c579d7a40 blob([0x39f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7810 loaded (sbid 0x2a69 ref_map(0x39f000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x8a000~2000: 0x0~2000 Blob(0x558c575f7880 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f78f0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10858 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f7880 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f78f0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=115, reqid=client.4540.0:435, at_version=58'46, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f7880 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f78f0 loaded (sbid 0x2a6a ref_map(0x3a1000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5792cbd0 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f78f0 loaded (sbid 0x2a6a ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=115, reqid=client.4540.0:435, at_version=58'46, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e58) pg 0x558c575ef000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5792cbd0) get_ref 0x0~2000 Blob(0x558c5792cbd0 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f78f0 loaded (sbid 0x2a6a ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5792cbd0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x8a000~2000: 0x0~2000 Blob(0x558c5792cbd0 blob([0x3a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f78f0 loaded (sbid 0x2a6a ref_map(0x3a1000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x8c000~2000: 0x0~2000 Blob(0x558c575f7960 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f79d0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10859 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f7960 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f79d0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f7960 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f79d0 loaded (sbid 0x2a6b ref_map(0x3a3000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5829c3f0 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f79d0 loaded (sbid 0x2a6b ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 58 dequeue_op 0x558c57f3fb80 prio 127 cost 0 latency 0.000902 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=115, reqid=client.4540.0:435, at_version=58'46, trim_to=0'0, roll_forward_to=57'41)) v2 pg pg[1.bs1( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 58'44 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5829c3f0) get_ref 0x0~2000 Blob(0x558c5829c3f0 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f79d0 loaded (sbid 0x2a6b ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5829c3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x8c000~2000: 0x0~2000 Blob(0x558c5829c3f0 blob([0x3a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f79d0 loaded (sbid 0x2a6b ref_map(0x3a3000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x8e000~2000: 0x0~2000 Blob(0x558c575f7a40 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7ab0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10860 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f7a40 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7ab0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 58'44 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=115, reqid=client.4540.0:435, at_version=58'46, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f7a40 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7ab0 loaded (sbid 0x2a6c ref_map(0x3a5000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57880a10 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ab0 loaded (sbid 0x2a6c ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880a10) get_ref 0x0~2000 Blob(0x558c57880a10 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7ab0 loaded (sbid 0x2a6c ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880a10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x8e000~2000: 0x0~2000 Blob(0x558c57880a10 blob([0x3a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7ab0 loaded (sbid 0x2a6c ref_map(0x3a5000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x90000~2000: 0x0~2000 Blob(0x558c575f7b20 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7b90 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10861 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c575f7b20 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7b90 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'45 (0'0,58'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 lcod 58'44 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'45], crt=57'41) [58'46 (58'45) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:435 2019-12-07T19:34:35.633784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c575f7b20 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7b90 loaded (sbid 0x2a6d ref_map(0x3a7000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57819e30 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7b90 loaded (sbid 0x2a6d ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57819e30) get_ref 0x0~2000 Blob(0x558c57819e30 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575f7b90 loaded (sbid 0x2a6d ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57819e30) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x90000~2000: 0x0~2000 Blob(0x558c57819e30 blob([0x3a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575f7b90 loaded (sbid 0x2a6d ref_map(0x3a7000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x92000~2000: 0x0~2000 Blob(0x558c5895ae00 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895ae70 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10862 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895ae00 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895ae70 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'45 crt=57'41 lcod 58'44 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 58'46 (58'45) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:435 2019-12-07T19:34:35.633784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895ae00 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895ae70 loaded (sbid 0x2a6e ref_map(0x3a9000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5794c930 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895ae70 loaded (sbid 0x2a6e ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'45 crt=57'41 lcod 58'44 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 46 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5794c930) get_ref 0x0~2000 Blob(0x558c5794c930 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895ae70 loaded (sbid 0x2a6e ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5794c930) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x92000~2000: 0x0~2000 Blob(0x558c5794c930 blob([0x3a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895ae70 loaded (sbid 0x2a6e ref_map(0x3a9000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'45 crt=57'41 lcod 58'44 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.639+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x94000~2000: 0x0~2000 Blob(0x558c5895aee0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895af50 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10863 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895aee0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895af50 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 58'46, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895aee0 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895af50 loaded (sbid 0x2a6f ref_map(0x3ab000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c580a5110 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895af50 loaded (sbid 0x2a6f ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c580a5110) get_ref 0x0~2000 Blob(0x558c580a5110 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895af50 loaded (sbid 0x2a6f ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c580a5110) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x94000~2000: 0x0~2000 Blob(0x558c580a5110 blob([0x3ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895af50 loaded (sbid 0x2a6f ref_map(0x3ab000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x96000~2000: 0x0~2000 Blob(0x558c5895afc0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b030 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10864 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895afc0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b030 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895afc0 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b030 loaded (sbid 0x2a70 ref_map(0x3ad000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5794ca10 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b030 loaded (sbid 0x2a70 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5794ca10) get_ref 0x0~2000 Blob(0x558c5794ca10 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b030 loaded (sbid 0x2a70 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5794ca10) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x96000~2000: 0x0~2000 Blob(0x558c5794ca10 blob([0x3ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b030 loaded (sbid 0x2a70 ref_map(0x3ad000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x98000~2000: 0x0~2000 Blob(0x558c5895b0a0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b110 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10865 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895b0a0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b110 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586e1500 seq 55 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895b0a0 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b110 loaded (sbid 0x2a71 ref_map(0x3af000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c578807e0 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b110 loaded (sbid 0x2a71 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c578807e0) get_ref 0x0~2000 Blob(0x558c578807e0 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b110 loaded (sbid 0x2a71 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c578807e0) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x98000~2000: 0x0~2000 Blob(0x558c578807e0 blob([0x3af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b110 loaded (sbid 0x2a71 ref_map(0x3af000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys = 0 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x9a000~2000: 0x0~2000 Blob(0x558c5895b180 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b1f0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10866 2019-12-07T19:34:35.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895b180 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b1f0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.639+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895b180 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b1f0 loaded (sbid 0x2a72 ref_map(0x3b1000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000058.00000000000000000046' <- 0000000058.00000000000000000046 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57880700 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b1f0 loaded (sbid 0x2a72 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880700) get_ref 0x0~2000 Blob(0x558c57880700 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b1f0 loaded (sbid 0x2a72 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880700) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x9a000~2000: 0x0~2000 Blob(0x558c57880700 blob([0x3b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b1f0 loaded (sbid 0x2a72 ref_map(0x3b1000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x9c000~2000: 0x0~2000 Blob(0x558c5895b260 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b2d0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10867 2019-12-07T19:34:35.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895b260 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b2d0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e1500 cost 6138 (1 ios * 4000 + 2138 bytes) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e1500 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895b260 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b2d0 loaded (sbid 0x2a73 ref_map(0x3b3000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57880620 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b2d0 loaded (sbid 0x2a73 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 602 (600 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880620) get_ref 0x0~2000 Blob(0x558c57880620 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b2d0 loaded (sbid 0x2a73 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880620) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e1500 allocated 0x[] released 0x[] 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x9c000~2000: 0x0~2000 Blob(0x558c57880620 blob([0x3b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b2d0 loaded (sbid 0x2a73 ref_map(0x3b3000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0x9e000~2000: 0x0~2000 Blob(0x558c5895b340 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b3b0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10868 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895b340 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b3b0 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1500 prepare 2019-12-07T19:34:35.639+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e1500 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895b340 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b3b0 loaded (sbid 0x2a74 ref_map(0x3b5000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1500 io_done 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57880540 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b3b0 loaded (sbid 0x2a74 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880540) get_ref 0x0~2000 Blob(0x558c57880540 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b3b0 loaded (sbid 0x2a74 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880540) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0x9e000~2000: 0x0~2000 Blob(0x558c57880540 blob([0x3b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b3b0 loaded (sbid 0x2a74 ref_map(0x3b5000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'45 crt=57'41 lcod 58'44 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'46 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0xa0000~2000: 0x0~2000 Blob(0x558c5895b420 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b490 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10869 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895b420 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b490 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f07187700 10 osd.7 58 dequeue_op 0x558c57f3fb80 finish 2019-12-07T19:34:35.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895b420 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b490 loaded (sbid 0x2a75 ref_map(0x3b7000~2000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c57880d20 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b490 loaded (sbid 0x2a75 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880d20) get_ref 0x0~2000 Blob(0x558c57880d20 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b490 loaded (sbid 0x2a75 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c57880d20) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xa0000~2000: 0x0~2000 Blob(0x558c57880d20 blob([0x3b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b490 loaded (sbid 0x2a75 ref_map(0x3b7000~2000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0xa2000~2000: 0x0~2000 Blob(0x558c5895b500 blob([0x3b9000~1000,0xb4a000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b570 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10870 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895b500 blob([0x3b9000~1000,0xb4a000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b570 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895b500 blob([0x3b9000~1000,0xb4a000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b570 loaded (sbid 0x2a76 ref_map(0x3b9000~1000=1,0xb4a000~1000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c5809aa80 blob([0x3b9000~1000,0xb4a000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b570 loaded (sbid 0x2a76 ref_map(0x3b9000~1000=2,0xb4a000~1000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5809aa80) get_ref 0x0~2000 Blob(0x558c5809aa80 blob([0x3b9000~1000,0xb4a000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b570 loaded (sbid 0x2a76 ref_map(0x3b9000~1000=2,0xb4a000~1000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c5809aa80) get_ref init 0x2000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xa2000~2000: 0x0~2000 Blob(0x558c5809aa80 blob([0x3b9000~1000,0xb4a000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5895b570 loaded (sbid 0x2a76 ref_map(0x3b9000~1000=2,0xb4a000~1000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup src 0xa4000~1000: 0x0~1000 Blob(0x558c5895b5e0 blob([0xb4b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5895b650 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10871 2019-12-07T19:34:35.639+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared Blob(0x558c5895b5e0 blob([0xb4b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5895b650 sbid 0x0)) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.9s4_head 0x558c54d0ef00) make_blob_shared now Blob(0x558c5895b5e0 blob([0xb4b000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5895b650 loaded (sbid 0x2a77 ref_map(0xb4b000~1000=1)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup new Blob(0x558c550b22a0 blob([0xb4b000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b650 loaded (sbid 0x2a77 ref_map(0xb4b000~1000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c550b22a0) get_ref 0x0~1000 Blob(0x558c550b22a0 blob([0xb4b000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5895b650 loaded (sbid 0x2a77 ref_map(0xb4b000~1000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.blob(0x558c550b22a0) get_ref init 0x1000, 1000 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dup dst 0xa4000~1000: 0x0~1000 Blob(0x558c550b22a0 blob([0xb4b000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5895b650 loaded (sbid 0x2a77 ref_map(0xb4b000~1000=2)))) 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dirty_range mark shard 0x28000 dirty 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) dirty_range mark shard 0x4f000 dirty 2019-12-07T19:34:35.639+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) dirty_range mark inline shard dirty 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# -> 4#1:9b8adabe:::smithi13140796-6:23# = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# snapset 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# snapset = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# 1 keys 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# 1 keys = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# hinfo_key (42 bytes) 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 22 keys 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 22 keys = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..' <- SNA_1_0000000000000021_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..' <- SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000058.00000000000000000045' <- 0000000058.00000000000000000045 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000058.00000000000000000046' <- 0000000058.00000000000000000046 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c579d3800 cost 96427 (1 ios * 4000 + 92427 bytes) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c579d3800 onodes 0x558c57a35400,0x558c58768000 shared_blobs 0x558c54e1c000,0x558c54e1c0e0,0x558c54e1c4d0,0x558c54e1c5b0,0x558c54e1c690,0x558c54e1c770,0x558c54e1c850,0x558c54e1c930,0x558c54e1ca10,0x558c54e1caf0,0x558c54e1cbd0,0x558c54e1ccb0,0x558c54e1cd90,0x558c54e1ce70,0x558c54e1cf50,0x558c54e1d030,0x558c54e1d110,0x558c57314000,0x558c574e0e00,0x558c574e1ce0,0x558c575f7110,0x558c575f71f0,0x558c575f73b0,0x558c575f7490,0x558c575f7570,0x558c575f7650,0x558c575f7730,0x558c575f7810,0x558c575f78f0,0x558c575f79d0,0x558c575f7ab0,0x558c575f7b90,0x558c576adc00,0x558c576adc70,0x558c57880f50,0x558c57881180,0x558c57881340,0x558c57881420,0x558c578de3f0,0x558c578de4d0,0x558c578de690,0x558c578de930,0x558c578deaf0,0x558c578ded20,0x558c578dee00,0x558c578deee0,0x558c578defc0,0x558c578df0a0,0x558c578df180,0x558c578df650,0x558c578f1730,0x558c578f17a0,0x558c5794e3f0,0x558c5794e4d0,0x558c5794e5b0,0x558c57970000,0x558c57970070,0x558c579700e0,0x558c57970f50,0x558c579b2150,0x558c579b2cb0,0x558c579b2d20,0x558c579b2d90,0x558c579b2e00,0x558c579b2e70,0x558c579b2ee0,0x558c57bea070,0x558c57dcac40,0x558c57dcb6c0,0x558c57f67880,0x558c57f678f0,0x558c57f67960,0x558c57f67a40,0x558c5809ae70,0x558c5895ae70,0x558c5895af50,0x558c5895b030,0x558c5895b110,0x558c5895b1f0,0x558c5895b2d0,0x558c5895b3b0,0x558c5895b490,0x558c5895b570,0x558c5895b650 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update shard 0x28000 is 543 bytes (was 535) from 20 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update shard 0x4f000 is 1214 bytes (was 862) from 44 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore.extentmap(0x558c57a35508) reshard 0x[0,ffffffff) of 3 shards on 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard shards [0,3) over 0x[0,ffffffff) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard extent_avg 23, target 500, slop 100 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard new shard 0x0 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard new shard 0x2a000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard new shard 0x53000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard new shard 0x7c000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard new [0x0(0x0 bytes),0x2a000(0x0 bytes),0x53000(0x0 bytes),0x7c000(0x0 bytes)] 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard old [0x0(0x21f bytes),0x28000(0x217 bytes),0x4f000(0x35e bytes)] 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard fin [0x0(0x0 bytes),0x2a000(0x0 bytes),0x53000(0x0 bytes),0x7c000(0x0 bytes)] 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# force 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update shard 0x0 is 570 bytes (was 0) from 21 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update shard 0x2a000 is 570 bytes (was 0) from 21 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update shard 0x53000 is 591 bytes (was 0) from 21 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update shard 0x7c000 is 574 bytes (was 0) from 21 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 210635 (210633 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update 4#1:9b8adabe:::smithi13140796-6:23# 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update inline shard 2290 bytes from 84 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore.extentmap(0x558c58768108) reshard 0x[0,ffffffff) of 0 shards on 4#1:9b8adabe:::smithi13140796-6:23# 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard new shard 0x0 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard new shard 0x24000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard new shard 0x48000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard new shard 0x6b000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard new shard 0x8e000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes),0x6b000(0x0 bytes),0x8e000(0x0 bytes)] 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard old [] 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes),0x6b000(0x0 bytes),0x8e000(0x0 bytes)] 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update 4#1:9b8adabe:::smithi13140796-6:23# force 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update shard 0x48000 is 494 bytes (was 0) from 18 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update shard 0x6b000 is 505 bytes (was 0) from 18 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore.extentmap(0x558c58768108) update shard 0x8e000 is 331 bytes (was 0) from 12 extents 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:23# is 123322 (123320 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x41229e 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ba is 11 SharedBlob(0x558c54e1c000 loaded (sbid 0x28ba ref_map(0x303000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bb is 11 SharedBlob(0x558c54e1c0e0 loaded (sbid 0x28bb ref_map(0x305000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c7 is 11 SharedBlob(0x558c54e1c4d0 loaded (sbid 0x28c7 ref_map(0x49d000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c6 is 11 SharedBlob(0x558c54e1c5b0 loaded (sbid 0x28c6 ref_map(0x49b000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c5 is 11 SharedBlob(0x558c54e1c690 loaded (sbid 0x28c5 ref_map(0x499000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c4 is 11 SharedBlob(0x558c54e1c770 loaded (sbid 0x28c4 ref_map(0x497000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bd is 11 SharedBlob(0x558c54e1c850 loaded (sbid 0x28bd ref_map(0x489000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bc is 11 SharedBlob(0x558c54e1c930 loaded (sbid 0x28bc ref_map(0x307000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c8 is 11 SharedBlob(0x558c54e1ca10 loaded (sbid 0x28c8 ref_map(0x49f000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c9 is 11 SharedBlob(0x558c54e1caf0 loaded (sbid 0x28c9 ref_map(0x4a1000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ca is 11 SharedBlob(0x558c54e1cbd0 loaded (sbid 0x28ca ref_map(0x4a3000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cb is 11 SharedBlob(0x558c54e1ccb0 loaded (sbid 0x28cb ref_map(0x4a5000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cc is 11 SharedBlob(0x558c54e1cd90 loaded (sbid 0x28cc ref_map(0x4a7000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cd is 11 SharedBlob(0x558c54e1ce70 loaded (sbid 0x28cd ref_map(0x4a9000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ce is 11 SharedBlob(0x558c54e1cf50 loaded (sbid 0x28ce ref_map(0x4ab000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28cf is 11 SharedBlob(0x558c54e1d030 loaded (sbid 0x28cf ref_map(0x4ad000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28d0 is 11 SharedBlob(0x558c54e1d110 loaded (sbid 0x28d0 ref_map(0x4af000~1000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a5a is 11 SharedBlob(0x558c57314000 loaded (sbid 0x2a5a ref_map(0x382000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c1 is 11 SharedBlob(0x558c574e0e00 loaded (sbid 0x28c1 ref_map(0x491000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c0 is 11 SharedBlob(0x558c574e1ce0 loaded (sbid 0x28c0 ref_map(0x48f000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a63 is 11 SharedBlob(0x558c575f7110 loaded (sbid 0x2a63 ref_map(0x393000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a61 is 11 SharedBlob(0x558c575f71f0 loaded (sbid 0x2a61 ref_map(0x38f000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a64 is 11 SharedBlob(0x558c575f73b0 loaded (sbid 0x2a64 ref_map(0x395000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a65 is 11 SharedBlob(0x558c575f7490 loaded (sbid 0x2a65 ref_map(0x397000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a66 is 11 SharedBlob(0x558c575f7570 loaded (sbid 0x2a66 ref_map(0x399000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a67 is 11 SharedBlob(0x558c575f7650 loaded (sbid 0x2a67 ref_map(0x39b000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a68 is 11 SharedBlob(0x558c575f7730 loaded (sbid 0x2a68 ref_map(0x39d000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a69 is 11 SharedBlob(0x558c575f7810 loaded (sbid 0x2a69 ref_map(0x39f000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a6a is 11 SharedBlob(0x558c575f78f0 loaded (sbid 0x2a6a ref_map(0x3a1000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a6b is 11 SharedBlob(0x558c575f79d0 loaded (sbid 0x2a6b ref_map(0x3a3000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a6c is 11 SharedBlob(0x558c575f7ab0 loaded (sbid 0x2a6c ref_map(0x3a5000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a6d is 11 SharedBlob(0x558c575f7b90 loaded (sbid 0x2a6d ref_map(0x3a7000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4b is 11 SharedBlob(0x558c576adc00 loaded (sbid 0x2a4b ref_map(0x33d000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4f is 11 SharedBlob(0x558c576adc70 loaded (sbid 0x2a4f ref_map(0x345000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b0 is 11 SharedBlob(0x558c57880f50 loaded (sbid 0x28b0 ref_map(0x2ef000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ad is 11 SharedBlob(0x558c57881180 loaded (sbid 0x28ad ref_map(0x2e9000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28af is 11 SharedBlob(0x558c57881340 loaded (sbid 0x28af ref_map(0x2ed000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ae is 11 SharedBlob(0x558c57881420 loaded (sbid 0x28ae ref_map(0x2eb000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b1 is 11 SharedBlob(0x558c578de3f0 loaded (sbid 0x28b1 ref_map(0x2f1000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b3 is 11 SharedBlob(0x558c578de4d0 loaded (sbid 0x28b3 ref_map(0x2f5000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b2 is 11 SharedBlob(0x558c578de690 loaded (sbid 0x28b2 ref_map(0x2f3000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b5 is 11 SharedBlob(0x558c578de930 loaded (sbid 0x28b5 ref_map(0x2f9000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b4 is 11 SharedBlob(0x558c578deaf0 loaded (sbid 0x28b4 ref_map(0x2f7000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c2 is 11 SharedBlob(0x558c578ded20 loaded (sbid 0x28c2 ref_map(0x493000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28c3 is 11 SharedBlob(0x558c578dee00 loaded (sbid 0x28c3 ref_map(0x495000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b6 is 11 SharedBlob(0x558c578deee0 loaded (sbid 0x28b6 ref_map(0x2fb000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b7 is 11 SharedBlob(0x558c578defc0 loaded (sbid 0x28b7 ref_map(0x2fd000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b8 is 11 SharedBlob(0x558c578df0a0 loaded (sbid 0x28b8 ref_map(0x2ff000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28b9 is 11 SharedBlob(0x558c578df180 loaded (sbid 0x28b9 ref_map(0x301000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a62 is 11 SharedBlob(0x558c578df650 loaded (sbid 0x2a62 ref_map(0x391000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28bf is 11 SharedBlob(0x558c578f1730 loaded (sbid 0x28bf ref_map(0x48d000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28be is 11 SharedBlob(0x558c578f17a0 loaded (sbid 0x28be ref_map(0x48b000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a5d is 11 SharedBlob(0x558c5794e3f0 loaded (sbid 0x2a5d ref_map(0x388000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a5e is 11 SharedBlob(0x558c5794e4d0 loaded (sbid 0x2a5e ref_map(0x38a000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a5f is 11 SharedBlob(0x558c5794e5b0 loaded (sbid 0x2a5f ref_map(0x38c000~1000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4c is 11 SharedBlob(0x558c57970000 loaded (sbid 0x2a4c ref_map(0x33f000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a50 is 11 SharedBlob(0x558c57970070 loaded (sbid 0x2a50 ref_map(0x347000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.002562251s (0.000000078s flush + 0.002562173s kv commit) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a54 is 15 SharedBlob(0x558c579700e0 loaded (sbid 0x2a54 ref_map(0x34f000~1000=2,0x377000~1000=2))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5876bb00,0x558c57bd2f00> 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a60 is 11 SharedBlob(0x558c57970f50 loaded (sbid 0x2a60 ref_map(0x38d000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876bb00 kv_submitted 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5876bb00 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a53 is 11 SharedBlob(0x558c579b2150 loaded (sbid 0x2a53 ref_map(0x34d000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4e is 11 SharedBlob(0x558c579b2cb0 loaded (sbid 0x2a4e ref_map(0x343000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876bb00 finishing 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4d is 11 SharedBlob(0x558c579b2d20 loaded (sbid 0x2a4d ref_map(0x341000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5876bb00 onodes 0x558c579de500 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5876bb00 done 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a52 is 11 SharedBlob(0x558c579b2d90 loaded (sbid 0x2a52 ref_map(0x34b000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2452 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5876bb00 [] 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a51 is 11 SharedBlob(0x558c579b2e00 loaded (sbid 0x2a51 ref_map(0x349000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a56 is 11 SharedBlob(0x558c579b2e70 loaded (sbid 0x2a56 ref_map(0x37a000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a55 is 11 SharedBlob(0x558c579b2ee0 loaded (sbid 0x2a55 ref_map(0x378000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2f00 kv_submitted 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd2f00 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ac is 11 SharedBlob(0x558c57bea070 loaded (sbid 0x28ac ref_map(0x2e7000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd2f00 finishing 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd2f00 onodes 0x558c575a3b80,0x558c58092000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28ab is 11 SharedBlob(0x558c57dcac40 loaded (sbid 0x28ab ref_map(0x2e5000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd2f00 done 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd3200 kv_queued 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd2f00 [] 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x28aa is 11 SharedBlob(0x558c57dcb6c0 loaded (sbid 0x28aa ref_map(0x2e3000~2000=3))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a5c is 11 SharedBlob(0x558c57f67880 loaded (sbid 0x2a5c ref_map(0x386000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a59 is 11 SharedBlob(0x558c57f678f0 loaded (sbid 0x2a59 ref_map(0x380000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.640+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 58/41 ECSubWriteReply(tid=293, last_complete=58'8, committed=1, applied=1)) v2 -- 0x558c57951e00 con 0x558c54c64000 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a58 is 11 SharedBlob(0x558c57f67960 loaded (sbid 0x2a58 ref_map(0x37e000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a57 is 11 SharedBlob(0x558c57f67a40 loaded (sbid 0x2a57 ref_map(0x37c000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 2 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a5b is 11 SharedBlob(0x558c5809ae70 loaded (sbid 0x2a5b ref_map(0x384000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 2 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a6e is 11 SharedBlob(0x558c5895ae70 loaded (sbid 0x2a6e ref_map(0x3a9000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a6f is 11 SharedBlob(0x558c5895af50 loaded (sbid 0x2a6f ref_map(0x3ab000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4610 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a70 is 11 SharedBlob(0x558c5895b030 loaded (sbid 0x2a70 ref_map(0x3ad000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a71 is 11 SharedBlob(0x558c5895b110 loaded (sbid 0x2a71 ref_map(0x3af000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a72 is 11 SharedBlob(0x558c5895b1f0 loaded (sbid 0x2a72 ref_map(0x3b1000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a73 is 11 SharedBlob(0x558c5895b2d0 loaded (sbid 0x2a73 ref_map(0x3b3000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a74 is 11 SharedBlob(0x558c5895b3b0 loaded (sbid 0x2a74 ref_map(0x3b5000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a75 is 11 SharedBlob(0x558c5895b490 loaded (sbid 0x2a75 ref_map(0x3b7000~2000=2))) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a3b80 had 1 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a76 is 15 SharedBlob(0x558c5895b570 loaded (sbid 0x2a76 ref_map(0x3b9000~1000=2,0xb4a000~1000=2))) 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6409 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a77 is 11 SharedBlob(0x558c5895b650 loaded (sbid 0x2a77 ref_map(0xb4b000~1000=2))) 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c579d3800 allocated 0x[] released 0x[] 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6432 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6432 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3800 prepare 2019-12-07T19:34:35.640+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c579d3800 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.640+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3800 io_done 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x41229e~1920 to file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x41229e 0x41229e~1920 to file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~3f0000 x_off 0x3e229e 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x29e 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xbbe and padding block with 0x442 2019-12-07T19:34:35.640+0000 7f8f08189700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=57'44 crt=57'44 lcod 57'44 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 58'46 2019-12-07T19:34:35.640+0000 7f8f08189700 10 osd.7 58 dequeue_op 0x558c5740d080 finish 2019-12-07T19:34:35.640+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x413bbe 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000125044s (0.000004409s flush + 0.000120635s kv commit) 2019-12-07T19:34:35.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586e1200,0x558c57bd3200,0x558c586e1500> 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1200 kv_submitted 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e1200 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 17474 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1200 finishing 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e1200 onodes 0x558c54e44780 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587666a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e142d0 space 0x558c587666a8 0x0~2000 writing nocache) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c587667f8 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2ea0 space 0x558c587667f8 0x0~2000 writing nocache) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58766948 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2e10 space 0x558c58766948 0x1000~1000 writing nocache) 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e1200 done 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 50242 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e1500 kv_submitted 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e1200 [] 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3200 kv_submitted 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd3200 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3200 finishing 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd3200 onodes 0x558c575a3b80 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd3200 done 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd3200 [] 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1500 kv_submitted 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e1500 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1500 finishing 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e1500 onodes 0x558c54e44780 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e1500 done 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e1500 [] 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 83010 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.640+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.640+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 115778 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5740c000 prio 196 cost 0 latency 0.002331 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 148546 < min_flush_size 524288 2019-12-07T19:34:35.640+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5740c000 finish 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 181314 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 58/41 ECSubWriteReply(tid=114, last_complete=58'45, committed=1, applied=1)) v2 -- 0x558c551afc00 con 0x558c54c5b400 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 214082 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 58/41 ECSubWriteReply(tid=115, last_complete=58'46, committed=1, applied=1)) v2 -- 0x558c57951400 con 0x558c54c5b400 2019-12-07T19:34:35.641+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 246850 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 279618 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 312386 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 345055 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 201 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d000 con 0x558c54c19800 2019-12-07T19:34:35.641+0000 7f8f24c58700 15 osd.7 58 enqueue_op 0x558c57335b80 prio 196 cost 0 latency 0.000019 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c57335b80 prio 196 cost 0 latency 0.000048 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 497 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58115400 con 0x558c54c64000 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c57335b80 finish 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5740de40 prio 196 cost 0 latency 0.000014 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5740de40 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 111 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57951c00 con 0x558c54c19c00 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5740de40 finish 2019-12-07T19:34:35.641+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511fb80 prio 196 cost 0 latency 0.000013 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511fb80 prio 196 cost 0 latency 0.000032 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511fb80 finish 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58768000 had 1 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 345078 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 345078 < min_flush_size 524288 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x413bbe~543f6 to file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x413bbe 0x413bbe~543f6 to file(ino 16 size 0x420000 mtime 2019-12-07T19:34:35.638508+0000 allocated 420000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~3f0000]) 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _allocate len 0x47fb4 from 0 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _allocate len 0x47fb4 from 1 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x50000/10000,50000,acd550000 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd550000~50000/10000,50000,acd550000 2019-12-07T19:34:35.641+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x470000 to include allocated 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 73 (was clean) 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x3e3bbe 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xbbe 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfb4 and padding block with 0x4c 2019-12-07T19:34:35.641+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x470000 2019-12-07T19:34:35.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 498 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332000 con 0x558c54c64000 2019-12-07T19:34:35.641+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511edc0 prio 196 cost 0 latency 0.000015 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511edc0 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 112 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57568800 con 0x558c54c19c00 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511edc0 finish 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511e2c0 prio 196 cost 0 latency 0.000015 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511e2c0 prio 196 cost 0 latency 0.000058 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511e2c0 finish 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 306 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5795d800 con 0x558c54c5b400 2019-12-07T19:34:35.641+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c587546e0 prio 196 cost 0 latency 0.000019 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c587546e0 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.641+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 288 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c54c5bc00 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=49, last_complete=58'44, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c58754840 prio 196 cost 0 latency 0.000009 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.641+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=58'44 tt=0'0 tid=49 reqid=client.4540.0:441 client_op=osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.641+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=57'42 crt=57'42 lcod 53'40 mlcod 53'40 active+clean ps=[e~1,15~1,1a~1]] repop_all_committed: repop tid 49 all committed 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b6fc0 58'44 rep_tid=49 committed?=1 r=0) 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] commit: repgather(0x558c576b6fc0 58'44 rep_tid=49 committed?=1 r=0) 2019-12-07T19:34:35.641+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 499 ==== MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=294, reqid=client.4540.0:439, at_version=0'0, trim_to=0'0, roll_forward_to=58'8)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57e7c000 con 0x558c54c64000 2019-12-07T19:34:35.641+0000 7f8f06986700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] log_op_stats osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 inb 0 outb 0 lat 0.006732 2019-12-07T19:34:35.641+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511fe40 prio 127 cost 0 latency 0.000019 epoch 58 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=294, reqid=client.4540.0:439, at_version=0'0, trim_to=0'0, roll_forward_to=58'8)) v2 2019-12-07T19:34:35.641+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=294, reqid=client.4540.0:439, at_version=0'0, trim_to=0'0, roll_forward_to=58'8)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] sending reply on osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 0x558c54d2c280 2019-12-07T19:34:35.641+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(441 smithi13140796-23 [append 0~0] v58'44 uv44 ondisk = 0) v8 -- 0x558c54d2c280 con 0x558c54c9f000 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=294, reqid=client.4540.0:439, at_version=0'0, trim_to=0'0, roll_forward_to=58'8)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=294, reqid=client.4540.0:439, at_version=0'0, trim_to=0'0, roll_forward_to=58'8)) v2) prio 127 cost 0 e58) pg 0x558c54d98000 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:35.641+0000 7f8f0197c700 10 osd.7 58 dequeue_op 0x558c5511fe40 prio 127 cost 0 latency 0.000046 MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=294, reqid=client.4540.0:439, at_version=0'0, trim_to=0'0, roll_forward_to=58'8)) v2 pg pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.641+0000 7f8f06986700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:193 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] removing repgather(0x558c576b6fc0 58'44 rep_tid=49 committed?=1 r=0) 2019-12-07T19:34:35.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.es4 58/41 ECSubWrite(tid=294, reqid=client.4540.0:439, at_version=0'0, trim_to=0'0, roll_forward_to=58'8)) v2 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] q front is repgather(0x558c576b6fc0 58'44 rep_tid=49 committed?=1 r=0) 2019-12-07T19:34:35.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=21'4 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'8], crt=21'4) [] 2019-12-07T19:34:35.641+0000 7f8f06986700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] remove_repop repgather(0x558c576b6fc0 58'44 rep_tid=49 committed?=1 r=0) 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'5 (21'2) clone 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:33:58.627337+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'6 (21'2) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:437 2019-12-07T19:34:35.633816+0000 0 ObjectCleanRegions clean_offsets: [0~81920,81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'44 tt=0'0 tid=49 reqid=client.4540.0:441 client_op=osd_op(client.4540.0:441 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'7 (58'6) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:438 2019-12-07T19:34:35.633837+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 0, new_object: 0 2019-12-07T19:34:35.641+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c587546e0 finish 2019-12-07T19:34:35.641+0000 7f8f0197c700 20 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'8 (58'7) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:439 2019-12-07T19:34:35.633841+0000 0 ObjectCleanRegions clean_offsets: [0~81920,81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.641+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 8 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.642+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5853ef00 seq 26 2019-12-07T19:34:35.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.es4_head 4#1:733d7474:::smithi13140796-5:head#7 onode 0x558c54ce3180 txc 0x558c5853ef00 2019-12-07T19:34:35.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.es4_head 4#1:733d7474:::smithi13140796-5:head#7 0x0 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57896380) put_ref 0x0~2000 Blob(0x558c57896380 blob([0x6b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=3)))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore.blob(0x558c578962a0) put_ref 0x0~2000 Blob(0x558c578962a0 blob([0x6b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=3)))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore.blob(0x558c57896070) put_ref 0x0~1000 Blob(0x558c57896070 blob([0x6bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=3)))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore.extentmap(0x558c54ce3288) dirty_range mark inline shard dirty 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57896380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=3)))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x6b7000~2000] 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=2))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c578962a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=3)))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x6b9000~2000] 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=2))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~1000: 0x0~1000 Blob(0x558c57896070 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=3)))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x6bb000~1000] 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=2))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.es4_head 4#1:733d7474:::smithi13140796-5:head#7 = 0 2019-12-07T19:34:35.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853ef00 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853ef00 onodes shared_blobs 0x558c54df8150,0x558c57896310,0x558c57896540 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a39 is 11 SharedBlob(0x558c54df8150 loaded (sbid 0x2a39 ref_map(0x6b9000~2000=2))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a38 is 11 SharedBlob(0x558c57896310 loaded (sbid 0x2a38 ref_map(0x6b7000~2000=2))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3a is 11 SharedBlob(0x558c57896540 loaded (sbid 0x2a3a ref_map(0x6bb000~1000=2))) 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853ef00 allocated 0x[] released 0x[] 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ef00 prepare 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853ef00 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ef00 io_done 2019-12-07T19:34:35.642+0000 7f8f0197c700 10 osd.7 58 dequeue_op 0x558c5511fe40 finish 2019-12-07T19:34:35.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.642+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c58754840 prio 196 cost 0 latency 0.000392 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.642+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.642+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c58754840 finish 2019-12-07T19:34:35.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.642+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 202 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58115600 con 0x558c54c19800 2019-12-07T19:34:35.642+0000 7f8f24c58700 15 osd.7 58 enqueue_op 0x558c57f3edc0 prio 196 cost 0 latency 0.000018 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.642+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.642+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 203 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59200 con 0x558c54c19800 2019-12-07T19:34:35.642+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c57f3edc0 prio 196 cost 0 latency 0.000039 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.642+0000 7f8f24c58700 15 osd.7 58 enqueue_op 0x558c57f3f600 prio 196 cost 0 latency 0.000009 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.642+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.642+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c57f3edc0 finish 2019-12-07T19:34:35.642+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.642+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.642+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.642+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c57f3f600 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c57f3f600 finish 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 500 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58513200 con 0x558c54c64000 2019-12-07T19:34:35.643+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1501166 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.643+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511e840 prio 196 cost 0 latency 0.000012 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x467fb4 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.643+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.643+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 289 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57951400 con 0x558c54c5bc00 2019-12-07T19:34:35.643+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511e840 prio 196 cost 0 latency 0.000076 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.643+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (73) on file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]), flushing log 2019-12-07T19:34:35.643+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511e840 finish 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:35.643+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.643+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c587549a0 prio 196 cost 0 latency 0.000016 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.643+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 73 len 0x4a crc 0x509a2047) 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x48000~1000 to file(ino 1 size 0x48000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x48000 0x48000~1000 to file(ino 1 size 0x48000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x49000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x48000 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x49000 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:35.643+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 113 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57951e00 con 0x558c54c19c00 2019-12-07T19:34:35.643+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511e9a0 prio 196 cost 0 latency 0.000010 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000037 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c587549a0 prio 196 cost 0 latency 0.000139 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 73 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c587549a0 finish 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002797515s (0.000000110s flush + 0.002797405s kv commit) 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c579d3800> 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 76 < min_flush_size 524288 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3800 kv_submitted 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c579d3800 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d3800 finishing 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c579d3800 onodes 0x558c57a35400,0x558c58768000 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c579d3800 done 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511e9a0 prio 196 cost 0 latency 0.000110 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c579d3800 [] 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.643+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511e9a0 finish 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54ce3180 had 1 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:35.643+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 58/41 ECSubWriteReply(tid=292, last_complete=58'46, committed=1, applied=1)) v2 -- 0x558c57951000 con 0x558c54c64000 2019-12-07T19:34:35.643+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1326 < min_flush_size 524288 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1326 < min_flush_size 524288 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x467fb4~52e to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x467fb4 0x467fb4~52e to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x437fb4 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfb4 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4e2 and padding block with 0xb1e 2019-12-07T19:34:35.643+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4684e2 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.643+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000082430s (0.000000081s flush + 0.000082349s kv commit) 2019-12-07T19:34:35.643+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853ef00> 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ef00 kv_submitted 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853ef00 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853ef00 finishing 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853ef00 onodes 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853ef00 done 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853ef00 [] 2019-12-07T19:34:35.643+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.643+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.643+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.643+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 58/41 ECSubWriteReply(tid=294, last_complete=58'8, committed=1, applied=1)) v2 -- 0x558c548cc200 con 0x558c54c64000 2019-12-07T19:34:35.643+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:35.643+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 307 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58115800 con 0x558c54c5b400 2019-12-07T19:34:35.643+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c58754b00 prio 196 cost 0 latency 0.000011 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.643+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.644+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 308 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c576b5c00 con 0x558c54c5b400 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.644+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c58754c60 prio 196 cost 0 latency 0.000005 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.644+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c58754b00 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=50, last_complete=58'45, committed=1, applied=1)) v2 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=58'45 tt=0'0 tid=50 reqid=client.4540.0:442 client_op=osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'44 crt=57'42 lcod 57'42 mlcod 57'42 active+clean ps=[e~1,15~1,1a~1]] repop_all_committed: repop tid 50 all committed 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b70e0 58'45 rep_tid=50 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] commit: repgather(0x558c576b70e0 58'45 rep_tid=50 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f06986700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] log_op_stats osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 inb 0 outb 0 lat 0.008914 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] sending reply on osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 0x558c575a2f00 2019-12-07T19:34:35.644+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(442 smithi13140796-23 [rollback 21] v58'45 uv45 ondisk = 0) v8 -- 0x558c575a2f00 con 0x558c54c9f000 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:35.644+0000 7f8f06986700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 58:194 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] removing repgather(0x558c576b70e0 58'45 rep_tid=50 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] q front is repgather(0x558c576b70e0 58'45 rep_tid=50 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f06986700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] remove_repop repgather(0x558c576b70e0 58'45 rep_tid=50 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'45 tt=0'0 tid=50 reqid=client.4540.0:442 client_op=osd_op(client.4540.0:442 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [rollback 21] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:35.644+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c58754b00 finish 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c58754c60 prio 196 cost 0 latency 0.000130 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=51, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=58'46 tt=0'0 tid=51 reqid=client.4540.0:443 client_op=osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'45 crt=57'42 lcod 58'44 mlcod 58'44 active+clean ps=[e~1,15~1,1a~1]] repop_all_committed: repop tid 51 all committed 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b7200 58'46 rep_tid=51 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] commit: repgather(0x558c576b7200 58'46 rep_tid=51 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f0297e700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] log_op_stats osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 inb 0 outb 0 lat 0.009050 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] sending reply on osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) v8 0x558c575a3900 2019-12-07T19:34:35.644+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(443 smithi13140796-23 [append 0~0] v58'46 uv46 ondisk = 0) v8 -- 0x558c575a3900 con 0x558c54c9f000 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:35.644+0000 7f8f0297e700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] publish_stats_to_osd 58:195 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] removing repgather(0x558c576b7200 58'46 rep_tid=51 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] q front is repgather(0x558c576b7200 58'46 rep_tid=51 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f0297e700 15 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] remove_repop repgather(0x558c576b7200 58'46 rep_tid=51 committed?=1 r=0) 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=58'46 tt=0'0 tid=51 reqid=client.4540.0:443 client_op=osd_op(client.4540.0:443 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~0] snapc 23=[23,21,20,1b] ondisk+write+known_if_redirected e58) roll_forward_to=57'42 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=52 reqid=client.4540.0:443 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=52 reqid=client.4540.0:443 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:35.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 501 ==== MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=295, reqid=client.4540.0:440, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c575be000 con 0x558c54c64000 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:35.644+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511e580 prio 127 cost 0 latency 0.000017 epoch 58 MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=295, reqid=client.4540.0:440, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:35.644+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=295, reqid=client.4540.0:440, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.644+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 58/28 ECSubWrite(tid=52, reqid=client.4540.0:443, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c576c9000 con 0x558c54c64000 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=295, reqid=client.4540.0:440, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.644+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 58/28 ECSubWrite(tid=52, reqid=client.4540.0:443, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c586ce000 con 0x558c54c19800 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=295, reqid=client.4540.0:440, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e58) pg 0x558c5500a000 2019-12-07T19:34:35.644+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 58/28 ECSubWrite(tid=52, reqid=client.4540.0:443, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c586ce800 con 0x558c54c5bc00 2019-12-07T19:34:35.644+0000 7f8f04181700 10 osd.7 58 dequeue_op 0x558c5511e580 prio 127 cost 0 latency 0.000061 MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=295, reqid=client.4540.0:440, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 pg pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.644+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 58/28 ECSubWrite(tid=52, reqid=client.4540.0:443, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c586cf000 con 0x558c54c5b400 2019-12-07T19:34:35.644+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.9s4 58/41 ECSubWrite(tid=295, reqid=client.4540.0:440, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:35.644+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 58/28 ECSubWrite(tid=52, reqid=client.4540.0:443, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c586cf800 con 0x558c54c19c00 2019-12-07T19:34:35.644+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=57'44 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'46], crt=57'44) [] 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'45 (57'44) clone 1:9b8adabe:::smithi13140796-6:23 by unknown.0.0:0 2019-12-07T19:34:32.653894+0000 0 snaps [23,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=57'42 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] append_log log((0'0,58'46], crt=57'42) [] 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'46 (57'44) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:440 2019-12-07T19:34:35.636266+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] rollforward: entry=58'43 (53'38) clone 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.644+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 46 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] rollforward: entry=58'44 (53'38) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:441 2019-12-07T19:34:35.636323+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.644+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.644+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.644+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] rollforward: entry=58'45 (58'44) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:442 2019-12-07T19:34:35.636336+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 0, new_object: 0 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] rollforward: entry=58'46 (58'45) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:443 2019-12-07T19:34:35.636340+0000 0 ObjectCleanRegions clean_offsets: [0~507904,507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] append_log approx pg log length = 46 2019-12-07T19:34:35.644+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:35.644+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57957200 seq 87 2019-12-07T19:34:35.644+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.644+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.644+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.644+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.644+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.644+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:35.644+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57957200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:35.644+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57957200 onodes shared_blobs 2019-12-07T19:34:35.644+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57957200 allocated 0x[] released 0x[] 2019-12-07T19:34:35.644+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957200 prepare 2019-12-07T19:34:35.644+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57957200 2019-12-07T19:34:35.644+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957200 io_done 2019-12-07T19:34:35.644+0000 7f8f04181700 10 osd.7 58 dequeue_op 0x558c5511e580 finish 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57bd3500 seq 64 2019-12-07T19:34:35.644+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head#2d onode 0x558c58092000 txc 0x558c57bd3500 2019-12-07T19:34:35.644+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head#2d 0x0 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff52d0) put_ref 0x0~2000 Blob(0x558c57ff52d0 blob([0x56e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff53b0) put_ref 0x0~2000 Blob(0x558c57ff53b0 blob([0x570000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5490) put_ref 0x0~2000 Blob(0x558c57ff5490 blob([0x572000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5570) put_ref 0x0~2000 Blob(0x558c57ff5570 blob([0x574000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff58f0) put_ref 0x0~2000 Blob(0x558c57ff58f0 blob([0x576000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5810) put_ref 0x0~2000 Blob(0x558c57ff5810 blob([0x578000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5730) put_ref 0x0~2000 Blob(0x558c57ff5730 blob([0x57a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5650) put_ref 0x0~2000 Blob(0x558c57ff5650 blob([0x57c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4e70) put_ref 0x0~2000 Blob(0x558c57ff4e70 blob([0x57e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff4f50) put_ref 0x0~2000 Blob(0x558c57ff4f50 blob([0x580000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5030) put_ref 0x0~2000 Blob(0x558c57ff5030 blob([0x582000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff59d0) put_ref 0x0~2000 Blob(0x558c57ff59d0 blob([0x584000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5ab0) put_ref 0x0~2000 Blob(0x558c57ff5ab0 blob([0x586000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5b90) put_ref 0x0~2000 Blob(0x558c57ff5b90 blob([0x588000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5c70) put_ref 0x0~2000 Blob(0x558c57ff5c70 blob([0x58a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.blob(0x558c57ff5d50) put_ref 0x0~1000 Blob(0x558c57ff5d50 blob([0x58c000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58092108) dirty_range mark inline shard dirty 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c57ff52d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x56e000~2000] 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4684e2~471 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c57ff53b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x570000~2000] 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4684e2 0x4684e2~471 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57ff5490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x572000~2000] 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x4384e2 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4e2 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57ff5570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x574000~2000] 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x953 and padding block with 0x6ad 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57ff58f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x576000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c57ff5810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x578000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57ff5730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x57a000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c57ff5650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x57c000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c57ff4e70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x57e000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c57ff4f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x580000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c57ff5030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x582000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c57ff59d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x584000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c57ff5ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x586000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c57ff5b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x588000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c57ff5c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x58a000~2000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~1000: 0x0~1000 Blob(0x558c57ff5d50 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=3)))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x58c000~1000] 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head#2d = 0 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x468953 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.644+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000086661s (0.000000081s flush + 0.000086580s kv commit) 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd3500 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd3500 onodes shared_blobs 0x558c57ff4e00,0x558c57ff4ee0,0x558c57ff4fc0,0x558c57ff5340,0x558c57ff5420,0x558c57ff5500,0x558c57ff55e0,0x558c57ff56c0,0x558c57ff57a0,0x558c57ff5880,0x558c57ff5960,0x558c57ff5a40,0x558c57ff5b20,0x558c57ff5c00,0x558c57ff5ce0,0x558c57ff5dc0 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57957200> 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a42 is 11 SharedBlob(0x558c57ff4e00 loaded (sbid 0x2a42 ref_map(0x57c000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957200 kv_submitted 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57957200 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a43 is 11 SharedBlob(0x558c57ff4ee0 loaded (sbid 0x2a43 ref_map(0x57e000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a44 is 11 SharedBlob(0x558c57ff4fc0 loaded (sbid 0x2a44 ref_map(0x580000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957200 finishing 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3b is 11 SharedBlob(0x558c57ff5340 loaded (sbid 0x2a3b ref_map(0x56e000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57957200 onodes 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57957200 done 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3c is 11 SharedBlob(0x558c57ff5420 loaded (sbid 0x2a3c ref_map(0x570000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3d is 11 SharedBlob(0x558c57ff5500 loaded (sbid 0x2a3d ref_map(0x572000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57957200 [] 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3e is 11 SharedBlob(0x558c57ff55e0 loaded (sbid 0x2a3e ref_map(0x574000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a41 is 11 SharedBlob(0x558c57ff56c0 loaded (sbid 0x2a41 ref_map(0x57a000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a40 is 11 SharedBlob(0x558c57ff57a0 loaded (sbid 0x2a40 ref_map(0x578000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a3f is 11 SharedBlob(0x558c57ff5880 loaded (sbid 0x2a3f ref_map(0x576000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a45 is 11 SharedBlob(0x558c57ff5960 loaded (sbid 0x2a45 ref_map(0x582000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a46 is 11 SharedBlob(0x558c57ff5a40 loaded (sbid 0x2a46 ref_map(0x584000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a47 is 11 SharedBlob(0x558c57ff5b20 loaded (sbid 0x2a47 ref_map(0x586000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a48 is 11 SharedBlob(0x558c57ff5c00 loaded (sbid 0x2a48 ref_map(0x588000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a49 is 11 SharedBlob(0x558c57ff5ce0 loaded (sbid 0x2a49 ref_map(0x58a000~2000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a4a is 11 SharedBlob(0x558c57ff5dc0 loaded (sbid 0x2a4a ref_map(0x58c000~1000=2))) 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd3500 allocated 0x[] released 0x[] 2019-12-07T19:34:35.644+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 58/41 ECSubWriteReply(tid=295, last_complete=58'46, committed=1, applied=1)) v2 -- 0x558c57951200 con 0x558c54c64000 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3500 prepare 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd3500 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3500 io_done 2019-12-07T19:34:35.644+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:35.644+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c58754c60 finish 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1605 < min_flush_size 524288 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092000 had 1 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1628 < min_flush_size 524288 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1628 < min_flush_size 524288 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x468953~65c to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x468953 0x468953~65c to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x438953 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x953 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xfaf and padding block with 0x51 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x468faf 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000154527s (0.000000101s flush + 0.000154426s kv commit) 2019-12-07T19:34:35.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57bd3500> 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3500 kv_submitted 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd3500 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3500 finishing 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd3500 onodes 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd3500 done 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd3500 [] 2019-12-07T19:34:35.644+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.644+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.644+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.645+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 309 ==== MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=116, reqid=client.4540.0:435, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c57e7c800 con 0x558c54c5b400 2019-12-07T19:34:35.645+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c58754dc0 prio 127 cost 0 latency 0.000017 epoch 58 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=116, reqid=client.4540.0:435, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:35.645+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=116, reqid=client.4540.0:435, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e58) 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=116, reqid=client.4540.0:435, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=116, reqid=client.4540.0:435, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e58) pg 0x558c575ef000 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c58754dc0 prio 127 cost 0 latency 0.000042 MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=116, reqid=client.4540.0:435, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 pg pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 58/41 ECSubWrite(tid=116, reqid=client.4540.0:435, at_version=0'0, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'46], crt=57'41) [] 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'42 (45'29) clone 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:22.635389+0000 0 snaps [23,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'43 (45'29) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:432 2019-12-07T19:34:35.629323+0000 0 ObjectCleanRegions clean_offsets: [0~1343488,2048000~18446744073707503615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'44 (58'43) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:433 2019-12-07T19:34:35.633226+0000 0 ObjectCleanRegions clean_offsets: [0~2048000,2736128~18446744073706815487], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'45 (58'44) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:434 2019-12-07T19:34:35.633769+0000 0 ObjectCleanRegions clean_offsets: [0~2736128,2818048~18446744073706733567], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.645+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 204 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b9200 con 0x558c54c19800 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=58'46 (58'45) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:435 2019-12-07T19:34:35.633784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:35.645+0000 7f8f24c58700 15 osd.7 58 enqueue_op 0x558c57f3fa20 prio 196 cost 0 latency 0.000019 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.645+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 46 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.645+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:35.645+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c57f3fa20 prio 196 cost 0 latency 0.000042 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.645+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.645+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c57f3fa20 finish 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a2c00 seq 56 2019-12-07T19:34:35.645+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a2c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a2c00 onodes shared_blobs 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a2c00 allocated 0x[] released 0x[] 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2c00 prepare 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a2c00 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2c00 io_done 2019-12-07T19:34:35.645+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c58754dc0 finish 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.645+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 502 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57951200 con 0x558c54c64000 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x468faf~471 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.645+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511f080 prio 196 cost 0 latency 0.000012 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x468faf 0x468faf~471 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.645+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x438faf 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xfaf 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x420 and padding block with 0xbe0 2019-12-07T19:34:35.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.645+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511f080 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.645+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c5511f080 finish 2019-12-07T19:34:35.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x469420 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:35.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000148288s (0.000000112s flush + 0.000148176s kv commit) 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:35.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586a2c00> 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:35.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2c00 kv_submitted 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a2c00 2019-12-07T19:34:35.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2c00 finishing 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a2c00 onodes 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a2c00 done 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:35.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a2c00 [] 2019-12-07T19:34:35.645+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:35.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:35.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.645+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 58/41 ECSubWriteReply(tid=116, last_complete=58'46, committed=1, applied=1)) v2 -- 0x558c54c58800 con 0x558c54c5b400 2019-12-07T19:34:35.645+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.645+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 290 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c58800 con 0x558c54c5bc00 2019-12-07T19:34:35.645+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c58754f20 prio 196 cost 0 latency 0.000011 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.645+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.645+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c58754f20 prio 196 cost 0 latency 0.000028 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.645+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.645+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c58754f20 finish 2019-12-07T19:34:35.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.646+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 310 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5763ca00 con 0x558c54c5b400 2019-12-07T19:34:35.646+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c58755080 prio 196 cost 0 latency 0.000012 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.646+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.646+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 114 ==== MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57951000 con 0x558c54c19c00 2019-12-07T19:34:35.646+0000 7f8f24457700 15 osd.7 58 enqueue_op 0x558c5511e6e0 prio 196 cost 0 latency 0.000010 epoch 58 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.646+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) 2019-12-07T19:34:35.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.646+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c58755080 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) queued 2019-12-07T19:34:35.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.646+0000 7f8f0297e700 10 osd.7 58 dequeue_op 0x558c58755080 finish 2019-12-07T19:34:35.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2) prio 196 cost 0 e58) pg 0x558c57676000 2019-12-07T19:34:35.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.646+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511e6e0 prio 196 cost 0 latency 0.000065 MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 pg pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.646+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 58/28 ECSubWriteReply(tid=52, last_complete=58'46, committed=1, applied=1)) v2 2019-12-07T19:34:35.646+0000 7f8f06986700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=52 reqid=client.4540.0:443 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:35.646+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:35.646+0000 7f8f06986700 20 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:35.646+0000 7f8f06986700 10 osd.7 58 dequeue_op 0x558c5511e6e0 finish 2019-12-07T19:34:35.646+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:35.646+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 311 ==== MOSDECSubOpRead(1.bs1 58/41 ECSubRead(tid=117, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c579dec80 con 0x558c54c5b400 2019-12-07T19:34:35.646+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c58755340 prio 127 cost 0 latency 0.000015 epoch 58 MOSDECSubOpRead(1.bs1 58/41 ECSubRead(tid=117, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:35.646+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 58/41 ECSubRead(tid=117, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e58) 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 58/41 ECSubRead(tid=117, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e58) queued 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 58/41 ECSubRead(tid=117, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e58) pg 0x558c575ef000 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c58755340 prio 127 cost 0 latency 0.000040 MOSDECSubOpRead(1.bs1 58/41 ECSubRead(tid=117, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpRead(1.bs1 58/41 ECSubRead(tid=117, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:35.646+0000 7f8f0317f700 25 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:35.646+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0xac000 (704512) 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 = 4096 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] get_hash_info: found on disk, size 704512 2019-12-07T19:34:35.646+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:34:35.646+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 58/41 ECSubReadReply(tid=117, attrs_read=0)) v2 -- 0x558c5895c000 con 0x558c54c5b400 2019-12-07T19:34:35.646+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c58755340 finish 2019-12-07T19:34:35.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:35.693+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 10 osd.7 58 tick_without_osd_lock 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f43b000/0x40000000/0x1680000000, data 0xd3b472/0xbc3000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 25 osd.7 58 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:32.576231+0000 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 25 osd.7 58 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576218+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 25 osd.7 58 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 25 osd.7 58 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576154+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 25 osd.7 58 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 20 osd.7 58 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 20 osd.7 58 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 20 osd.7 58 scrub_load_below_threshold loadavg per cpu 0.045 < max 5 = yes 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 20 osd.7 58 sched_scrub load_is_low=1 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 10 osd.7 58 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:08.239446+0000 > 2019-12-07T19:34:35.725411+0000 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 20 osd.7 58 sched_scrub done 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 10 osd.7 58 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 20 osd.7 58 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:35.724+0000 7f8f1fbbe700 10 osd.7 58 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:35.743+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.793+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.844+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.894+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.944+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:35.995+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.044+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.094+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.144+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.195+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.245+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.258+0000 7f8f21451700 10 osd.7 58 tick 2019-12-07T19:34:36.258+0000 7f8f21451700 10 osd.7 58 do_waiters -- start 2019-12-07T19:34:36.258+0000 7f8f21451700 10 osd.7 58 do_waiters -- finish 2019-12-07T19:34:36.258+0000 7f8f21451700 20 osd.7 58 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:36.295+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.308+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c5526f400 0x558c54e34580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:36.308+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c5526f400 0x558c54e34580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:36.308+0000 7f8f24457700 10 osd.7 58 ms_handle_authentication new session 0x558c57bfa680 con 0x558c5526f400 entity client.admin addr 2019-12-07T19:34:36.308+0000 7f8f24457700 10 osd.7 58 ms_handle_authentication session 0x558c57bfa680 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:36.308+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2023282104 conn(0x558c5526f400 0x558c54e34580 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4538 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:36.308+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4538 172.21.15.131:0/2023282104 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c57edef00 con 0x558c5526f400 2019-12-07T19:34:36.308+0000 7f8f164bb700 20 osd.7 58 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:36.308+0000 7f8f164bb700 10 osd.7 58 do_waiters -- start 2019-12-07T19:34:36.308+0000 7f8f164bb700 10 osd.7 58 do_waiters -- finish 2019-12-07T19:34:36.308+0000 7f8f164bb700 20 osd.7 58 _dispatch 0x558c57edef00 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:36.308+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2023282104 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce0700 con 0x558c5526f400 2019-12-07T19:34:36.331+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4538 172.21.15.131:0/2023282104 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c575ccd20 con 0x558c5526f400 2019-12-07T19:34:36.331+0000 7f8f164bb700 20 osd.7 58 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:36.331+0000 7f8f164bb700 10 osd.7 58 do_waiters -- start 2019-12-07T19:34:36.331+0000 7f8f164bb700 10 osd.7 58 do_waiters -- finish 2019-12-07T19:34:36.331+0000 7f8f164bb700 20 osd.7 58 _dispatch 0x558c575ccd20 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:36.332+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2023282104 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce16c0 con 0x558c5526f400 2019-12-07T19:34:36.335+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2023282104 conn(0x558c5526f400 msgr2=0x558c54e34580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 72 2019-12-07T19:34:36.335+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2023282104 conn(0x558c5526f400 msgr2=0x558c54e34580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:36.335+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2023282104 conn(0x558c5526f400 0x558c54e34580 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:36.335+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2023282104 conn(0x558c5526f400 0x558c54e34580 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:36.335+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] reap_dead start 2019-12-07T19:34:36.335+0000 7f8f164bb700 2 osd.7 58 ms_handle_reset con 0x558c5526f400 session 0x558c57bfa680 2019-12-07T19:34:36.345+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.395+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.417+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:36.446+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.496+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.518+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:36.546+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.557+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.1 172.21.15.131:0/37078 18 ==== osd_ping(ping e58 up_from 13 ping_stamp 2019-12-07T19:34:36.557596+0000/60.014519984s send_stamp 60.014519984s delta_ub 0.021720869s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551afc00 con 0x558c54c18800 2019-12-07T19:34:36.557+0000 7f8f23c56700 20 osd.7 58 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021264912s,0.021720869s]) 2019-12-07T19:34:36.557+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e58 up_from 13 ping_stamp 2019-12-07T19:34:36.557596+0000/60.014519984s send_stamp 59.993255072s delta_ub 18446744073.688286704s) v5 -- 0x558c576b5400 con 0x558c54c18800 2019-12-07T19:34:36.557+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.1 172.21.15.131:0/37078 18 ==== osd_ping(ping e58 up_from 13 ping_stamp 2019-12-07T19:34:36.557596+0000/60.014519984s send_stamp 60.014519984s delta_ub 0.021720869s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c551afc00 con 0x558c54c19000 2019-12-07T19:34:36.557+0000 7f8f23c56700 20 osd.7 58 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021165243s,0.021720869s]) 2019-12-07T19:34:36.557+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e58 up_from 13 ping_stamp 2019-12-07T19:34:36.557596+0000/60.014519984s send_stamp 59.993354741s delta_ub 18446744073.688386373s) v5 -- 0x558c57612600 con 0x558c54c19000 2019-12-07T19:34:36.596+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1496495 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 312 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (unknown 0 0 0) 0x558c57a52700 con 0x558c54c5b400 2019-12-07T19:34:36.619+0000 7f8f23c56700 15 osd.7 58 enqueue_peering_evt 1.3s4 epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s) 2019-12-07T19:34:36.619+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e59) pg 0x558c5513e000 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq _add_slot_waiter 1.3s4 peering, item epoch is 59, will wait on OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.619+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 313 ==== osd_map(59..59 src has 1..59) v4 ==== 661+0+0 (unknown 0 0 0) 0x558c5852c6c0 con 0x558c54c5b400 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 osd.7 58 OSD::ms_dispatch: osd_map(59..59 src has 1..59) v4 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 osd.7 58 do_waiters -- start 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 osd.7 58 do_waiters -- finish 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 osd.7 58 _dispatch 0x558c5852c6c0 osd_map(59..59 src has 1..59) v4 2019-12-07T19:34:36.619+0000 7f8f1199c700 3 osd.7 58 handle_osd_map epochs [59,59], i have 58, src has [1,59] 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 osd.7 58 handle_osd_map got inc map for epoch 59 2019-12-07T19:34:36.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 314 ==== MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=118, reqid=osd.4.0:118, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 ==== 3458+0+0 (unknown 0 0 0) 0x558c586cf000 con 0x558c54c5b400 2019-12-07T19:34:36.619+0000 7f8f23c56700 15 osd.7 58 enqueue_op 0x558c581cd340 prio 127 cost 0 latency 0.000034 epoch 59 MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=118, reqid=osd.4.0:118, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:36.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3bb4e826:::osdmap.58:0# 0x0~0 2019-12-07T19:34:36.619+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=118, reqid=osd.4.0:118, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e59) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5898b0a0 blob([0x2a7000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c55106000 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574d2b60 blob([!~1000,0x2a8000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558c57f90f50 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3bb4e826:::osdmap.58:0# 0x0~1448 = 5192 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=118, reqid=osd.4.0:118, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e59) queued 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 osd.7 58 add_map_bl 58 5192 bytes 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=118, reqid=osd.4.0:118, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e59) pg 0x558c575ef000 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c581cd340 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=118, reqid=osd.4.0:118, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 pg pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 58 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 58 -> 59 (as per caller) 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=118, reqid=osd.4.0:118, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 snap_mapper.update_snaps 1:d07269f7:::smithi13140796-28:23 1b,20,21 was 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 snap_mapper.get_snaps 1:d07269f7:::smithi13140796-28:23 1b,20,21,23 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 osd.7 58 got_full_map 59, nothing requested 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 snap_mapper.set_snaps 1:d07269f7:::smithi13140796-28:23 1b,20,21 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 snap_mapper.update_snaps rm SNA_1_0000000000000023_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 osd.7 58 handle_osd_map pg_num_history pg_num_history(e59 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {59={}} 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'46], crt=58'46) [59'47 (58'42) modify 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:36.619807+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,59'48 (58'46) modify 1:d07269f7:::smithi13140796-28:head by unknown.0.0:0 2019-12-07T19:34:36.619807+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 osd.7 58 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e59 [1,59] lci=[13,59]) 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c5876a000 seq 50 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 59'47 (0'0,59'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 59'47 (58'42) modify 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:36.619807+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c9a7972f:::inc_osdmap.59:0# key 0x7f7fffffffffffffffc9a797'/!inc_osdmap.59!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 59'48 (58'46) modify 1:d07269f7:::smithi13140796-28:head by unknown.0.0:0 2019-12-07T19:34:36.619807+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c9a7972f:::inc_osdmap.59:0# 0x0~269 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1278 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c9a7972f:::inc_osdmap.59:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 48 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5876a000 1 blobs 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbc5000~1000/1000,1000,0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbc5000~1000] 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f915e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f915e0 blob([0xbc5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91650 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 59'47, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c57f915e0) get_ref 0x0~269 Blob(0x558c57f915e0 blob([0xbc5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91650 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c57f915e0) get_ref init 0x1000, 1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c57f915e0 blob([0xbc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91650 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57f91668 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f288) dirty_range mark inline shard dirty 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c9a7972f:::inc_osdmap.59:0# 0x0~269 = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:36b4e826:::osdmap.59:0# key 0x7f7fffffffffffffff36b4e8'&!osdmap.59!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:36b4e826:::osdmap.59:0# 0x0~1474 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1279 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:36b4e826:::osdmap.59:0# 0x0~1474 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~474 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb8c on front/back, now 0x0~1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5876a000 2 blobs 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbc6000~2000/1000,2000,0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbc6000~2000] 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a4d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a4d90 blob([0xbc6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5030 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c580a4d90) get_ref 0x0~1000 Blob(0x558c580a4d90 blob([0xbc6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5030 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c580a4d90) get_ref init 0x1000, 1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c580a4d90 blob([0xbc6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580a5030 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c580a5048 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a4a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a4a80 blob([!~1000,0xbc7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3110 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c580a4a80) get_ref 0x1000~474 Blob(0x558c580a4a80 blob([!~1000,0xbc7000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c574e3110 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c580a4a80) get_ref init 0x2000, 1000 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a2f00 seq 57 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~474: 0x1000~474 Blob(0x558c580a4a80 blob([!~1000,0xbc7000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,474]) SharedBlob(0x558c574e3110 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c574e3128 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1474 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f508) dirty_range mark inline shard dirty 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:36b4e826:::osdmap.59:0# 0x0~1474 = 0 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# 1 keys 2019-12-07T19:34:36.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# 1 keys = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# hinfo_key (42 bytes) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5876a000 0 blobs 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- SNA_1_0000000000000023_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.619+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000059.00000000000000000047' <- 0000000059.00000000000000000047 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000059.00000000000000000048' <- 0000000059.00000000000000000048 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a2f00 cost 8429 (1 ios * 4000 + 4429 bytes) 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a2f00 onodes 0x558c54e44780,0x558c57f35400 shared_blobs 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 594 (592 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore.extentmap(0x558c57f35508) update 1#1:d07269f7:::smithi13140796-28:23# 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:23# is 1567 (413 bytes onode + 2 bytes spanning blobs + 1152 bytes inline extents) 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a2f00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2f00 prepare 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a2f00 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2f00 io_done 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 59'48 2019-12-07T19:34:36.619+0000 7f8f0317f700 10 osd.7 58 dequeue_op 0x558c581cd340 finish 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5036 < min_flush_size 524288 2019-12-07T19:34:36.619+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35400 had 1 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5059 < min_flush_size 524288 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5059 < min_flush_size 524288 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x469420~13c3 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5876a000 0 blobs 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x469420 0x469420~13c3 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x439420 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x420 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7e3 and padding block with 0x81d 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5876a000 cost 23334 (4 ios * 4000 + 7334 bytes) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5876a000 onodes 0x558c53a91900,0x558c55276c80,0x558c5851f180,0x558c5851f400 shared_blobs 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f288) update #-1:c9a7972f:::inc_osdmap.59:0# 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c9a7972f:::inc_osdmap.59:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f508) update #-1:36b4e826:::osdmap.59:0# 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f508) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:36b4e826:::osdmap.59:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5876a000 allocated 0x[bc5000~3000] released 0x[] 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 freelist allocate 0xbc5000~3000 2019-12-07T19:34:36.619+0000 7f8f1199c700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46a7e3 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876a000 prepare 2019-12-07T19:34:36.619+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5876a000 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000126189s (0.000000177s flush + 0.000126012s kv commit) 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586a2f00> 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2f00 kv_submitted 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a2f00 2019-12-07T19:34:36.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a2f00 finishing 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a2f00 onodes 0x558c54e44780,0x558c57f35400 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a2f00 done 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:36.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a2f00 [] 2019-12-07T19:34:36.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.619+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 58/41 ECSubWriteReply(tid=118, last_complete=59'48, committed=1, applied=1)) v2 -- 0x558c54c58a00 con 0x558c54c5b400 2019-12-07T19:34:36.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.619+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876a000 aio_wait 2019-12-07T19:34:36.619+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5876a000 2019-12-07T19:34:36.619+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876a000 io_done 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:36.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851f180 had 1 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851f400 had 1 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46a7e3~226c to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46a7e3 0x46a7e3~226c to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x43a7e3 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7e3 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa4f and padding block with 0x5b1 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46ca4f 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000190914s (0.000035571s flush + 0.000155343s kv commit) 2019-12-07T19:34:36.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5876a000> 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876a000 kv_submitted 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5876a000 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c5876a000 osr 0x558c54ea4b40 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 47 0x1b000~1000 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 48 0x1b000~1000 crc dc082dc3 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7 58 _committed_osd_maps 59..59 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 47 0x2000~1000 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7 58 advance to epoch 59 (<= last 59 <= newest_map 59) 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 48 0x2000~1000 crc ce9827e9 2019-12-07T19:34:36.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.620+0000 7f8f141a1700 7 osd.7 59 consume_map version 59 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.8s5 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.0s3 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.1s1 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.9s4 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.as5 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.2s0 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.bs1 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.3s4 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.cs0 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.4s0 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.5s4 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.ds2 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.es4 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.6s2 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.fs5 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 identify_splits_and_merges 1.7s4 e58 to e59 pg_nums {18=16} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 59 prune_pg_created 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:0.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:1.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:2.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:3.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 pending_peering first epoch 59 <= 59, requeueing 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:3._wake_pg_slot _wake_pg_slot 1.3s4 to_process <> waiting <> waiting_peering {59=} 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:4.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:5.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:6.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7:7.consume_map 59 (was 58) 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e59) pg 0x558c5513e000 2019-12-07T19:34:36.620+0000 7f8f141a1700 10 osd.7 59 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.8s5 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.0s3 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.1s1 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.9s4 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.as5 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c57380000 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.2s0 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 58 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.bs1 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c5736a000 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.3s4 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.cs0 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.4s0 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.5s4 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.ds2 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.es4 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 58 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c57676000 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.6s2 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c575ef000 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c54d98000 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c57e8b000 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.620+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean ps=[e~1,15~1,1a~1]] state: Active advmap 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] on_active_advmap new removed_snaps [23~1], snap_trimq now [23~1] 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 58 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 58 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c5500d000 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 58 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 58 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:36.620+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.620+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:36.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c5500a000 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.620+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 58 while current is 59 2019-12-07T19:34:36.620+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 58 while current is 59 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:36.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c57522000 2019-12-07T19:34:36.620+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.620+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:36.620+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:36.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.620+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:36.620+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.620+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.620+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 58 -> 59 2019-12-07T19:34:36.620+0000 7f8f06185700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.621+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.fs5 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 58 -> 59 2019-12-07T19:34:36.621+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c57e88000 2019-12-07T19:34:36.620+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] exit NotTrimming 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 58 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:36.621+0000 7f8f04181700 10 osd.7 pg_epoch: 58 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:36.621+0000 7f8f141a1700 15 osd.7 59 enqueue_peering_evt 1.7s4 epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.621+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c57383000 2019-12-07T19:34:36.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.621+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.621+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.621+0000 7f8f141a1700 7 osd.7 59 activate_map version 59 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 58 -> 59 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 58 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.621+0000 7f8f0297e700 15 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 58:196 2019-12-07T19:34:36.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1]] state: Active advmap 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c54d5f000 2019-12-07T19:34:36.621+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f04181700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] on_active_advmap new removed_snaps [23~1], snap_trimq now [23~1] 2019-12-07T19:34:36.621+0000 7f8f0297e700 7 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 58 -> 59 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.621+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c55141000 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 58 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.621+0000 7f8f0297e700 15 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 58:197 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 58 -> 59 2019-12-07T19:34:36.621+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:36.621+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 58 while current is 59 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.621+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 58 -> 59 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 58 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 58 while current is 59 2019-12-07T19:34:36.621+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.621+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 58 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.621+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.621+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c548ab000 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 58 -> 59 2019-12-07T19:34:36.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 58 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:36.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] exit NotTrimming 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1]] state: Active advmap 2019-12-07T19:34:36.621+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:36.621+0000 7f8f0918b700 20 osd.7 59 agent_entry empty queue 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] on_active_advmap new removed_snaps [23~1], snap_trimq now [23~1] 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming: trimming 23 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:36.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:198 2019-12-07T19:34:36.621+0000 7f8f03980700 15 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:167 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:36.621+0000 7f8f03980700 7 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 59 queueing pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] for snaptrim 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=59) prio 5 cost 1048576 e59) 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.621+0000 7f8f03980700 15 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 58:168 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=59) prio 5 cost 1048576 e59) queued 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.621+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:199 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=59) prio 5 cost 1048576 e59) pg 0x558c57676000 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:36.620+0000 7f8f0317f700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] state: Active: handling ActMap 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] Active: kicking snap trim 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming: trimming 23 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 23 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] exit NotTrimming 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s) 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming 2019-12-07T19:34:36.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:169 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/WaitReservation 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 59 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] for snaptrim 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57bccb00) upper_bound after SNA_1_0000000000000023_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C' 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=59) prio 5 cost 1048576 e59) 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c586a3200 seq 69 2019-12-07T19:34:36.621+0000 7f8f0297e700 15 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 57:155 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57bccb00) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:36.621+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f0297e700 7 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] state: handle ActMap primary 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=59) prio 5 cost 1048576 e59) queued 2019-12-07T19:34:36.621+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000023_.0_0000000000000001.C) returns 0 SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:36.621+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:36.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:170 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:36.621+0000 7f8f0297e700 15 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 58:156 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a3200 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a3200 onodes shared_blobs 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=59) prio 5 cost 1048576 e59) pg 0x558c57e88000 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a3200 allocated 0x[] released 0x[] 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57bccb00) upper_bound after SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. key 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3200 prepare 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 23 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c57bccb00) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..' 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a3200 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3200 io_done 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.621+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..) returns 0 SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239810 in raw 0x558c55239810 len 69 nref 1) ) 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c5513e000 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:36.621+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.621+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57bccb80) upper_bound after SNA_1_0000000000000023_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.2' 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_object_context: found obc in cache: 0x558c585278c0 2019-12-07T19:34:36.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 503 ==== pg_lease(1.0s3 pg_lease(ru 68.076978842s ub 76.078109650s int 16s) e58/58) v1 ==== 58+0+0 (crc 0 0 0) 0x558c548cc200 con 0x558c54c64000 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57bccb80) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:36.621+0000 7f8f24457700 15 osd.7 59 enqueue_peering_evt 1.0s3 epoch_sent: 58 epoch_requested: 58 MLease epoch 58 from osd.1 pg_lease(ru 68.076978842s ub 76.078109650s int 16s) 2019-12-07T19:34:36.621+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 MLease epoch 58 from osd.1 pg_lease(ru 68.076978842s ub 76.078109650s int 16s)) prio 255 cost 10 e58) 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_object_context: 0x558c585278c0 1:3a5a3f5b:::smithi13140796-23:23 rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:23(58'43 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:36.621+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000023_.0_0000000000000001.2) returns 0 SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c552382c0 in raw 0x558c552382c0 len 70 nref 1) ) 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 MLease epoch 58 from osd.1 pg_lease(ru 68.076978842s ub 76.078109650s int 16s)) prio 255 cost 10 e58) queued 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 58 epoch_requested: 58 MLease epoch 58 from osd.1 pg_lease(ru 68.076978842s ub 76.078109650s int 16s)) prio 255 cost 10 e58) pg 0x558c5500d000 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1]] SnapTrimmer state: adding snap 23 to purged_snaps 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(58'46 client.4540.0:443 dirty|data_digest s 507904 uv 46 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:36.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 58 epoch_requested: 58 MLease epoch 58 from osd.1 pg_lease(ru 68.076978842s ub 76.078109650s int 16s) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.621+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 1:3a5a3f5b:::smithi13140796-23:23 old_snaps 1b,20,21,23 old snapset 23=[]:{23=[23,21,20,1b]} 2019-12-07T19:34:36.621+0000 7f8f0898a700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease pg_lease(ru 68.076978842s ub 76.078109650s int 16s) 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_ub 0.021720869s -> ru 68.055257973s 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: purged_snaps now [e~1,15~1,1a~1,23~1], snap_trimq now [] 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] simple_opc_create 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:36.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c58111500 seq 31 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until now 68.055257973s 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] peer_clock_delta_lb 0.021165243s -> ruub 76.056944407s 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 1:3a5a3f5b:::smithi13140796-23:23 snaps 1b,20,21,23 -> 1b,20,21 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46ca4f~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] proc_lease readable_until_ub now 76.056944407s 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46ca4f 0x46ca4f~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x43ca4f 2019-12-07T19:34:36.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa4f 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58111500 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xef4 and padding block with 0x10c 2019-12-07T19:34:36.621+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.0s0 pg_lease_ack(ruub 76.078109650s) e59/59) v1 -- 0x558c581e9860 con 0x558c54c64000 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58111500 onodes shared_blobs 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58111500 allocated 0x[] released 0x[] 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 1:3a5a3f5b:::smithi13140796-23:23 new snapset 23=[]:{23=[21,20,1b]} on 1:3a5a3f5b:::smithi13140796-23:head(58'46 client.4540.0:443 dirty|data_digest s 507904 uv 46 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:36.621+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111500 prepare 2019-12-07T19:34:36.621+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58111500 2019-12-07T19:34:36.621+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111500 io_done 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 1:3a5a3f5b:::smithi13140796-23:23 writing updated snapset on 1:3a5a3f5b:::smithi13140796-23:head, snapset is 23=[]:{23=[21,20,1b]} 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] share_pg_info 2019-12-07T19:34:36.621+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c574e7800 con 0x558c54c64000 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] new_repop rep_tid 53 (no op) 2019-12-07T19:34:36.621+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c576b8e00 con 0x558c54c19800 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] new_repop: repgather(0x558c576b70e0 0'0 rep_tid=53 committed?=0 r=0) 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] simple_opc_submit 0x558c576b70e0 2019-12-07T19:34:36.621+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c574e5c00 con 0x558c54c5bc00 2019-12-07T19:34:36.621+0000 7f8f06986700 7 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] issue_repop rep_tid 53 o 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:36.621+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c574e5500 con 0x558c54c5b400 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:23 v=59'48 tt=0'0 tid=53 reqid=osd.7.0:53 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:36.621+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c57390e00 con 0x558c54c19c00 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46cef4 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] exit Trimming 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131671s (0.000000083s flush + 0.000131588s kv commit) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586a3200> 2019-12-07T19:34:36.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3200 kv_submitted 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a3200 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key = 0 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3200 finishing 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a3200 onodes 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a3200 done 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: NotTrimming: trimming 23 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:36.621+0000 7f8f07988700 15 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 59:171 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:23 projected size 507904 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[23~1] ps=[e~1,15~1,1a~1]] exit Trimming/WaitReservation 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a3200 [] 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] enter NotTrimming 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46cef4~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46cef4 0x46cef4~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:36.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:157 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x43cef4 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xef4 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x399 and padding block with 0xc67 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:36.621+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] snap_trimmer complete 2019-12-07T19:34:36.621+0000 7f8f0b990700 10 osd.7 59 queueing pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[23~1] ps=[e~1,15~1,1a~1]] for snaptrim 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=59) prio 5 cost 1048576 e59) 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:36.621+0000 7f8f0b990700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:36.621+0000 7f8f0b990700 15 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:158 2019-12-07T19:34:36.621+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:23 v=59'48 tt=0'0 tid=53 reqid=osd.7.0:53 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:23 v=59'48 tt=0'0 tid=53 reqid=osd.7.0:53 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:23 v=59'48 tt=0'0 tid=53 reqid=osd.7.0:53 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46d399 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:23, current size is 507904 buffers are {} 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131895s (0.000000083s flush + 0.000131812s kv commit) 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58111500> 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111500 kv_submitted 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58111500 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111500 finishing 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58111500 onodes 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58111500 done 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:36.621+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58111500 [] 2019-12-07T19:34:36.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): new_size start 507904 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=59) prio 5 cost 1048576 e59) queued 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=59) prio 5 cost 1048576 e59) pg 0x558c548ab000 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:23 resetting hinfo to logical size 507904 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 23 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): new_size start 507904 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): to_overwrite: {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c5730ee00) upper_bound after SNA_1_0000000000000023_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.4' 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): to_append: {} 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c5730ee00) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000023_.0_0000000000000001.4) returns 0 SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:23={},1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1]] SnapTrimmer state: adding snap 23 to purged_snaps 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:23 v=59'48 tt=0'0 tid=53 reqid=osd.7.0:53 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: purged_snaps now [e~1,15~1,1a~1,23~1], snap_trimq now [] 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:23,{} 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c57bd3800 seq 48 2019-12-07T19:34:36.621+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:34:36.621+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd3800 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd3800 onodes shared_blobs 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd3800 allocated 0x[] released 0x[] 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3800 prepare 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd3800 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3800 io_done 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] share_pg_info 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 send_incremental_map 58 -> 59 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:36.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c57a55800 con 0x558c54c64000 2019-12-07T19:34:36.621+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c9a7972f:::inc_osdmap.59:0# 0x0~0 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:36.621+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f915e0 blob([0xbc5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c57f91650 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:36.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c57ed3c00 con 0x558c54c19800 2019-12-07T19:34:36.621+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c9a7972f:::inc_osdmap.59:0# 0x0~269 = 617 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 add_map_inc_bl 59 617 bytes 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(59..59 src has 1..59) v4 -- 0x558c55275440 con 0x558c54c64000 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 58 -> 59 (shared) 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46d399~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:36.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46d399 0x46d399~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 59/28 ECSubWrite(tid=53, reqid=osd.7.0:53, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c58156000 con 0x558c54c64000 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x43d399 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x399 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x83e and padding block with 0x7c2 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 send_incremental_map 58 -> 59 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:36.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c58228000 con 0x558c54c5b400 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(59..59 src has 1..59) v4 -- 0x558c5822c000 con 0x558c54c19800 2019-12-07T19:34:36.621+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 -- 0x558c58228700 con 0x558c54c19c00 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 58 -> 59 (shared) 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 59/28 ECSubWrite(tid=53, reqid=osd.7.0:53, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c58156800 con 0x558c54c19800 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 send_incremental_map 58 -> 59 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] exit Trimming 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(59..59 src has 1..59) v4 -- 0x558c5822c240 con 0x558c54c5bc00 2019-12-07T19:34:36.621+0000 7f8f0297e700 15 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 59:159 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 58 -> 59 (shared) 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 59/28 ECSubWrite(tid=53, reqid=osd.7.0:53, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c579e1800 con 0x558c54c5bc00 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] enter NotTrimming 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 59/28 ECSubWrite(tid=53, reqid=osd.7.0:53, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c579e1000 con 0x558c54c5b400 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:36.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] snap_trimmer complete 2019-12-07T19:34:36.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 send_incremental_map 58 -> 59 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:36.621+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(59..59 src has 1..59) v4 -- 0x558c5822c480 con 0x558c54c19c00 2019-12-07T19:34:36.621+0000 7f8f06986700 10 osd.7 59 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 58 -> 59 (shared) 2019-12-07T19:34:36.622+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 59/28 ECSubWrite(tid=53, reqid=osd.7.0:53, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 -- 0x558c579e2000 con 0x558c54c19c00 2019-12-07T19:34:36.622+0000 7f8f06986700 20 snap_mapper.update_snaps 1:3a5a3f5b:::smithi13140796-23:23 1b,20,21 was 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46d83e 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000200476s (0.000000131s flush + 0.000200345s kv commit) 2019-12-07T19:34:36.622+0000 7f8f06986700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:23 1b,20,21,23 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.622+0000 7f8f06986700 20 snap_mapper.set_snaps 1:3a5a3f5b:::smithi13140796-23:23 1b,20,21 2019-12-07T19:34:36.622+0000 7f8f06986700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:36.622+0000 7f8f06986700 20 snap_mapper.update_snaps rm SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:36.622+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:36.622+0000 7f8f06986700 20 trim 59'47 (58'43) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.622+0000 7f8f06986700 20 trim 59'48 (58'46) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 58'46 (0'0,58'46] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] append_log log((0'0,58'46], crt=58'46) [59'47 (58'43) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,59'48 (58'46) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'47 (0'0,59'47] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 lua=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] add_log_entry 59'47 (58'43) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57bd3800> 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3800 kv_submitted 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd3800 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 lua=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] add_log_entry 59'48 (58'46) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3800 finishing 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd3800 onodes 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd3800 done 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd3800 [] 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 lua=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] append_log approx pg log length = 48 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 lua=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.622+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.622+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 59'47, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 NullEvt) prio 255 cost 10 e59) pg 0x558c54d5c000 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58748600 seq 65 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 1 keys 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map: 59 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 1 keys = 0 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 58 -> 59 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key (42 bytes) 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 58 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] state: Started advmap 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:36.622+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] plpg_on_pool_change 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- SNA_1_0000000000000023_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] hit_set_clear 2019-12-07T19:34:36.622+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000059.00000000000000000047' <- 0000000059.00000000000000000047 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] agent_stop 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000059.00000000000000000048' <- 0000000059.00000000000000000048 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 59 advance_pg new pool opts old pool opts 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] handle_activate_map: 59 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58748600 cost 8447 (1 ios * 4000 + 4447 bytes) 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] activate_map 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58748600 onodes 0x558c575a3b80,0x558c57bd4500 shared_blobs 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore.extentmap(0x558c575a3c88) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] activate_map: Not dirtying info: last_persisted is 57 while current is 59 2019-12-07T19:34:36.622+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1011 (573 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4608) update 0#1:3a5a3f5b:::smithi13140796-23:23# 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:23# is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58748600 allocated 0x[] released 0x[] 2019-12-07T19:34:36.622+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 NullEvt 2019-12-07T19:34:36.622+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:36.622+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748600 prepare 2019-12-07T19:34:36.622+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58748600 2019-12-07T19:34:36.622+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748600 io_done 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 lua=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] op_applied version 59'48 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4760 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b70e0 59'48 rep_tid=53 committed?=0 r=0) (no op) 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] calc_trim_to_aggressive limit = 58'46 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a3b80 had 1 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd4500 had 1 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4783 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4783 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46d83e~12af to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/WaitRepops 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46d83e 0x46d83e~12af to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x43d83e 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x83e 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xaed and padding block with 0x513 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] snap_trimmer complete 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46eaed 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000139625s (0.000000116s flush + 0.000139509s kv commit) 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 205 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58537500 con 0x558c54c19800 2019-12-07T19:34:36.622+0000 7f8f24c58700 15 osd.7 59 enqueue_peering_evt 1.ds2 epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s) 2019-12-07T19:34:36.622+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58748600> 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748600 kv_submitted 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58748600 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748600 finishing 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58748600 onodes 0x558c575a3b80,0x558c57bd4500 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58748600 done 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58748600 [] 2019-12-07T19:34:36.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e59) pg 0x558c55141000 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s) 2019-12-07T19:34:36.622+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 206 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c57ed3c00 con 0x558c54c19800 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.622+0000 7f8f24c58700 15 osd.7 59 enqueue_peering_evt 1.5s4 epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s) 2019-12-07T19:34:36.622+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:36.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58756300 seq 67 2019-12-07T19:34:36.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e59) pg 0x558c57522000 2019-12-07T19:34:36.622+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:36.622+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s) 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58756300 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.622+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58756300 onodes shared_blobs 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58756300 allocated 0x[] released 0x[] 2019-12-07T19:34:36.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 291 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58228000 con 0x558c54c5bc00 2019-12-07T19:34:36.622+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756300 prepare 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58756300 2019-12-07T19:34:36.622+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756300 io_done 2019-12-07T19:34:36.622+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:36.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c5853af00 seq 24 2019-12-07T19:34:36.622+0000 7f8f23c56700 15 osd.7 59 enqueue_peering_evt 1.fs5 epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s) 2019-12-07T19:34:36.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.622+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:36.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:36.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:36.622+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:36.622+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853af00 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853af00 onodes shared_blobs 2019-12-07T19:34:36.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853af00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.622+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853af00 prepare 2019-12-07T19:34:36.622+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853af00 2019-12-07T19:34:36.622+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853af00 io_done 2019-12-07T19:34:36.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 315 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54c58a00 con 0x558c54c5b400 2019-12-07T19:34:36.622+0000 7f8f23c56700 15 osd.7 59 enqueue_op 0x558c581ccdc0 prio 196 cost 0 latency 0.000009 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.622+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e59) pg 0x558c57383000 2019-12-07T19:34:36.622+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s) 2019-12-07T19:34:36.622+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.622+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:36.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:36.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c583b6c00 seq 32 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:36.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:36.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b6c00 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b6c00 onodes shared_blobs 2019-12-07T19:34:36.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b6c00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6c00 prepare 2019-12-07T19:34:36.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b6c00 2019-12-07T19:34:36.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6c00 io_done 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c581ccdc0 prio 196 cost 0 latency 0.000077 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.622+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c581ccdc0 finish 2019-12-07T19:34:36.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2332 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3498 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3521 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3521 < min_flush_size 524288 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 115 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c548cc200 con 0x558c54c19c00 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46eaed~dc1 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c5511f600 prio 196 cost 0 latency 0.000014 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46eaed 0x46eaed~dc1 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x43eaed 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xaed 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x8ae and padding block with 0x752 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.622+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c5511f600 prio 196 cost 0 latency 0.000056 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.622+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c5511f600 finish 2019-12-07T19:34:36.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46f8ae 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000151995s (0.000000127s flush + 0.000151868s kv commit) 2019-12-07T19:34:36.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58756300,0x558c5853af00,0x558c583b6c00> 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756300 kv_submitted 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58756300 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756300 finishing 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58756300 onodes 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58756300 done 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58756300 [] 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853af00 kv_submitted 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853af00 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853af00 finishing 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853af00 onodes 2019-12-07T19:34:36.623+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853af00 done 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853af00 [] 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6c00 kv_submitted 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b6c00 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6c00 finishing 2019-12-07T19:34:36.623+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b6c00 onodes 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b6c00 done 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b6c00 [] 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.623+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 292 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:36.623+0000 7f8f23c56700 15 osd.7 59 enqueue_peering_evt 1.6s2 epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:36.623+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e59) pg 0x558c54d5c000 2019-12-07T19:34:36.623+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 293 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c5737aa00 con 0x558c54c5bc00 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:36.623+0000 7f8f23c56700 15 osd.7 59 enqueue_peering_evt 1.1s1 epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s) 2019-12-07T19:34:36.623+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:36.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:36.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5853f200 seq 110 2019-12-07T19:34:36.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e59) pg 0x558c5736a000 2019-12-07T19:34:36.623+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:36.623+0000 7f8f08189700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s) 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853f200 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853f200 onodes shared_blobs 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853f200 allocated 0x[] released 0x[] 2019-12-07T19:34:36.623+0000 7f8f08189700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f200 prepare 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853f200 2019-12-07T19:34:36.623+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f200 io_done 2019-12-07T19:34:36.623+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:36.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c578aa900 seq 25 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.623+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:36.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:36.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:36.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:36.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578aa900 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578aa900 onodes shared_blobs 2019-12-07T19:34:36.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578aa900 allocated 0x[] released 0x[] 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46f8ae~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa900 prepare 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46f8ae 0x46f8ae~4a5 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.623+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578aa900 2019-12-07T19:34:36.623+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa900 io_done 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~440000 x_off 0x43f8ae 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x8ae 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.623+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd53 and padding block with 0x2ad 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x46fd53 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000113489s (0.000000117s flush + 0.000113372s kv commit) 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853f200> 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 685 < min_flush_size 524288 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f200 kv_submitted 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853f200 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1177 < min_flush_size 524288 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f200 finishing 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853f200 onodes 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853f200 done 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853f200 [] 2019-12-07T19:34:36.623+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1200 < min_flush_size 524288 2019-12-07T19:34:36.623+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1200 < min_flush_size 524288 2019-12-07T19:34:36.623+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.623+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x46fd53~4b0 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x46fd53 0x46fd53~4b0 to file(ino 16 size 0x470000 mtime 2019-12-07T19:34:35.642539+0000 allocated 470000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~440000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _allocate len 0x203 from 0 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _allocate len 0x203 from 1 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd5a0000 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd5a0000~10000/10000,10000,acd5a0000 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x480000 to include allocated 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 74 (was clean) 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~450000 x_off 0x43fd53 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd53 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x203 and padding block with 0xdfd 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x480000 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x470203 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.623+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 207 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c58400 con 0x558c54c19800 2019-12-07T19:34:36.623+0000 7f8f24c58700 15 osd.7 59 enqueue_op 0x558c57335b80 prio 196 cost 0 latency 0.000014 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.623+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (74) on file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]), flushing log 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 74 len 0x4a crc 0x5eef1591) 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x49000~1000 to file(ino 1 size 0x49000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x49000 0x49000~1000 to file(ino 1 size 0x49000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x4a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:36.623+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c57335b80 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.623+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x49000 2019-12-07T19:34:36.623+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 504 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f600 con 0x558c54c64000 2019-12-07T19:34:36.623+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c57334f20 prio 196 cost 0 latency 0.000011 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.623+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.623+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.623+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c57335b80 finish 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.623+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c57334f20 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x4a000 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c57334f20 finish 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000021 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 74 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000379415s (0.000000061s flush + 0.000379354s kv commit) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c578aa900> 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa900 kv_submitted 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578aa900 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa900 finishing 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578aa900 onodes 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578aa900 done 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578aa900 [] 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 316 ==== MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=119, reqid=osd.4.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c579e1800 con 0x558c54c5b400 2019-12-07T19:34:36.624+0000 7f8f23c56700 15 osd.7 59 enqueue_op 0x558c581cc840 prio 127 cost 0 latency 0.000018 epoch 59 MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=119, reqid=osd.4.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:36.624+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=119, reqid=osd.4.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e59) 2019-12-07T19:34:36.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 294 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57612000 con 0x558c54c5bc00 2019-12-07T19:34:36.624+0000 7f8f23c56700 15 osd.7 59 enqueue_op 0x558c581cc9a0 prio 196 cost 0 latency 0.000005 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.624+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=119, reqid=osd.4.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e59) queued 2019-12-07T19:34:36.624+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=119, reqid=osd.4.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e59) pg 0x558c575ef000 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 osd.7 59 dequeue_op 0x558c581cc840 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=119, reqid=osd.4.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 pg pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.bs1 59/41 ECSubWrite(tid=119, reqid=osd.4.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c581cc9a0 prio 196 cost 0 latency 0.000053 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,59'48], crt=58'46) [] 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=53, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=59'47 (58'42) modify 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:36.619807+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:23 v=59'48 tt=0'0 tid=53 reqid=osd.7.0:53 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=58'46 crt=58'46 lcod 58'45 mlcod 58'45 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] repop_all_committed: repop tid 53 all committed 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] eval_repop repgather(0x558c576b70e0 59'48 rep_tid=53 committed?=1 r=0) (no op) 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] commit: repgather(0x558c576b70e0 59'48 rep_tid=53 committed?=1 r=0) 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=59'48 (58'46) modify 1:d07269f7:::smithi13140796-28:head by unknown.0.0:0 2019-12-07T19:34:36.619807+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 48 2019-12-07T19:34:36.624+0000 7f8f0297e700 15 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:200 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] removing repgather(0x558c576b70e0 59'48 rep_tid=53 committed?=1 r=0) 2019-12-07T19:34:36.624+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] q front is repgather(0x558c576b70e0 59'48 rep_tid=53 committed?=1 r=0) 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] exit Trimming/WaitRepops 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a3500 seq 58 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:36.624+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 59 queueing pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] for snaptrim 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=59) prio 5 cost 1048576 e59) 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1] 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a3500 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=59) prio 5 cost 1048576 e59) queued 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a3500 onodes shared_blobs 2019-12-07T19:34:36.624+0000 7f8f0297e700 15 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] publish_stats_to_osd 59:201 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a3500 allocated 0x[] released 0x[] 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3500 prepare 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] remove_repop repgather(0x558c576b70e0 59'48 rep_tid=53 committed?=1 r=0) 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a3500 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3500 io_done 2019-12-07T19:34:36.624+0000 7f8f0317f700 10 osd.7 59 dequeue_op 0x558c581cc840 finish 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:23 v=59'48 tt=0'0 tid=53 reqid=osd.7.0:53 roll_forward_to=58'46 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1138 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:23 v=0'0 tt=0'0 tid=54 reqid=osd.7.0:53 roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1161 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1161 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:36.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 317 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (unknown 0 0 0) 0x558c5737aa00 con 0x558c54c5b400 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x470203~489 to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x470203 0x470203~489 to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:23 v=0'0 tt=0'0 tid=54 reqid=osd.7.0:53 roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:36.624+0000 7f8f23c56700 15 osd.7 59 enqueue_peering_evt 1.bs1 epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~450000 x_off 0x440203 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x203 2019-12-07T19:34:36.624+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x68c and padding block with 0x974 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:36.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 59/28 ECSubWrite(tid=54, reqid=osd.7.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c586d0000 con 0x558c54c64000 2019-12-07T19:34:36.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e59) pg 0x558c575ef000 2019-12-07T19:34:36.624+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s) 2019-12-07T19:34:36.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 59/28 ECSubWrite(tid=54, reqid=osd.7.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c586d0800 con 0x558c54c19800 2019-12-07T19:34:36.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 59/28 ECSubWrite(tid=54, reqid=osd.7.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c586d1000 con 0x558c54c5bc00 2019-12-07T19:34:36.624+0000 7f8f07187700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.624+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:36.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 59/28 ECSubWrite(tid=54, reqid=osd.7.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c586d1800 con 0x558c54c5b400 2019-12-07T19:34:36.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:36.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586e1800 seq 59 2019-12-07T19:34:36.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 59/28 ECSubWrite(tid=54, reqid=osd.7.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c5822e000 con 0x558c54c19c00 2019-12-07T19:34:36.624+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:36.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:36.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:36.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=58'46 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] append_log log((0'0,59'48], crt=58'46) [] 2019-12-07T19:34:36.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e1800 cost 5305 (1 ios * 4000 + 1305 bytes) 2019-12-07T19:34:36.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e1800 onodes shared_blobs 2019-12-07T19:34:36.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e1800 allocated 0x[] released 0x[] 2019-12-07T19:34:36.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1800 prepare 2019-12-07T19:34:36.624+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e1800 2019-12-07T19:34:36.624+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1800 io_done 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] rollforward: entry=59'47 (58'43) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.624+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x47068c 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 505 ==== MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=296, reqid=osd.1.0:296, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 ==== 3545+0+0 (crc 0 0 0) 0x558c58156000 con 0x558c54c64000 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] rollforward: entry=59'48 (58'46) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:36.622603+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.624+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c57335e40 prio 127 cost 0 latency 0.000024 epoch 59 MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=296, reqid=osd.1.0:296, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:36.624+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=296, reqid=osd.1.0:296, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e59) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000178302s (0.000000100s flush + 0.000178202s kv commit) 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] append_log approx pg log length = 48 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1142 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.624+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x47068c~48d to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x47068c 0x47068c~48d to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~450000 x_off 0x44068c 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x68c 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb19 and padding block with 0x4e7 2019-12-07T19:34:36.624+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=296, reqid=osd.1.0:296, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e59) queued 2019-12-07T19:34:36.624+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=296, reqid=osd.1.0:296, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2) prio 127 cost 0 e59) pg 0x558c5500a000 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c57bd3b00 seq 66 2019-12-07T19:34:36.624+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:36.624+0000 7f8f04181700 10 osd.7 59 dequeue_op 0x558c57335e40 prio 127 cost 0 latency 0.000079 MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=296, reqid=osd.1.0:296, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 pg pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586a3500> 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3500 kv_submitted 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a3500 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3500 finishing 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a3500 onodes 2019-12-07T19:34:36.624+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=296, reqid=osd.1.0:296, at_version=59'48, trim_to=0'0, roll_forward_to=58'46)) v2 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a3500 done 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e1800 kv_submitted 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a3500 [] 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.624+0000 7f8f04181700 20 snap_mapper.update_snaps 1:9b8adabe:::smithi13140796-6:23 21 was 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57bd3b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57bd3b00 onodes shared_blobs 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57bd3b00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3b00 prepare 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57bd3b00 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3b00 io_done 2019-12-07T19:34:36.624+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c581cc9a0 finish 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f04181700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:23 21,23 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=59) prio 5 cost 1048576 e59) pg 0x558c57676000 2019-12-07T19:34:36.624+0000 7f8f04181700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:23 21 2019-12-07T19:34:36.624+0000 7f8f04181700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:36.624+0000 7f8f04181700 20 snap_mapper.update_snaps rm SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] snap_trimmer posting 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x470b19 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 23 2019-12-07T19:34:36.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123985s (0.000000066s flush + 0.000123919s kv commit) 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586e1800> 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.624+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 58'46 (0'0,58'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'46], crt=58'46) [59'47 (58'45) modify 1:9b8adabe:::smithi13140796-6:23 by unknown.0.0:0 2019-12-07T19:34:36.625143+0000 0 snaps [21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,59'48 (58'46) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:36.625143+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1800 kv_submitted 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e1800 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1800 finishing 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e1800 onodes 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e1800 done 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e1800 [] 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'47 (0'0,59'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 59'47 (58'45) modify 1:9b8adabe:::smithi13140796-6:23 by unknown.0.0:0 2019-12-07T19:34:36.625143+0000 0 snaps [21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x470b19~471 to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x470b19 0x470b19~471 to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~450000 x_off 0x440b19 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5517b080) upper_bound after SNA_1_0000000000000023_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.0_0000000000000001.C' 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb19 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf8a and padding block with 0x76 2019-12-07T19:34:36.624+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 59'48 (58'46) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:36.625143+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5517b080) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..' 2019-12-07T19:34:36.624+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 48 2019-12-07T19:34:36.624+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000023_.0_0000000000000001.C) returns 0 SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239810 in raw 0x558c55239810 len 69 nref 1) ) 2019-12-07T19:34:36.624+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.624+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim trimq=[23~1] ps=[e~1,15~1,1a~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:36.624+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 59'47, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim ps=[e~1,15~1,1a~1]] SnapTrimmer state: adding snap 23 to purged_snaps 2019-12-07T19:34:36.624+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: purged_snaps now [e~1,15~1,1a~1,23~1], snap_trimq now [] 2019-12-07T19:34:36.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58748c00 seq 67 2019-12-07T19:34:36.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:36.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:36.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58748c00 cost 5305 (1 ios * 4000 + 1305 bytes) 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58748c00 onodes shared_blobs 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58748c00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x470f8a 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748c00 prepare 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58748c00 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:36.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748c00 io_done 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c57957500 seq 88 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000114230s (0.000000063s flush + 0.000114167s kv commit) 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# 1 keys 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] share_pg_info 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# 1 keys = 0 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# hinfo_key (42 bytes) 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1142 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e59/59) v1 -- 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x470f8a~48d to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e59/59) v1 -- 0x558c58137800 con 0x558c54c19800 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x470f8a 0x470f8a~48d to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~450000 x_off 0x440f8a 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf8a 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x417 and padding block with 0xbe9 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e59/59) v1 -- 0x558c54e9b800 con 0x558c54c5bc00 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6..' <- SNA_1_0000000000000023_.4_0000000000000001.9D15B5D7.23.smithi13140796-6.. 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.624+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000059.00000000000000000047' <- 0000000059.00000000000000000047 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000059.00000000000000000048' <- 0000000059.00000000000000000048 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:36.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e59/59) v1 -- 0x558c58236000 con 0x558c54c5b400 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:36.624+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57957500 cost 8516 (1 ios * 4000 + 4516 bytes) 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57957500 onodes 0x558c57a35400,0x558c58768000 shared_blobs 2019-12-07T19:34:36.624+0000 7f8f04181700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:36.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e59/59) v1 -- 0x558c58236700 con 0x558c54c19c00 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57bd3b00> 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3b00 kv_submitted 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57bd3b00 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57bd3b00 finishing 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57bd3b00 onodes 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57bd3b00 done 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58748c00 kv_submitted 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57bd3b00 [] 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean+snaptrim ps=[e~1,15~1,1a~1,23~1]] exit Trimming 2019-12-07T19:34:36.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.624+0000 7f8f06986700 15 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 59:202 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x471417 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] enter NotTrimming 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122427s (0.000000061s flush + 0.000122366s kv commit) 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58748c00> 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748c00 kv_submitted 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58748c00 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748c00 finishing 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58748c00 onodes 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58748c00 done 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58748c00 [] 2019-12-07T19:34:36.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:36.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.624+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] snap_trimmer complete 2019-12-07T19:34:36.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.624+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 210627 (210625 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:36.625+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 59/41 ECSubWriteReply(tid=119, last_complete=59'48, committed=1, applied=1)) v2 -- 0x558c58513200 con 0x558c54c5b400 2019-12-07T19:34:36.625+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:36.625+0000 7f8f04181700 20 bluestore.extentmap(0x558c58768108) update 4#1:9b8adabe:::smithi13140796-6:23# 2019-12-07T19:34:36.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:23# is 123322 (123320 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:36.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57957500 allocated 0x[] released 0x[] 2019-12-07T19:34:36.625+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957500 prepare 2019-12-07T19:34:36.625+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57957500 2019-12-07T19:34:36.625+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957500 io_done 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.625+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=58'46 crt=58'46 lcod 58'46 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 59'48 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 27625 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f04181700 10 osd.7 59 dequeue_op 0x558c57335e40 finish 2019-12-07T19:34:36.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.625+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 60393 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 93161 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 125929 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 158697 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 116 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58115c00 con 0x558c54c19c00 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 191465 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c57335ce0 prio 196 cost 0 latency 0.000015 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.625+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 224233 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 257001 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c57335ce0 prio 196 cost 0 latency 0.000044 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:36.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 289769 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c57335ce0 finish 2019-12-07T19:34:36.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 322537 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 295 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57612600 con 0x558c54c5bc00 2019-12-07T19:34:36.625+0000 7f8f23c56700 15 osd.7 59 enqueue_op 0x558c581cd8c0 prio 196 cost 0 latency 0.000009 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.625+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 336967 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.625+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c581cd8c0 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:36.625+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.625+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c581cd8c0 finish 2019-12-07T19:34:36.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58768000 had 1 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 336990 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 336990 < min_flush_size 524288 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x471417~5245e to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x471417 0x471417~5245e to file(ino 16 size 0x480000 mtime 2019-12-07T19:34:36.624950+0000 allocated 480000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~450000]) 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _allocate len 0x43875 from 0 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _allocate len 0x43875 from 1 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x50000/10000,50000,acd5b0000 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd5b0000~50000/10000,50000,acd5b0000 2019-12-07T19:34:36.625+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x4d0000 to include allocated 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 75 (was clean) 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x441417 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x417 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x875 and padding block with 0x78b 2019-12-07T19:34:36.625+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x4d0000 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c3875 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 506 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841e200 con 0x558c54c64000 2019-12-07T19:34:36.626+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c57335340 prio 196 cost 0 latency 0.000011 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.626+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (75) on file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]), flushing log 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:36.626+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c57335340 prio 196 cost 0 latency 0.000051 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:36.626+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 318 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c551afc00 con 0x558c54c5b400 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 75 len 0x4a crc 0xf12c03f6) 2019-12-07T19:34:36.626+0000 7f8f23c56700 15 osd.7 59 enqueue_op 0x558c581cc580 prio 196 cost 0 latency 0.000014 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.626+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x4a000~1000 to file(ino 1 size 0x4a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:36.626+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c57335340 finish 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x4a000 0x4a000~1000 to file(ino 1 size 0x4a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x4b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4a000 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.626+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c581cc580 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.626+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c581cc580 finish 2019-12-07T19:34:36.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x4b000 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:36.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000029 2019-12-07T19:34:36.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 208 ==== MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f400 con 0x558c54c19800 2019-12-07T19:34:36.626+0000 7f8f24c58700 15 osd.7 59 enqueue_op 0x558c5511f1e0 prio 196 cost 0 latency 0.000013 epoch 59 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.626+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) queued 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2) prio 196 cost 0 e59) pg 0x558c57676000 2019-12-07T19:34:36.626+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c5511f1e0 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:36.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 507 ==== pg_info2(1.as5 1.as5 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:36.626+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 59/28 ECSubWriteReply(tid=54, last_complete=59'48, committed=1, applied=1)) v2 2019-12-07T19:34:36.626+0000 7f8f24457700 15 osd.7 59 enqueue_peering_evt 1.as5 epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s) 2019-12-07T19:34:36.626+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.626+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:23 v=0'0 tt=0'0 tid=54 reqid=osd.7.0:53 roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:36.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e59) pg 0x558c57e8b000 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:36.626+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c5511f1e0 finish 2019-12-07T19:34:36.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:36.626+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s) 2019-12-07T19:34:36.626+0000 7f8f03980700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.626+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:36.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:36.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58260c00 seq 29 2019-12-07T19:34:36.626+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:36.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:36.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:36.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:36.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:36.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58260c00 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58260c00 onodes shared_blobs 2019-12-07T19:34:36.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58260c00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260c00 prepare 2019-12-07T19:34:36.626+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58260c00 2019-12-07T19:34:36.626+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260c00 io_done 2019-12-07T19:34:36.626+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:36.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 508 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:36.627+0000 7f8f24457700 15 osd.7 59 enqueue_peering_evt 1.7s4 epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s) 2019-12-07T19:34:36.627+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e59) pg 0x558c54d5f000 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s) 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5875cf00 seq 89 2019-12-07T19:34:36.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875cf00 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875cf00 onodes shared_blobs 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875cf00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cf00 prepare 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875cf00 2019-12-07T19:34:36.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cf00 io_done 2019-12-07T19:34:36.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 509 ==== MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=297, reqid=osd.1.0:297, at_version=59'10, trim_to=0'0, roll_forward_to=58'8)) v2 ==== 3449+0+0 (crc 0 0 0) 0x558c586d0000 con 0x558c54c64000 2019-12-07T19:34:36.627+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000031 epoch 59 MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=297, reqid=osd.1.0:297, at_version=59'10, trim_to=0'0, roll_forward_to=58'8)) v2 2019-12-07T19:34:36.627+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=297, reqid=osd.1.0:297, at_version=59'10, trim_to=0'0, roll_forward_to=58'8)) v2) prio 127 cost 0 e59) 2019-12-07T19:34:36.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=297, reqid=osd.1.0:297, at_version=59'10, trim_to=0'0, roll_forward_to=58'8)) v2) prio 127 cost 0 e59) queued 2019-12-07T19:34:36.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.627+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=297, reqid=osd.1.0:297, at_version=59'10, trim_to=0'0, roll_forward_to=58'8)) v2) prio 127 cost 0 e59) pg 0x558c54d98000 2019-12-07T19:34:36.627+0000 7f8f05984700 10 osd.7 59 dequeue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=297, reqid=osd.1.0:297, at_version=59'10, trim_to=0'0, roll_forward_to=58'8)) v2 pg pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.627+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=297, reqid=osd.1.0:297, at_version=59'10, trim_to=0'0, roll_forward_to=58'8)) v2 2019-12-07T19:34:36.627+0000 7f8f05984700 20 snap_mapper.update_snaps 1:733d7474:::smithi13140796-5:23 1b,20,21 was 2019-12-07T19:34:36.627+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.627+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:36.627+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.627+0000 7f8f05984700 20 snap_mapper.get_snaps 1:733d7474:::smithi13140796-5:23 1b,20,21,23 2019-12-07T19:34:36.627+0000 7f8f05984700 20 snap_mapper.set_snaps 1:733d7474:::smithi13140796-5:23 1b,20,21 2019-12-07T19:34:36.627+0000 7f8f05984700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:36.627+0000 7f8f05984700 20 snap_mapper.update_snaps rm SNA_1_0000000000000023_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:36.627+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 58'8 (0'0,58'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,58'8], crt=58'8) [59'9 (58'5) modify 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:34:36.628652+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,59'10 (58'8) modify 1:733d7474:::smithi13140796-5:head by unknown.0.0:0 2019-12-07T19:34:36.628652+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:36.627+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'9 (0'0,59'9] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'8 crt=58'8 lcod 58'8 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 59'9 (58'5) modify 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:34:36.628652+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.627+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'8 crt=58'8 lcod 58'8 active mbc={} ps=[e~1,15~1,1a~1]] add_log_entry 59'10 (58'8) modify 1:733d7474:::smithi13140796-5:head by unknown.0.0:0 2019-12-07T19:34:36.628652+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.627+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'8 crt=58'8 lcod 58'8 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 10 2019-12-07T19:34:36.627+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'8 crt=58'8 lcod 58'8 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.627+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.627+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 59'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c586e0f00 seq 27 2019-12-07T19:34:36.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:23# 1 keys 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:23# 1 keys = 0 2019-12-07T19:34:36.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# hinfo_key (42 bytes) 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys = 0 2019-12-07T19:34:36.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:36.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000023_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- SNA_1_0000000000000023_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:36.628+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000059.00000000000000000009' <- 0000000059.00000000000000000009 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000059.00000000000000000010' <- 0000000059.00000000000000000010 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e0f00 cost 8436 (1 ios * 4000 + 4436 bytes) 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e0f00 onodes 0x558c579de500,0x558c579dea00 shared_blobs 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de608) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 658 (571 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore.extentmap(0x558c579deb08) update 4#1:733d7474:::smithi13140796-5:23# 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:23# is 497 (410 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e0f00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0f00 prepare 2019-12-07T19:34:36.628+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e0f00 2019-12-07T19:34:36.628+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0f00 io_done 2019-12-07T19:34:36.628+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=58'8 crt=58'8 lcod 58'8 active mbc={} ps=[e~1,15~1,1a~1]] op_applied version 59'10 2019-12-07T19:34:36.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f05984700 10 osd.7 59 dequeue_op 0x558c57334dc0 finish 2019-12-07T19:34:36.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 75 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.003220362s (0.000000059s flush + 0.003220303s kv commit) 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57957500> 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957500 kv_submitted 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57957500 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57957500 finishing 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57957500 onodes 0x558c57a35400,0x558c58768000 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57957500 done 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57957500 [] 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2332 < min_flush_size 524288 2019-12-07T19:34:36.628+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6377 < min_flush_size 524288 2019-12-07T19:34:36.628+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 59/41 ECSubWriteReply(tid=296, last_complete=59'48, committed=1, applied=1)) v2 -- 0x558c57613600 con 0x558c54c64000 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579de500 had 1 2019-12-07T19:34:36.628+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579dea00 had 1 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6400 < min_flush_size 524288 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6400 < min_flush_size 524288 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c3875~1900 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c3875 0x4c3875~1900 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x493875 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x875 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x175 and padding block with 0xe8b 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c5175 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.000119068s (0.000000116s flush + 0.000118952s kv commit) 2019-12-07T19:34:36.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58260c00,0x558c5875cf00,0x558c586e0f00> 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260c00 kv_submitted 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58260c00 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260c00 finishing 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58260c00 onodes 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58260c00 done 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58260c00 [] 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cf00 kv_submitted 2019-12-07T19:34:36.628+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875cf00 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875cf00 finishing 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875cf00 onodes 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875cf00 done 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875cf00 [] 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.628+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0f00 kv_submitted 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e0f00 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0f00 finishing 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e0f00 onodes 0x558c579de500,0x558c579dea00 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0f00 done 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:36.628+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e0f00 [] 2019-12-07T19:34:36.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.628+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 59/41 ECSubWriteReply(tid=297, last_complete=59'10, committed=1, applied=1)) v2 -- 0x558c57518000 con 0x558c54c64000 2019-12-07T19:34:36.628+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.628+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:36.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 510 ==== MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=298, reqid=osd.1.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c5822e000 con 0x558c54c64000 2019-12-07T19:34:36.629+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c573354a0 prio 127 cost 0 latency 0.000018 epoch 59 MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=298, reqid=osd.1.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:36.629+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=298, reqid=osd.1.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e59) 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=298, reqid=osd.1.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e59) queued 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=298, reqid=osd.1.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e59) pg 0x558c5500a000 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 59 dequeue_op 0x558c573354a0 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=298, reqid=osd.1.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 pg pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.9s4 59/41 ECSubWrite(tid=298, reqid=osd.1.0:296, at_version=0'0, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=58'46 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,59'48], crt=58'46) [] 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=59'47 (58'45) modify 1:9b8adabe:::smithi13140796-6:23 by unknown.0.0:0 2019-12-07T19:34:36.625143+0000 0 snaps [21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=59'48 (58'46) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:36.625143+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 48 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.629+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.629+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c58540c00 seq 89 2019-12-07T19:34:36.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58540c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58540c00 onodes shared_blobs 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58540c00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 prepare 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58540c00 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 io_done 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 59 dequeue_op 0x558c573354a0 finish 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c5175~471 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c5175 0x4c5175~471 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x495175 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x175 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5e6 and padding block with 0xa1a 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c55e6 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124254s (0.000000129s flush + 0.000124125s kv commit) 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58540c00> 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 kv_submitted 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58540c00 2019-12-07T19:34:36.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58540c00 finishing 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58540c00 onodes 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58540c00 done 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:36.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58540c00 [] 2019-12-07T19:34:36.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.629+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 59/41 ECSubWriteReply(tid=298, last_complete=59'48, committed=1, applied=1)) v2 -- 0x558c548cc200 con 0x558c54c64000 2019-12-07T19:34:36.629+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 511 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:36.629+0000 7f8f24457700 15 osd.7 59 enqueue_peering_evt 1.9s4 epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s) 2019-12-07T19:34:36.629+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e59) pg 0x558c5500a000 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s) 2019-12-07T19:34:36.629+0000 7f8f04181700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.629+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c578ab200 seq 90 2019-12-07T19:34:36.629+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578ab200 cost 5305 (1 ios * 4000 + 1305 bytes) 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578ab200 onodes shared_blobs 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578ab200 allocated 0x[] released 0x[] 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab200 prepare 2019-12-07T19:34:36.629+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578ab200 2019-12-07T19:34:36.629+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab200 io_done 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.629+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1142 < min_flush_size 524288 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c55e6~48d to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c55e6 0x4c55e6~48d to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x4955e6 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5e6 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa73 and padding block with 0x58d 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c5a73 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122118s (0.000000086s flush + 0.000122032s kv commit) 2019-12-07T19:34:36.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c578ab200> 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab200 kv_submitted 2019-12-07T19:34:36.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578ab200 2019-12-07T19:34:36.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578ab200 finishing 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578ab200 onodes 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578ab200 done 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:36.630+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578ab200 [] 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.630+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 512 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:36.630+0000 7f8f24457700 15 osd.7 59 enqueue_peering_evt 1.0s3 epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s) 2019-12-07T19:34:36.630+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 513 ==== MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=299, reqid=osd.1.0:297, at_version=0'0, trim_to=0'0, roll_forward_to=59'10)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c579e2000 con 0x558c54c64000 2019-12-07T19:34:36.630+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c573358c0 prio 127 cost 0 latency 0.000017 epoch 59 MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=299, reqid=osd.1.0:297, at_version=0'0, trim_to=0'0, roll_forward_to=59'10)) v2 2019-12-07T19:34:36.630+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=299, reqid=osd.1.0:297, at_version=0'0, trim_to=0'0, roll_forward_to=59'10)) v2) prio 127 cost 0 e59) 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e59) pg 0x558c5500d000 2019-12-07T19:34:36.630+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s) 2019-12-07T19:34:36.630+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.630+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c5875a600 seq 50 2019-12-07T19:34:36.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=299, reqid=osd.1.0:297, at_version=0'0, trim_to=0'0, roll_forward_to=59'10)) v2) prio 127 cost 0 e59) queued 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=299, reqid=osd.1.0:297, at_version=0'0, trim_to=0'0, roll_forward_to=59'10)) v2) prio 127 cost 0 e59) pg 0x558c54d98000 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875a600 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875a600 onodes shared_blobs 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875a600 allocated 0x[] released 0x[] 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 osd.7 59 dequeue_op 0x558c573358c0 prio 127 cost 0 latency 0.000103 MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=299, reqid=osd.1.0:297, at_version=0'0, trim_to=0'0, roll_forward_to=59'10)) v2 pg pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a600 prepare 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875a600 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a600 io_done 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] _handle_message: MOSDECSubOpWrite(1.es4 59/41 ECSubWrite(tid=299, reqid=osd.1.0:297, at_version=0'0, trim_to=0'0, roll_forward_to=59'10)) v2 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=58'8 active mbc={} ps=[e~1,15~1,1a~1]] append_log log((0'0,59'10], crt=58'8) [] 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=59'9 (58'5) modify 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:34:36.628652+0000 0 snaps [1b,20,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1]] rollforward: entry=59'10 (58'8) modify 1:733d7474:::smithi13140796-5:head by unknown.0.0:0 2019-12-07T19:34:36.628652+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c5a73~4a5 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c5a73 0x4c5a73~4a5 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1]] append_log approx pg log length = 10 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x495a73 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa73 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf18 and padding block with 0xe8 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1]] append_log transaction_applied = 1 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:36.630+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5853f500 seq 28 2019-12-07T19:34:36.630+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853f500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853f500 onodes shared_blobs 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853f500 allocated 0x[] released 0x[] 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f500 prepare 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853f500 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f500 io_done 2019-12-07T19:34:36.630+0000 7f8f0197c700 10 osd.7 59 dequeue_op 0x558c573358c0 finish 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c5f18 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000140182s (0.000000103s flush + 0.000140079s kv commit) 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875a600> 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a600 kv_submitted 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875a600 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a600 finishing 2019-12-07T19:34:36.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 514 ==== pg_info2(1.es4 1.es4 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875a600 onodes 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875a600 done 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875a600 [] 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c5f18~471 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c5f18 0x4c5f18~471 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f24457700 15 osd.7 59 enqueue_peering_evt 1.es4 epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s) 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x495f18 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf18 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x389 and padding block with 0xc77 2019-12-07T19:34:36.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e59) pg 0x558c54d98000 2019-12-07T19:34:36.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 515 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e59/59) v1 ==== 1108+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:36.630+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s) 2019-12-07T19:34:36.630+0000 7f8f24457700 15 osd.7 59 enqueue_peering_evt 1.8s5 epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s) 2019-12-07T19:34:36.630+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e59) 2019-12-07T19:34:36.630+0000 7f8f05984700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.630+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:36.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:36.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c586e0c00 seq 29 2019-12-07T19:34:36.630+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:36.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:36.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:36.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:36.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e0c00 cost 5305 (1 ios * 4000 + 1305 bytes) 2019-12-07T19:34:36.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e0c00 onodes shared_blobs 2019-12-07T19:34:36.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:36.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0c00 prepare 2019-12-07T19:34:36.630+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e0c00 2019-12-07T19:34:36.630+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0c00 io_done 2019-12-07T19:34:36.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e59) queued 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e59) pg 0x558c57380000 2019-12-07T19:34:36.630+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] do_peering_event: epoch_sent: 59 epoch_requested: 59 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s) 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c6389 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.630+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1]] proc_primary_info updating purged_snaps to [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000211523s (0.000000085s flush + 0.000211438s kv commit) 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.630+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1142 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5875a900 seq 54 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1165 < min_flush_size 524288 2019-12-07T19:34:36.630+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c6389~48d to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c6389 0x4c6389~48d to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x496389 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x389 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x816 and padding block with 0x7ea 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875a900 cost 5323 (1 ios * 4000 + 1323 bytes) 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875a900 onodes shared_blobs 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875a900 allocated 0x[] released 0x[] 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a900 prepare 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853f500> 2019-12-07T19:34:36.630+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875a900 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.630+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a900 io_done 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f500 kv_submitted 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853f500 2019-12-07T19:34:36.630+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f500 finishing 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853f500 onodes 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853f500 done 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0c00 kv_submitted 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853f500 [] 2019-12-07T19:34:36.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.630+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.630+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 59/41 ECSubWriteReply(tid=299, last_complete=59'10, committed=1, applied=1)) v2 -- 0x558c57519a00 con 0x558c54c64000 2019-12-07T19:34:36.630+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c6816 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000160804s (0.000000056s flush + 0.000160748s kv commit) 2019-12-07T19:34:36.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:36.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1166 < min_flush_size 524288 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1189 < min_flush_size 524288 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c6816~4a5 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c6816 0x4c6816~4a5 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x496816 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x816 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcbb and padding block with 0x345 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586e0c00> 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0c00 kv_submitted 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e0c00 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0c00 finishing 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e0c00 onodes 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0c00 done 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e0c00 [] 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.631+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.631+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c6cbb 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:36.631+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000131064s (0.000000080s flush + 0.000130984s kv commit) 2019-12-07T19:34:36.631+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875a900> 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a900 kv_submitted 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875a900 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875a900 finishing 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875a900 onodes 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875a900 done 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:36.631+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875a900 [] 2019-12-07T19:34:36.631+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:36.631+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:36.631+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:36.646+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0259315 2019-12-07T19:34:36.696+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:34:36.696+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:36.696+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 48 0x2000~1000 2019-12-07T19:34:36.696+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc ce9827e9 2019-12-07T19:34:36.696+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 48 0x1b000~1000 2019-12-07T19:34:36.696+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc dc082dc3 2019-12-07T19:34:36.696+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.696+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:36.696+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 10 osd.7 59 tick_without_osd_lock 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f438000/0x40000000/0x1680000000, data 0xd3cb4f/0xbc6000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 25 osd.7 59 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:32.576231+0000 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 25 osd.7 59 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576218+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 25 osd.7 59 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 25 osd.7 59 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576154+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 25 osd.7 59 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 20 osd.7 59 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 10 osd.7 59 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 20 osd.7 59 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:36.734+0000 7f8f1fbbe700 10 osd.7 59 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:36.746+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.796+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.821+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:36.847+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.897+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.946+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:36.997+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.047+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.097+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.145+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 18 ==== osd_ping(ping e59 up_from 13 ping_stamp 2019-12-07T19:34:37.146433+0000/60.618152120s send_stamp 60.618152120s delta_ub 0.036618268s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57519a00 con 0x558c552d1c00 2019-12-07T19:34:37.145+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 18 ==== osd_ping(ping e59 up_from 13 ping_stamp 2019-12-07T19:34:37.146433+0000/60.618152120s send_stamp 60.618152120s delta_ub 0.036618268s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c552d1800 2019-12-07T19:34:37.145+0000 7f8f24457700 20 osd.7 59 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036144324s,0.036618268s]) 2019-12-07T19:34:37.145+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e59 up_from 13 ping_stamp 2019-12-07T19:34:37.146433+0000/60.618152120s send_stamp 60.582007796s delta_ub 18446744073.673407292s) v5 -- 0x558c57518000 con 0x558c552d1c00 2019-12-07T19:34:37.145+0000 7f8f24c58700 20 osd.7 59 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036125134s,0.036618268s]) 2019-12-07T19:34:37.145+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e59 up_from 13 ping_stamp 2019-12-07T19:34:37.146433+0000/60.618152120s send_stamp 60.582026986s delta_ub 18446744073.673426482s) v5 -- 0x558c58332400 con 0x558c552d1800 2019-12-07T19:34:37.147+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.155+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 18 ==== osd_ping(ping e59 up_from 13 ping_stamp 2019-12-07T19:34:37.155605+0000/60.613117797s send_stamp 60.613117797s delta_ub 0.022409567s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c576b5400 con 0x558c54c18000 2019-12-07T19:34:37.155+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 18 ==== osd_ping(ping e59 up_from 13 ping_stamp 2019-12-07T19:34:37.155605+0000/60.613117797s send_stamp 60.613117797s delta_ub 0.022409567s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c548cc200 con 0x558c552d1400 2019-12-07T19:34:37.155+0000 7f8f23c56700 20 osd.7 59 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021941250s,0.022409567s]) 2019-12-07T19:34:37.155+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e59 up_from 13 ping_stamp 2019-12-07T19:34:37.155605+0000/60.613117797s send_stamp 60.591176547s delta_ub 18446744073.687610366s) v5 -- 0x558c5763c600 con 0x558c54c18000 2019-12-07T19:34:37.155+0000 7f8f24457700 20 osd.7 59 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021922996s,0.022409567s]) 2019-12-07T19:34:37.155+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e59 up_from 13 ping_stamp 2019-12-07T19:34:37.155605+0000/60.613117797s send_stamp 60.591194801s delta_ub 18446744073.687628620s) v5 -- 0x558c57518000 con 0x558c552d1400 2019-12-07T19:34:37.197+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.213+0000 7f8f21451700 10 osd.7 59 tick 2019-12-07T19:34:37.213+0000 7f8f21451700 10 osd.7 59 do_waiters -- start 2019-12-07T19:34:37.213+0000 7f8f21451700 10 osd.7 59 do_waiters -- finish 2019-12-07T19:34:37.213+0000 7f8f21451700 20 osd.7 59 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 2019-12-07T19:34:37.224+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:37.247+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.297+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.347+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.397+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.448+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.498+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.548+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.598+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1499699 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 26 ==== osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 ==== 257+0+0 (crc 0 0 0) 0x558c583b38c0 con 0x558c54c9f000 2019-12-07T19:34:37.624+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c54e78420 prio 63 cost 0 latency 0.000031 epoch 60 osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 27 ==== osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 ==== 257+0+0 (crc 0 0 0) 0x558c583b3b80 con 0x558c54c9f000 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) queued 2019-12-07T19:34:37.624+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c54e79a20 prio 63 cost 0 latency 0.000011 epoch 60 osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) 2019-12-07T19:34:37.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 28 ==== osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 ==== 257+0+0 (crc 0 0 0) 0x558c58b7b8c0 con 0x558c54c9f000 2019-12-07T19:34:37.624+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c54e78420 prio 63 cost 0 latency 0.000070 osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.624+0000 7f8f24457700 15 osd.7 59 enqueue_op 0x558c54e78580 prio 63 cost 0 latency 0.000012 epoch 60 osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.624+0000 7f8f0297e700 10 osd.7 59 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 58 -> 60 (as per caller) 2019-12-07T19:34:37.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) 2019-12-07T19:34:37.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_request min 60, queue on waiting_for_map client.4540 2019-12-07T19:34:37.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) queued 2019-12-07T19:34:37.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=60}) v3 -- 0x558c5763ce00 con 0x558c5476d000 2019-12-07T19:34:37.624+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) queued 2019-12-07T19:34:37.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.624+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c54e79a20 prio 63 cost 0 latency 0.000110 osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.624+0000 7f8f06986700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:34:37.624+0000 7f8f06986700 10 osd.7 59 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:34:37.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.624+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c54e78580 prio 63 cost 0 latency 0.000130 osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:34:37.624+0000 7f8f0297e700 10 osd.7 59 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:37.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.624+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 33 ==== osd_map(60..60 src has 1..60) v4 ==== 683+0+0 (crc 0 0 0) 0x558c5738e900 con 0x558c5476d000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 osd.7 59 OSD::ms_dispatch: osd_map(60..60 src has 1..60) v4 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 osd.7 59 do_waiters -- start 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 osd.7 59 do_waiters -- finish 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 osd.7 59 _dispatch 0x558c5738e900 osd_map(60..60 src has 1..60) v4 2019-12-07T19:34:37.624+0000 7f8f164bb700 3 osd.7 59 handle_osd_map epochs [60,60], i have 59, src has [1,60] 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 osd.7 59 handle_osd_map got inc map for epoch 60 2019-12-07T19:34:37.624+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:36b4e826:::osdmap.59:0# 0x0~0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1474 size 0x1474 (5236) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4d90 blob([0xbc6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580a5030 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580a4a80 blob([!~1000,0xbc7000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,474]) SharedBlob(0x558c574e3110 sbid 0x0)) need 0x1000~474 cache has 0x[1000~474] 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:36b4e826:::osdmap.59:0# 0x0~1474 = 5236 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 osd.7 59 add_map_bl 59 5236 bytes 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 osd.7 59 got_full_map 60, nothing requested 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 osd.7 59 handle_osd_map pg_num_history pg_num_history(e60 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {60={}} 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 osd.7 59 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e60 [1,60] lci=[13,60]) 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c582b6c00 seq 51 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cf5f972f:::inc_osdmap.60:0# key 0x7f7fffffffffffffffcf5f97'/!inc_osdmap.60!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cf5f972f:::inc_osdmap.60:0# 0x0~27f 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1280 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cf5f972f:::inc_osdmap.60:0# 0x0~27f - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~27f 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd81 on front/back, now 0x0~1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6c00 1 blobs 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbc8000~1000/1000,1000,0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbc8000~1000] 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57ff41c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5757e150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57ff41c0 blob([0xbc8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e150 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c57ff41c0) get_ref 0x0~27f Blob(0x558c57ff41c0 blob([0xbc8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5757e150 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c57ff41c0) get_ref init 0x1000, 1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~27f: 0x0~27f Blob(0x558c57ff41c0 blob([0xbc8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x27f) SharedBlob(0x558c5757e150 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5757e168 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x27f 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58768608) dirty_range mark inline shard dirty 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cf5f972f:::inc_osdmap.60:0# 0x0~27f = 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:33c4e826:::osdmap.60:0# key 0x7f7fffffffffffffff33c4e8'&!osdmap.60!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:33c4e826:::osdmap.60:0# 0x0~1458 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1281 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:33c4e826:::osdmap.60:0# 0x0~1458 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~458 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xba8 on front/back, now 0x0~1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6c00 2 blobs 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbc9000~2000/1000,2000,0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbc9000~2000] 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574f1f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c579d7960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574f1f80 blob([0xbc9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579d7960 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c574f1f80) get_ref 0x0~1000 Blob(0x558c574f1f80 blob([0xbc9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579d7960 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c574f1f80) get_ref init 0x1000, 1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c574f1f80 blob([0xbc9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579d7960 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c579d7978 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580f7030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f70a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580f7030 blob([!~1000,0xbca000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f70a0 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c580f7030) get_ref 0x1000~458 Blob(0x558c580f7030 blob([!~1000,0xbca000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c580f70a0 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c580f7030) get_ref init 0x2000, 1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~458: 0x1000~458 Blob(0x558c580f7030 blob([!~1000,0xbca000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,458]) SharedBlob(0x558c580f70a0 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c580f70b8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1458 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58768388) dirty_range mark inline shard dirty 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:33c4e826:::osdmap.60:0# 0x0~1458 = 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6c00 0 blobs 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6c00 0 blobs 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b6c00 cost 23328 (4 ios * 4000 + 7328 bytes) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b6c00 onodes 0x558c53a91900,0x558c55276c80,0x558c58768280,0x558c58768500 shared_blobs 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58768388) update #-1:33c4e826:::osdmap.60:0# 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58768388) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:33c4e826:::osdmap.60:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58768608) update #-1:cf5f972f:::inc_osdmap.60:0# 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore.extentmap(0x558c58768608) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cf5f972f:::inc_osdmap.60:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b6c00 allocated 0x[bc8000~3000] released 0x[] 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 freelist allocate 0xbc8000~3000 2019-12-07T19:34:37.624+0000 7f8f164bb700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6c00 prepare 2019-12-07T19:34:37.624+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c582b6c00 2019-12-07T19:34:37.625+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6c00 aio_wait 2019-12-07T19:34:37.625+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b6c00 2019-12-07T19:34:37.625+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6c00 io_done 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4933 < min_flush_size 524288 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58768280 had 1 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58768500 had 1 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3855810560 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997491) bluestore 86 GiB free (0.959872), bluefs_ratio 0.0399092 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8871 < min_flush_size 524288 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8871 < min_flush_size 524288 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c6cbb~22a7 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c6cbb 0x4c6cbb~22a7 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x496cbb 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcbb 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf62 and padding block with 0x9e 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4c8f62 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:37.625+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000192882s (0.000045582s flush + 0.000147300s kv commit) 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:37.625+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b6c00> 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c575c2280> 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6c00 kv_submitted 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b6c00 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c582b6c00 osr 0x558c54ea4b40 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 49 0x1b000~1000 crc 98166b30 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7 59 _committed_osd_maps 60..60 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7 59 advance to epoch 60 (<= last 60 <= newest_map 60) 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 49 0x2000~1000 crc e10dd049 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6600 deferred_cleanup 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b6600 onodes 0x558c53a91900,0x558c55276c80,0x558c5795b400,0x558c5795b680 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5742beb8 in 0x558c53a59340) _finish_write discard buffer(0x558c551a8750 space 0x558c5742beb8 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 7 osd.7 60 consume_map version 60 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57efce88 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2fc0 space 0x558c57efce88 0x1000~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.8s5 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829c1d8 in 0x558c53a59340) _finish_write discard buffer(0x558c586c2f30 space 0x558c5829c1d8 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.0s3 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6600 done 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.1s1 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6900 deferred_cleanup 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.9s4 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b6600 [] 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.as5 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.2s0 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.bs1 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.3s4 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.cs0 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.4s0 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6900 deferred_cleanup 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.5s4 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b6900 onodes 0x558c53a91900,0x558c55276c80,0x558c5795b900,0x558c5795bb80 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.ds2 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.es4 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579d7588 in 0x558c53a59340) _finish_write discard buffer(0x558c580f2cf0 space 0x558c579d7588 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.6s2 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f6ef8 in 0x558c53a59340) _finish_write discard buffer(0x558c580f2d80 space 0x558c580f6ef8 0x1000~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.fs5 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5898b7b8 in 0x558c53a59340) _finish_write discard buffer(0x558c586c3170 space 0x558c5898b7b8 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 identify_splits_and_merges 1.7s4 e59 to e60 pg_nums {18=16} 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6900 done 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 60 prune_pg_created 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578aaf00 deferred_cleanup 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:0.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b6900 [] 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:1.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:2.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:3.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:4.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aaf00 deferred_cleanup 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:5.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578aaf00 onodes 0x558c53a91900,0x558c55276c80,0x558c5851ec80,0x558c5851ef00 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:6.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3908 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8f290 space 0x558c550b3908 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7:7.consume_map 60 (was 59) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55106018 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8f320 space 0x558c55106018 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90f68 in 0x558c53a59340) _finish_write discard buffer(0x558c58c8f3b0 space 0x558c57f90f68 0x1000~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 10 osd.7 60 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578aaf00 done 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5876a000 deferred_cleanup 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578aaf00 [] 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.8s5 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876a000 deferred_cleanup 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5876a000 onodes 0x558c53a91900,0x558c55276c80,0x558c5851f180,0x558c5851f400 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e3128 in 0x558c53a59340) _finish_write discard buffer(0x558c58534cf0 space 0x558c574e3128 0x1000~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91668 in 0x558c53a59340) _finish_write discard buffer(0x558c58534bd0 space 0x558c57f91668 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.0s3 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5048 in 0x558c53a59340) _finish_write discard buffer(0x558c58534c60 space 0x558c580a5048 0x0~1000 writing nocache) 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5876a000 done 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6c00 deferred_queued 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5876a000 [] 2019-12-07T19:34:37.625+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c57380000 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.1s1 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.9s4 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.as5 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c5500d000 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.2s0 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.bs1 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.3s4 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c5736a000 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 59 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.cs0 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c575ef000 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c57e8b000 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.4s0 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 59 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.5s4 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.ds2 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.1s1 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.es4 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.6s2 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 59 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.fs5 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f141a1700 15 osd.7 60 enqueue_peering_evt 1.7s4 epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:37.625+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f141a1700 7 osd.7 60 activate_map version 60 2019-12-07T19:34:37.625+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f04982700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c57383000 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c5500a000 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c57e88000 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c57522000 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c57676000 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c5513e000 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s0 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 59 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 59 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 59 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 59 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active advmap 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active advmap 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:37.625+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:37.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:37.625+0000 7f8f07988700 7 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c54d98000 2019-12-07T19:34:37.625+0000 7f8f06986700 7 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c54d5f000 2019-12-07T19:34:37.625+0000 7f8f07988700 15 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 59:172 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f06986700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 59:203 2019-12-07T19:34:37.625+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 59 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:37.625+0000 7f8f0918b700 20 osd.7 60 agent_entry empty queue 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] requeue_map_waiters client.4540 0x558c54e78420,0x558c54e79a20,0x558c54e78580 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c548ab000 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) queued 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.625+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 59 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c54e78420 prio 63 cost 0 latency 0.001528 osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 pg pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active advmap 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c54d5c000 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: osd_op(client.4540.0:449 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.625+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) queued 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 59 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op: op osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 60 epoch_requested: 60 NullEvt) prio 255 cost 10 e60) pg 0x558c55141000 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:37.625+0000 7f8f0297e700 7 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 60 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 59 -> 60 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f0297e700 15 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 59:160 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 59 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) queued 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:37.625+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(57'42 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 19=[]:{} 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 60 advance_pg new pool opts old pool opts 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 60 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(57'42 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 60 2019-12-07T19:34:37.625+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06986700 25 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(57'42 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:37.625+0000 7f8f06185700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 60 epoch_requested: 60 NullEvt 2019-12-07T19:34:37.625+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:37.625+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0)) 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx 0x558c58236e00 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [append 0~0] ov 57'42 av 60'49 snapc 25=[25,21,20,1b] snapset 19=[]:{} 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [append 0~0] 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op append 0~0 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [write 1622016~0] 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op write 1622016~0 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] clean_regions modifiedclean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=19=[]:{} snapc=25=[25,21,20,1b] 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] got greedy write on clone_obc obc(1:33f5bf01:::smithi13140796-2:25 rwstate(write n=1 w=0)) 2019-12-07T19:34:37.625+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] cloning v 57'42 to 1:33f5bf01:::smithi13140796-2:25 v 60'49 snaps=[25,21,20,1b] snapset=19=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.625+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x558c58236e00 op modify 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] set mtime to 2019-12-07T19:34:37.625406+0000 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] final snapset 25=[]:{25=[25,21,20,1b]} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx alloc reply 0x558c57bd4780 result 0 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] calc_trim_to_aggressive limit = 59'48 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op order client.4540 tid 449 last was 380 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] new_repop rep_tid 55 on osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] new_repop: repgather(0x558c576b6fc0 0'0 rep_tid=55 committed?=0 r=0) 2019-12-07T19:34:37.626+0000 7f8f06986700 7 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] issue_repop rep_tid 55 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=60'50 tt=0'0 tid=55 reqid=client.4540.0:449 client_op=osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:25# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0x0000000000000025ffffffffffffffff'o' 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:37.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:25 projected size 1622016 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=60'50 tt=0'0 tid=55 reqid=client.4540.0:449 client_op=osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_state_to_reads: invalidating cache after this op 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=60'50 tt=0'0 tid=55 reqid=client.4540.0:449 client_op=osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=60'50 tt=0'0 tid=55 reqid=client.4540.0:449 client_op=osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:25, current size is 0 buffers are {} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): new_size start 1622016 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_append: {} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:25 resetting hinfo to logical size 1622016 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): new_size start 1622016 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_append: {} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:25={},1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=60'50 tt=0'0 tid=55 reqid=client.4540.0:449 client_op=osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:25,{} 2019-12-07T19:34:37.626+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 send_incremental_map 59 -> 60 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:37.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cf5f972f:::inc_osdmap.60:0# 0x0~0 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~27f size 0x27f (639) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57ff41c0 blob([0xbc8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x27f) SharedBlob(0x558c5757e150 sbid 0x0)) need 0x0~27f cache has 0x[0~27f] 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cf5f972f:::inc_osdmap.60:0# 0x0~27f = 639 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 add_map_inc_bl 60 639 bytes 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(60..60 src has 1..60) v4 -- 0x558c5822c900 con 0x558c54c64000 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 59 -> 60 (shared) 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 60/28 ECSubWrite(tid=55, reqid=client.4540.0:449, at_version=60'50, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c57dff000 con 0x558c54c64000 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 send_incremental_map 59 -> 60 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(60..60 src has 1..60) v4 -- 0x558c5822cb40 con 0x558c54c19800 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 59 -> 60 (shared) 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 60/28 ECSubWrite(tid=55, reqid=client.4540.0:449, at_version=60'50, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c57dff800 con 0x558c54c19800 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 send_incremental_map 59 -> 60 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(60..60 src has 1..60) v4 -- 0x558c5822cd80 con 0x558c54c5bc00 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 59 -> 60 (shared) 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 60/28 ECSubWrite(tid=55, reqid=client.4540.0:449, at_version=60'50, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c57e01800 con 0x558c54c5bc00 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 send_incremental_map 59 -> 60 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(60..60 src has 1..60) v4 -- 0x558c5822cfc0 con 0x558c54c5b400 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 59 -> 60 (shared) 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 60/28 ECSubWrite(tid=55, reqid=client.4540.0:449, at_version=60'50, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c54895800 con 0x558c54c5b400 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 send_incremental_map 59 -> 60 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(60..60 src has 1..60) v4 -- 0x558c5822d200 con 0x558c54c19c00 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 60 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 59 -> 60 (shared) 2019-12-07T19:34:37.626+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 60/28 ECSubWrite(tid=55, reqid=client.4540.0:449, at_version=60'50, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c5831b000 con 0x558c54c19c00 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.add_oid 1:33f5bf01:::smithi13140796-2:25 1b,20,21,25 2019-12-07T19:34:37.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.get_snaps 1:33f5bf01:::smithi13140796-2:25 got.empty() 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.set_snaps 1:33f5bf01:::smithi13140796-2:25 1b,20,21,25 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_0000000000000021_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.626+0000 7f8f06986700 20 snap_mapper.add_oid set SNA_1_0000000000000025_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.626+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:37.626+0000 7f8f06986700 20 trim 60'49 (57'42) clone 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [25,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.626+0000 7f8f06986700 20 trim 60'50 (57'42) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:449 2019-12-07T19:34:37.625406+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 59'48 (0'0,59'48] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log log((0'0,59'48], crt=59'48) [60'49 (57'42) clone 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [25,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,60'50 (57'42) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:449 2019-12-07T19:34:37.625406+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0] 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'49 (0'0,60'49] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] add_log_entry 60'49 (57'42) clone 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [25,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] add_log_entry 60'50 (57'42) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:449 2019-12-07T19:34:37.625406+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log approx pg log length = 50 2019-12-07T19:34:37.626+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:37.626+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:37.626+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 60'49, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58748f00 seq 68 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:25# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0x0000000000000025ffffffffffffffff'o' 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:37.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:25# 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1282 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.onode(0x558c58295400).flush flush done 2019-12-07T19:34:37.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# 0x0 2019-12-07T19:34:37.626+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:25# 0x0~63000 -> 0x0~63000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x0~2000: 0x0~2000 Blob(0x558c58b792d0 blob([0xa47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10872 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b792d0 blob([0xa47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b792d0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c5898b2d0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5898b2d0) get_ref 0x0~2000 Blob(0x558c5898b2d0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5898b2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x0~2000: 0x0~2000 Blob(0x558c5898b2d0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x2000~2000: 0x0~2000 Blob(0x558c58b793b0 blob([0xa49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10873 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b793b0 blob([0xa49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b793b0 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551062a0 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c551062a0) get_ref 0x0~2000 Blob(0x558c551062a0 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c551062a0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c551062a0 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x4000~2000: 0x0~2000 Blob(0x558c58b79490 blob([0xa4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10874 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79490 blob([0xa4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79490 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c57566850 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c57566850) get_ref 0x0~2000 Blob(0x558c57566850 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c57566850) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57566850 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x6000~2000: 0x0~2000 Blob(0x558c58b79570 blob([0xa4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10875 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79570 blob([0xa4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79570 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c5898b8f0 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5898b8f0) get_ref 0x0~2000 Blob(0x558c5898b8f0 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5898b8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c5898b8f0 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x8000~2000: 0x0~2000 Blob(0x558c58b79650 blob([0xa4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10876 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79650 blob([0xa4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79650 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c5809b1f0 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5809b1f0) get_ref 0x0~2000 Blob(0x558c5809b1f0 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5809b1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5809b1f0 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0xa000~2000: 0x0~2000 Blob(0x558c58b79730 blob([0xa51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10877 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79730 blob([0xa51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79730 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c5809b340 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5809b340) get_ref 0x0~2000 Blob(0x558c5809b340 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5809b340) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5809b340 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0xc000~2000: 0x0~2000 Blob(0x558c58b79810 blob([0xa53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10878 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79810 blob([0xa53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79810 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c574d3260 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c574d3260) get_ref 0x0~2000 Blob(0x558c574d3260 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c574d3260) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c574d3260 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0xe000~2000: 0x0~2000 Blob(0x558c58b798f0 blob([0xa55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10879 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b798f0 blob([0xa55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b798f0 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c5794d730 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5794d730) get_ref 0x0~2000 Blob(0x558c5794d730 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c5794d730) get_ref init 0x2000, 1000 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c5794d730 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x10000~2000: 0x0~2000 Blob(0x558c58b799d0 blob([0xa57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10880 2019-12-07T19:34:37.626+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b799d0 blob([0xa57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 sbid 0x0)) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b799d0 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=1)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c580a49a0 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=2)))) 2019-12-07T19:34:37.626+0000 7f8f06986700 20 bluestore.blob(0x558c580a49a0) get_ref 0x0~2000 Blob(0x558c580a49a0 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c580a49a0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c580a49a0 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x12000~2000: 0x0~2000 Blob(0x558c58b79ab0 blob([0xa59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10881 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79ab0 blob([0xa59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79ab0 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c57396770 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c57396770) get_ref 0x0~2000 Blob(0x558c57396770 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c57396770) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c57396770 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x14000~2000: 0x0~2000 Blob(0x558c58b79b90 blob([0xa5b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10882 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79b90 blob([0xa5b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79b90 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c5794de30 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c5794de30) get_ref 0x0~2000 Blob(0x558c5794de30 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c5794de30) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c5794de30 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x16000~2000: 0x0~2000 Blob(0x558c58b79c70 blob([0xa5d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10883 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79c70 blob([0xa5d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79c70 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c574d2af0 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c574d2af0) get_ref 0x0~2000 Blob(0x558c574d2af0 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c574d2af0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c574d2af0 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x18000~2000: 0x0~2000 Blob(0x558c58b79d50 blob([0xa5f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10884 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79d50 blob([0xa5f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79d50 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c574d3810 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c574d3810) get_ref 0x0~2000 Blob(0x558c574d3810 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c574d3810) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c574d3810 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c58b79e30 blob([0xa61000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10885 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79e30 blob([0xa61000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79e30 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c5794c0e0 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c5794c0e0) get_ref 0x0~2000 Blob(0x558c5794c0e0 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c5794c0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c5794c0e0 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c58b79f10 blob([0xa63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10886 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b79f10 blob([0xa63000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b79f10 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c57397ce0 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c57397ce0) get_ref 0x0~2000 Blob(0x558c57397ce0 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c57397ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57397ce0 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c58b80000 blob([0xa65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10887 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80000 blob([0xa65000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80000 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c54d3d2d0 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c54d3d2d0) get_ref 0x0~2000 Blob(0x558c54d3d2d0 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c54d3d2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c54d3d2d0 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x20000~2000: 0x0~2000 Blob(0x558c58b800e0 blob([0xa67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10888 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b800e0 blob([0xa67000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b800e0 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c574d3340 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c574d3340) get_ref 0x0~2000 Blob(0x558c574d3340 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c574d3340) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c574d3340 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x22000~2000: 0x0~2000 Blob(0x558c58b801c0 blob([0xa69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10889 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b801c0 blob([0xa69000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b801c0 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c580a47e0 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c580a47e0) get_ref 0x0~2000 Blob(0x558c580a47e0 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c580a47e0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c580a47e0 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x24000~2000: 0x0~2000 Blob(0x558c58b802a0 blob([0xa6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10890 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b802a0 blob([0xa6b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b802a0 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c58b78230 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c58b78230) get_ref 0x0~2000 Blob(0x558c58b78230 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c58b78230) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c58b78230 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x26000~2000: 0x0~2000 Blob(0x558c58b80380 blob([0xa6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10891 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80380 blob([0xa6d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80380 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c58b78150 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c58b78150) get_ref 0x0~2000 Blob(0x558c58b78150 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c58b78150) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c58b78150 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x28000~2000: 0x0~2000 Blob(0x558c58b80460 blob([0xa6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10892 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80460 blob([0xa6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80460 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c580a4af0 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c580a4af0) get_ref 0x0~2000 Blob(0x558c580a4af0 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c580a4af0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c580a4af0 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c58b80540 blob([0xa71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10893 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80540 blob([0xa71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80540 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106230 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106230) get_ref 0x0~2000 Blob(0x558c55106230 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106230) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c55106230 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c58b80620 blob([0xa73000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10894 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80620 blob([0xa73000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80620 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106690 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106690) get_ref 0x0~2000 Blob(0x558c55106690 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106690) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c55106690 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c58b80700 blob([0xa75000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10895 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80700 blob([0xa75000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80700 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551061c0 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551061c0) get_ref 0x0~2000 Blob(0x558c551061c0 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551061c0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c551061c0 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x30000~2000: 0x0~2000 Blob(0x558c58b807e0 blob([0xa77000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10896 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b807e0 blob([0xa77000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b807e0 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551060e0 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551060e0) get_ref 0x0~2000 Blob(0x558c551060e0 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551060e0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c551060e0 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x32000~2000: 0x0~2000 Blob(0x558c58b808c0 blob([0xa79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10897 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b808c0 blob([0xa79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b808c0 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106150 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106150) get_ref 0x0~2000 Blob(0x558c55106150 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106150) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c55106150 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x34000~2000: 0x0~2000 Blob(0x558c58b809a0 blob([0xa7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10898 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b809a0 blob([0xa7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b809a0 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551064d0 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551064d0) get_ref 0x0~2000 Blob(0x558c551064d0 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551064d0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c551064d0 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x36000~2000: 0x0~2000 Blob(0x558c58b80a80 blob([0xa7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10899 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80a80 blob([0xa7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80a80 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551063f0 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551063f0) get_ref 0x0~2000 Blob(0x558c551063f0 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551063f0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c551063f0 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x38000~2000: 0x0~2000 Blob(0x558c58b80b60 blob([0xa7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10900 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80b60 blob([0xa7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80b60 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106460 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106460) get_ref 0x0~2000 Blob(0x558c55106460 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106460) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c55106460 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c58b80c40 blob([0xa81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10901 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80c40 blob([0xa81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80c40 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551065b0 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551065b0) get_ref 0x0~2000 Blob(0x558c551065b0 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551065b0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c551065b0 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c58b80d20 blob([0xa83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10902 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80d20 blob([0xa83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80d20 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106620 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106620) get_ref 0x0~2000 Blob(0x558c55106620 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106620) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c55106620 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c58b80e00 blob([0xa85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10903 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80e00 blob([0xa85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80e00 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106700 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106700) get_ref 0x0~2000 Blob(0x558c55106700 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106700) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c55106700 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x40000~2000: 0x0~2000 Blob(0x558c58b80ee0 blob([0xa87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10904 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80ee0 blob([0xa87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80ee0 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106770 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106770) get_ref 0x0~2000 Blob(0x558c55106770 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106770) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c55106770 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x42000~2000: 0x0~2000 Blob(0x558c58b80fc0 blob([0xa89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10905 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b80fc0 blob([0xa89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b80fc0 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551067e0 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551067e0) get_ref 0x0~2000 Blob(0x558c551067e0 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551067e0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c551067e0 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x44000~2000: 0x0~2000 Blob(0x558c58b810a0 blob([0xa8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10906 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b810a0 blob([0xa8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b810a0 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106850 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106850) get_ref 0x0~2000 Blob(0x558c55106850 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106850) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c55106850 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x46000~2000: 0x0~2000 Blob(0x558c58b81180 blob([0xa8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10907 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81180 blob([0xa8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81180 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551068c0 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551068c0) get_ref 0x0~2000 Blob(0x558c551068c0 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551068c0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c551068c0 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x48000~2000: 0x0~2000 Blob(0x558c58b81260 blob([0xa8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10908 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81260 blob([0xa8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81260 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106930 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106930) get_ref 0x0~2000 Blob(0x558c55106930 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106930) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c55106930 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c58b81340 blob([0xa91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10909 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81340 blob([0xa91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81340 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c551069a0 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551069a0) get_ref 0x0~2000 Blob(0x558c551069a0 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c551069a0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c551069a0 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c58b81420 blob([0xa93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10910 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81420 blob([0xa93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81420 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106a10 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106a10) get_ref 0x0~2000 Blob(0x558c55106a10 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106a10) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c55106a10 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c58b81500 blob([0xa95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10911 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81500 blob([0xa95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81500 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106a80 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106a80) get_ref 0x0~2000 Blob(0x558c55106a80 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106a80) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c55106a80 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x50000~2000: 0x0~2000 Blob(0x558c58b815e0 blob([0xa97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10912 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b815e0 blob([0xa97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b815e0 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106af0 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106af0) get_ref 0x0~2000 Blob(0x558c55106af0 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106af0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c55106af0 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x52000~2000: 0x0~2000 Blob(0x558c58b816c0 blob([0xa99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10913 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b816c0 blob([0xa99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b816c0 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106b60 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106b60) get_ref 0x0~2000 Blob(0x558c55106b60 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106b60) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c55106b60 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x54000~2000: 0x0~2000 Blob(0x558c58b817a0 blob([0xa9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10914 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b817a0 blob([0xa9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b817a0 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106bd0 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106bd0) get_ref 0x0~2000 Blob(0x558c55106bd0 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c55106bd0 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x56000~2000: 0x0~2000 Blob(0x558c58b81880 blob([0xa9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10915 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81880 blob([0xa9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81880 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106c40 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106c40) get_ref 0x0~2000 Blob(0x558c55106c40 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106c40) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c55106c40 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x58000~2000: 0x0~2000 Blob(0x558c58b81960 blob([0xa9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10916 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81960 blob([0xa9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81960 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106cb0 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106cb0) get_ref 0x0~2000 Blob(0x558c55106cb0 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c55106cb0 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c58b81a40 blob([0xaa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10917 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81a40 blob([0xaa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81a40 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106d20 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106d20) get_ref 0x0~2000 Blob(0x558c55106d20 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106d20) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c55106d20 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c58b81b20 blob([0xaa3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10918 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81b20 blob([0xaa3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81b20 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c55106d90 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106d90) get_ref 0x0~2000 Blob(0x558c55106d90 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c55106d90) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c55106d90 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x5e000~2000: 0x0~2000 Blob(0x558c58b81c00 blob([0xaa5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10919 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81c00 blob([0xaa5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81c00 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c578f0e00 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c578f0e00) get_ref 0x0~2000 Blob(0x558c578f0e00 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c578f0e00) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x5e000~2000: 0x0~2000 Blob(0x558c578f0e00 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x60000~2000: 0x0~2000 Blob(0x558c58b81ce0 blob([0xaa7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10920 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81ce0 blob([0xaa7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81ce0 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c578f1ea0 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c578f1ea0) get_ref 0x0~2000 Blob(0x558c578f1ea0 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c578f1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x60000~2000: 0x0~2000 Blob(0x558c578f1ea0 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup src 0x62000~1000: 0x0~1000 Blob(0x558c58b81dc0 blob([0xaa9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10921 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared Blob(0x558c58b81dc0 blob([0xaa9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 sbid 0x0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) make_blob_shared now Blob(0x558c58b81dc0 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=1)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup new Blob(0x558c578f1f80 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c578f1f80) get_ref 0x0~1000 Blob(0x558c578f1f80 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.blob(0x558c578f1f80) get_ref init 0x1000, 1000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dup dst 0x62000~1000: 0x0~1000 Blob(0x558c578f1f80 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=2)))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) dirty_range mark inline shard dirty 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dirty_range mark inline shard dirty 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:25# = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# snapset 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# snapset = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# 1 keys 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# 1 keys = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# hinfo_key (42 bytes) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# hinfo_key (42 bytes) = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' <- OBJ_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' <- SNA_1_000000000000001B_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' <- SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' <- SNA_1_0000000000000021_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000025_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' <- SNA_1_0000000000000025_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000060.00000000000000000049' <- 0000000060.00000000000000000049 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000060.00000000000000000050' <- 0000000060.00000000000000000050 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58748f00 cost 8693 (1 ios * 4000 + 4693 bytes) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58748f00 onodes 0x558c57bd4c80,0x558c58295400 shared_blobs 0x558c58b79340,0x558c58b79420,0x558c58b79500,0x558c58b795e0,0x558c58b796c0,0x558c58b797a0,0x558c58b79880,0x558c58b79960,0x558c58b79a40,0x558c58b79b20,0x558c58b79c00,0x558c58b79ce0,0x558c58b79dc0,0x558c58b79ea0,0x558c58b79f80,0x558c58b80070,0x558c58b80150,0x558c58b80230,0x558c58b80310,0x558c58b803f0,0x558c58b804d0,0x558c58b805b0,0x558c58b80690,0x558c58b80770,0x558c58b80850,0x558c58b80930,0x558c58b80a10,0x558c58b80af0,0x558c58b80bd0,0x558c58b80cb0,0x558c58b80d90,0x558c58b80e70,0x558c58b80f50,0x558c58b81030,0x558c58b81110,0x558c58b811f0,0x558c58b812d0,0x558c58b813b0,0x558c58b81490,0x558c58b81570,0x558c58b81650,0x558c58b81730,0x558c58b81810,0x558c58b818f0,0x558c58b819d0,0x558c58b81ab0,0x558c58b81b90,0x558c58b81c70,0x558c58b81d50,0x558c58b81e30 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) update 0#1:33f5bf01:::smithi13140796-2:25# 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) update inline shard 1350 bytes from 50 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore.extentmap(0x558c57bd4d88) reshard 0x[0,ffffffff) of 0 shards on 0#1:33f5bf01:::smithi13140796-2:25# 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard new shard 0x0 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard new shard 0x24000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard new shard 0x48000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard old [] 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) update 0#1:33f5bf01:::smithi13140796-2:25# force 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) update shard 0x48000 is 380 bytes (was 0) from 14 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:25# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update inline shard 1350 bytes from 50 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore.extentmap(0x558c58295508) reshard 0x[0,ffffffff) of 0 shards on 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard new shard 0x0 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard new shard 0x24000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard new shard 0x48000 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard old [] 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head# force 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update shard 0x48000 is 380 bytes (was 0) from 14 extents 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 596 (594 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a78 is 11 SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a79 is 11 SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7a is 11 SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7b is 11 SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7c is 11 SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7d is 11 SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7e is 11 SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7f is 11 SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a80 is 11 SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a81 is 11 SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a82 is 11 SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a83 is 11 SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a84 is 11 SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a85 is 11 SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a86 is 11 SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a87 is 11 SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a88 is 11 SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a89 is 11 SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8a is 11 SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8b is 11 SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8c is 11 SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8d is 11 SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8e is 11 SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8f is 11 SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a90 is 11 SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a91 is 11 SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a92 is 11 SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a93 is 11 SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a94 is 11 SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a95 is 11 SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a96 is 11 SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a97 is 11 SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a98 is 11 SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a99 is 11 SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9a is 11 SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9b is 11 SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9c is 11 SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9d is 11 SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9e is 11 SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9f is 11 SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa0 is 11 SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa1 is 11 SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa2 is 11 SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa3 is 11 SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa4 is 11 SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa5 is 11 SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa6 is 11 SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa7 is 11 SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa8 is 11 SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa9 is 11 SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=2))) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58748f00 allocated 0x[] released 0x[] 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748f00 prepare 2019-12-07T19:34:37.627+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58748f00 2019-12-07T19:34:37.627+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748f00 io_done 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op_applied version 60'50 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] eval_repop repgather(0x558c576b6fc0 60'50 rep_tid=55 committed?=0 r=0) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c54e78420 finish 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) queued 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c54e79a20 prio 63 cost 0 latency 0.003381 osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 pg pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: osd_op(client.4540.0:450 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8379 < min_flush_size 524288 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op: op osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(60'50 client.4540.0:449 dirty|data_digest s 1622016 uv 50 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(60'50 client.4540.0:449 dirty|data_digest s 1622016 uv 50 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:37.627+0000 7f8f06986700 25 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(60'50 client.4540.0:449 dirty|data_digest s 1622016 uv 50 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=2 w=0)) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx 0x558c58237500 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd4c80 had 1 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [rollback 20] ov 60'50 av 60'51 snapc 25=[25,21,20,1b] snapset 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8402 < min_flush_size 524288 2019-12-07T19:34:37.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8402 < min_flush_size 524288 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [rollback 20] 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op rollback 20 2019-12-07T19:34:37.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _rollback_to 1:33f5bf01:::smithi13140796-2:head snapid 20 2019-12-07T19:34:37.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4c8f62~20d2 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4c8f62 0x4c8f62~20d2 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] find_object_context 1:33f5bf01:::smithi13140796-2:20 @20 snapset 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x498f62 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: found obc in cache: 0x558c579e66c0 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf62 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:37.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x34 and padding block with 0xfcc 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: 0x558c579e66c0 1:33f5bf01:::smithi13140796-2:25 rwstate(write n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:25(60'49 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] find_object_context 1:33f5bf01:::smithi13140796-2:25 snapset 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] find_object_context 1:33f5bf01:::smithi13140796-2:25 clone_snaps [25,21,20,1b] contains 20 -- HIT 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _rollback_to deleting smithi13140796-2 and rolling back to old snap 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _rollback_to clearing omap flag on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=25=[]:{25=[25,21,20,1b]} snapc=25=[25,21,20,1b] 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x558c58237500 op modify 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] set mtime to 2019-12-07T19:34:37.625424+0000 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] final snapset 25=[]:{25=[25,21,20,1b]} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [1622016~18446744073707929599], clean_omap: 0, new_object: 0 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx alloc reply 0x558c57bd4a00 result 0 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] calc_trim_to_aggressive limit = 59'48 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op order client.4540 tid 450 last was 449 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] new_repop rep_tid 56 on osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] new_repop: repgather(0x558c576b6ea0 0'0 rep_tid=56 committed?=0 r=0) 2019-12-07T19:34:37.627+0000 7f8f06986700 7 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] issue_repop rep_tid 56 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=60'51 tt=0'0 tid=56 reqid=client.4540.0:450 client_op=osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=60'51 tt=0'0 tid=56 reqid=client.4540.0:450 client_op=osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=60'51 tt=0'0 tid=56 reqid=client.4540.0:450 client_op=osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.627+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=60'51 tt=0'0 tid=56 reqid=client.4540.0:450 client_op=osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): new_size start 1622016 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_append: {} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=60'51 tt=0'0 tid=56 reqid=client.4540.0:450 client_op=osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.627+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:37.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 60/28 ECSubWrite(tid=56, reqid=client.4540.0:450, at_version=60'51, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c575bf800 con 0x558c54c64000 2019-12-07T19:34:37.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 60/28 ECSubWrite(tid=56, reqid=client.4540.0:450, at_version=60'51, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c575c1800 con 0x558c54c19800 2019-12-07T19:34:37.627+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 60/28 ECSubWrite(tid=56, reqid=client.4540.0:450, at_version=60'51, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c575c0800 con 0x558c54c5bc00 2019-12-07T19:34:37.628+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 60/28 ECSubWrite(tid=56, reqid=client.4540.0:450, at_version=60'51, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c5748d000 con 0x558c54c5b400 2019-12-07T19:34:37.628+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 60/28 ECSubWrite(tid=56, reqid=client.4540.0:450, at_version=60'51, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c5748e800 con 0x558c54c19c00 2019-12-07T19:34:37.628+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:37.628+0000 7f8f06986700 20 trim 60'51 (60'50) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:450 2019-12-07T19:34:37.625424+0000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 0, new_object: 0 2019-12-07T19:34:37.628+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'50 (0'0,60'50] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log log((0'0,60'50], crt=59'48) [60'51 (60'50) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:450 2019-12-07T19:34:37.625424+0000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 0, new_object: 0] 2019-12-07T19:34:37.628+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'50 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] add_log_entry 60'51 (60'50) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:450 2019-12-07T19:34:37.625424+0000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 0, new_object: 0 2019-12-07T19:34:37.628+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'50 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log approx pg log length = 51 2019-12-07T19:34:37.628+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'50 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:37.628+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:37.628+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 516 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c548cc200 con 0x558c54c64000 2019-12-07T19:34:37.628+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 60'51, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:37.628+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57f3ef20 prio 196 cost 0 latency 0.000015 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.628+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58749200 seq 69 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) get_onode oid 0#1:33f5bf01:::smithi13140796-2:head#33 key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0xfffffffffffffffe0000000000000033'o' 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.cs0_head 0x558c54d38000) r -2 v.len 0 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4cb034 2019-12-07T19:34:37.628+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:head#33 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# -> 0#1:33f5bf01:::smithi13140796-2:head#33 = 0 2019-12-07T19:34:37.628+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# -> 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1283 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.onode(0x558c57bd4c80).flush flush done 2019-12-07T19:34:37.628+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0 2019-12-07T19:34:37.628+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# -> 0#1:33f5bf01:::smithi13140796-2:head# 0x0~63000 -> 0x0~63000 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000497858s (0.000000104s flush + 0.000497754s kv commit) 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x0~2000: 0x0~2000 Blob(0x558c5898b2d0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c585d4fc0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58748f00> 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c585d4fc0) get_ref 0x0~2000 Blob(0x558c585d4fc0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c585d4fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748f00 kv_submitted 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58748f00 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c585d4fc0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58748f00 finishing 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c551062a0 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58748f00 onodes 0x558c57bd4c80,0x558c58295400 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58748f00 done 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58226e00 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58749200 prepare 2019-12-07T19:34:37.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58748f00 [] 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226e00) get_ref 0x0~2000 Blob(0x558c58226e00 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226e00) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c58226e00 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57566850 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58226e70 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226e70) get_ref 0x0~2000 Blob(0x558c58226e70 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226e70) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c58226e70 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c5898b8f0 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58226f50 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226f50) get_ref 0x0~2000 Blob(0x558c58226f50 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226f50) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c58226f50 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c5809b1f0 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58226fc0 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226fc0) get_ref 0x0~2000 Blob(0x558c58226fc0 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58226fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c58226fc0 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c5809b340 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227030 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227030) get_ref 0x0~2000 Blob(0x558c58227030 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227030) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c58227030 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c574d3260 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582270a0 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582270a0) get_ref 0x0~2000 Blob(0x558c582270a0 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582270a0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c582270a0 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c5794d730 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227110 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227110) get_ref 0x0~2000 Blob(0x558c58227110 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227110) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c58227110 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c580a49a0 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227180 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227180) get_ref 0x0~2000 Blob(0x558c58227180 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227180) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c58227180 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57396770 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582271f0 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582271f0) get_ref 0x0~2000 Blob(0x558c582271f0 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582271f0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c582271f0 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x14000~2000: 0x0~2000 Blob(0x558c5794de30 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227260 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227260) get_ref 0x0~2000 Blob(0x558c58227260 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227260) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c58227260 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x16000~2000: 0x0~2000 Blob(0x558c574d2af0 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582272d0 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582272d0) get_ref 0x0~2000 Blob(0x558c582272d0 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582272d0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c582272d0 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x18000~2000: 0x0~2000 Blob(0x558c574d3810 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227340 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227340) get_ref 0x0~2000 Blob(0x558c58227340 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227340) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c58227340 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c5794c0e0 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582273b0 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582273b0) get_ref 0x0~2000 Blob(0x558c582273b0 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582273b0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c582273b0 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57397ce0 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227420 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227420) get_ref 0x0~2000 Blob(0x558c58227420 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227420) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c58227420 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c54d3d2d0 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227490 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227490) get_ref 0x0~2000 Blob(0x558c58227490 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227490) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c58227490 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x20000~2000: 0x0~2000 Blob(0x558c574d3340 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227500 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227500) get_ref 0x0~2000 Blob(0x558c58227500 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227500) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c58227500 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x22000~2000: 0x0~2000 Blob(0x558c580a47e0 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227570 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227570) get_ref 0x0~2000 Blob(0x558c58227570 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227570) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c58227570 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x24000~2000: 0x0~2000 Blob(0x558c58b78230 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582275e0 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582275e0) get_ref 0x0~2000 Blob(0x558c582275e0 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582275e0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c582275e0 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x26000~2000: 0x0~2000 Blob(0x558c58b78150 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227650 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227650) get_ref 0x0~2000 Blob(0x558c58227650 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227650) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c58227650 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x28000~2000: 0x0~2000 Blob(0x558c580a4af0 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582276c0 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582276c0) get_ref 0x0~2000 Blob(0x558c582276c0 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582276c0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c582276c0 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c55106230 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227730 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227730) get_ref 0x0~2000 Blob(0x558c58227730 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227730) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c58227730 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c55106690 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582277a0 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582277a0) get_ref 0x0~2000 Blob(0x558c582277a0 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582277a0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c582277a0 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c551061c0 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227810 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227810) get_ref 0x0~2000 Blob(0x558c58227810 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227810) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c58227810 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x30000~2000: 0x0~2000 Blob(0x558c551060e0 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227880 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227880) get_ref 0x0~2000 Blob(0x558c58227880 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227880) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c58227880 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x32000~2000: 0x0~2000 Blob(0x558c55106150 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c54d75810 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c54d75810) get_ref 0x0~2000 Blob(0x558c54d75810 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c54d75810) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c54d75810 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x34000~2000: 0x0~2000 Blob(0x558c551064d0 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c54d75260 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c54d75260) get_ref 0x0~2000 Blob(0x558c54d75260 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c54d75260) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c54d75260 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x36000~2000: 0x0~2000 Blob(0x558c551063f0 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58b78e70 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58b78e70) get_ref 0x0~2000 Blob(0x558c58b78e70 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58b78e70) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c58b78e70 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x38000~2000: 0x0~2000 Blob(0x558c55106460 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58b788c0 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58b788c0) get_ref 0x0~2000 Blob(0x558c58b788c0 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58b788c0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c58b788c0 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c551065b0 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c5898ba40 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c5898ba40) get_ref 0x0~2000 Blob(0x558c5898ba40 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c5898ba40) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c5898ba40 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c55106620 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c5898b1f0 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c5898b1f0) get_ref 0x0~2000 Blob(0x558c5898b1f0 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c5898b1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c5898b1f0 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c55106700 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c57db4e70 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c57db4e70) get_ref 0x0~2000 Blob(0x558c57db4e70 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c57db4e70) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c57db4e70 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x40000~2000: 0x0~2000 Blob(0x558c55106770 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582278f0 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582278f0) get_ref 0x0~2000 Blob(0x558c582278f0 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582278f0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c582278f0 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x42000~2000: 0x0~2000 Blob(0x558c551067e0 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227960 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227960) get_ref 0x0~2000 Blob(0x558c58227960 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227960) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c58227960 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x44000~2000: 0x0~2000 Blob(0x558c55106850 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c582279d0 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582279d0) get_ref 0x0~2000 Blob(0x558c582279d0 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c582279d0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c582279d0 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x46000~2000: 0x0~2000 Blob(0x558c551068c0 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 296 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c576b5400 con 0x558c54c5bc00 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227a40 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227a40) get_ref 0x0~2000 Blob(0x558c58227a40 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227a40) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c58227a40 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c581cdce0 prio 196 cost 0 latency 0.000016 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x48000~2000: 0x0~2000 Blob(0x558c55106930 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227ab0 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227ab0) get_ref 0x0~2000 Blob(0x558c58227ab0 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c58227ab0 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c551069a0 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227b20 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227b20) get_ref 0x0~2000 Blob(0x558c58227b20 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227b20) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c58227b20 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c55106a10 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227b90 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227b90) get_ref 0x0~2000 Blob(0x558c58227b90 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227b90) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c58227b90 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c55106a80 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227c00 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227c00) get_ref 0x0~2000 Blob(0x558c58227c00 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227c00) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c58227c00 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x50000~2000: 0x0~2000 Blob(0x558c55106af0 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227c70 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227c70) get_ref 0x0~2000 Blob(0x558c58227c70 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227c70) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c58227c70 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x52000~2000: 0x0~2000 Blob(0x558c55106b60 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227ce0 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227ce0) get_ref 0x0~2000 Blob(0x558c58227ce0 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c58227ce0 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x54000~2000: 0x0~2000 Blob(0x558c55106bd0 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227d50 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227d50) get_ref 0x0~2000 Blob(0x558c58227d50 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227d50) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c58227d50 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x56000~2000: 0x0~2000 Blob(0x558c55106c40 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227dc0 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227dc0) get_ref 0x0~2000 Blob(0x558c58227dc0 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c58227dc0 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x58000~2000: 0x0~2000 Blob(0x558c55106cb0 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227e30 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227e30) get_ref 0x0~2000 Blob(0x558c58227e30 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227e30) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c58227e30 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c55106d20 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227ea0 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227ea0) get_ref 0x0~2000 Blob(0x558c58227ea0 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c58227ea0 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c55106d90 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227f10 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227f10) get_ref 0x0~2000 Blob(0x558c58227f10 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227f10) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c58227f10 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x5e000~2000: 0x0~2000 Blob(0x558c578f0e00 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c58227f80 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227f80) get_ref 0x0~2000 Blob(0x558c58227f80 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c58227f80) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x5e000~2000: 0x0~2000 Blob(0x558c58227f80 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x60000~2000: 0x0~2000 Blob(0x558c578f1ea0 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c585e0000 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c585e0000) get_ref 0x0~2000 Blob(0x558c585e0000 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c585e0000) get_ref init 0x2000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x60000~2000: 0x0~2000 Blob(0x558c585e0000 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup src 0x62000~1000: 0x0~1000 Blob(0x558c578f1f80 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=2)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup new Blob(0x558c585e0070 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c585e0070) get_ref 0x0~1000 Blob(0x558c585e0070 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.blob(0x558c585e0070) get_ref init 0x1000, 1000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) dup dst 0x62000~1000: 0x0~1000 Blob(0x558c585e0070 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=3)))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) dirty_range mark inline shard dirty 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# -> 0#1:33f5bf01:::smithi13140796-2:head# = 0 2019-12-07T19:34:37.628+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:37.628+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:37.628+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000060.00000000000000000051' <- 0000000060.00000000000000000051 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58749200 cost 5785 (1 ios * 4000 + 1785 bytes) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58749200 onodes 0x558c57bd5400,0x558c58295400 shared_blobs 0x558c58b79340,0x558c58b79420,0x558c58b79500,0x558c58b795e0,0x558c58b796c0,0x558c58b797a0,0x558c58b79880,0x558c58b79960,0x558c58b79a40,0x558c58b79b20,0x558c58b79c00,0x558c58b79ce0,0x558c58b79dc0,0x558c58b79ea0,0x558c58b79f80,0x558c58b80070,0x558c58b80150,0x558c58b80230,0x558c58b80310,0x558c58b803f0,0x558c58b804d0,0x558c58b805b0,0x558c58b80690,0x558c58b80770,0x558c58b80850,0x558c58b80930,0x558c58b80a10,0x558c58b80af0,0x558c58b80bd0,0x558c58b80cb0,0x558c58b80d90,0x558c58b80e70,0x558c58b80f50,0x558c58b81030,0x558c58b81110,0x558c58b811f0,0x558c58b812d0,0x558c58b813b0,0x558c58b81490,0x558c58b81570,0x558c58b81650,0x558c58b81730,0x558c58b81810,0x558c58b818f0,0x558c58b819d0,0x558c58b81ab0,0x558c58b81b90,0x558c58b81c70,0x558c58b81d50,0x558c58b81e30 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) update inline shard 1350 bytes from 50 extents 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore.extentmap(0x558c57bd5508) reshard 0x[0,ffffffff) of 0 shards on 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard new shard 0x0 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard new shard 0x24000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard new shard 0x48000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard new [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard old [] 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard fin [0x0(0x0 bytes),0x24000(0x0 bytes),0x48000(0x0 bytes)] 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) update 0#1:33f5bf01:::smithi13140796-2:head# force 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) update shard 0x0 is 489 bytes (was 0) from 18 extents 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) update shard 0x24000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) update shard 0x48000 is 380 bytes (was 0) from 14 extents 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 596 (594 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update 0#1:33f5bf01:::smithi13140796-2:head#33 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update shard 0x0 is 489 bytes (was 489) from 18 extents 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update shard 0x24000 is 491 bytes (was 491) from 18 extents 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore.extentmap(0x558c58295508) update shard 0x48000 is 380 bytes (was 380) from 14 extents 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head#33 is 596 (594 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a78 is 11 SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a79 is 11 SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7a is 11 SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7b is 11 SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7c is 11 SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7d is 11 SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7e is 11 SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7f is 11 SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a80 is 11 SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a81 is 11 SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a82 is 11 SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a83 is 11 SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a84 is 11 SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a85 is 11 SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a86 is 11 SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a87 is 11 SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a88 is 11 SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a89 is 11 SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8a is 11 SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8b is 11 SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8c is 11 SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8d is 11 SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8e is 11 SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8f is 11 SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a90 is 11 SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a91 is 11 SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a92 is 11 SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a93 is 11 SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a94 is 11 SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a95 is 11 SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a96 is 11 SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a97 is 11 SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a98 is 11 SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a99 is 11 SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9a is 11 SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9b is 11 SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9c is 11 SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9d is 11 SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9e is 11 SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9f is 11 SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa0 is 11 SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa1 is 11 SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa2 is 11 SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa3 is 11 SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa4 is 11 SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa5 is 11 SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa6 is 11 SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa7 is 11 SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa8 is 11 SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa9 is 11 SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=3))) 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58749200 allocated 0x[] released 0x[] 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749200 prepare 2019-12-07T19:34:37.628+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58749200 2019-12-07T19:34:37.628+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749200 io_done 2019-12-07T19:34:37.628+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'50 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op_applied version 60'51 2019-12-07T19:34:37.628+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] eval_repop repgather(0x558c576b6ea0 60'51 rep_tid=56 committed?=0 r=0) 2019-12-07T19:34:37.628+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c54e79a20 finish 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8) prio 63 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.628+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c54e78580 prio 63 cost 0 latency 0.004475 osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 pg pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: osd_op(client.4540.0:451 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op: op osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6443 < min_flush_size 524288 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(write n=2 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(60'51 client.4540.0:450 dirty|data_digest s 1622016 uv 51 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(60'51 client.4540.0:450 dirty|data_digest s 1622016 uv 51 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:37.628+0000 7f8f0297e700 25 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op oi 1:33f5bf01:::smithi13140796-2:head(60'51 client.4540.0:450 dirty|data_digest s 1622016 uv 51 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=3 w=0)) 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd5400 had 1 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx 0x558c58228e00 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6466 < min_flush_size 524288 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6466 < min_flush_size 524288 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx 1:33f5bf01:::smithi13140796-2:head [append 0~0] ov 60'51 av 60'52 snapc 25=[25,21,20,1b] snapset 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4cb034~1942 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4cb034 0x4cb034~1942 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x49b034 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [append 0~0] 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x34 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:37.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x976 and padding block with 0x68a 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op append 0~0 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op 1:33f5bf01:::smithi13140796-2:head [write 1622016~0] 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op write 1622016~0 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] clean_regions modifiedclean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=25=[]:{25=[25,21,20,1b]} snapc=25=[25,21,20,1b] 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:37.628+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x558c58228e00 op modify 2019-12-07T19:34:37.628+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] set mtime to 2019-12-07T19:34:37.625428+0000 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] final snapset 25=[]:{25=[25,21,20,1b]} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] execute_ctx alloc reply 0x558c57572780 result 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] calc_trim_to_aggressive limit = 59'48 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op order client.4540 tid 451 last was 450 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] new_repop rep_tid 57 on osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] new_repop: repgather(0x558c576b70e0 0'0 rep_tid=57 committed?=0 r=0) 2019-12-07T19:34:37.629+0000 7f8f0297e700 7 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] issue_repop rep_tid 57 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=60'52 tt=0'0 tid=57 reqid=client.4540.0:451 client_op=osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=60'52 tt=0'0 tid=57 reqid=client.4540.0:451 client_op=osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=60'52 tt=0'0 tid=57 reqid=client.4540.0:451 client_op=osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=60'52 tt=0'0 tid=57 reqid=client.4540.0:451 client_op=osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): new_size start 1622016 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): to_append: {} 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=60'52 tt=0'0 tid=57 reqid=client.4540.0:451 client_op=osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:37.629+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 60/28 ECSubWrite(tid=57, reqid=client.4540.0:451, at_version=60'52, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c5822e800 con 0x558c54c64000 2019-12-07T19:34:37.629+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 60/28 ECSubWrite(tid=57, reqid=client.4540.0:451, at_version=60'52, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c5822f000 con 0x558c54c19800 2019-12-07T19:34:37.629+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 60/28 ECSubWrite(tid=57, reqid=client.4540.0:451, at_version=60'52, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c5822f800 con 0x558c54c5bc00 2019-12-07T19:34:37.629+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 60/28 ECSubWrite(tid=57, reqid=client.4540.0:451, at_version=60'52, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c58230000 con 0x558c54c5b400 2019-12-07T19:34:37.629+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 60/28 ECSubWrite(tid=57, reqid=client.4540.0:451, at_version=60'52, trim_to=0'0, roll_forward_to=59'48)) v2 -- 0x558c58230800 con 0x558c54c19c00 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 earliest_dup_version = 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 trim 60'52 (60'51) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:451 2019-12-07T19:34:37.625428+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'51 (0'0,60'51] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log log((0'0,60'51], crt=59'48) [60'52 (60'51) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:451 2019-12-07T19:34:37.625428+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0] 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'51 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] add_log_entry 60'52 (60'51) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:451 2019-12-07T19:34:37.625428+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'51 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log approx pg log length = 52 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'51 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:37.629+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 60'52, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585e4000 seq 70 2019-12-07T19:34:37.629+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000060.00000000000000000052' <- 0000000060.00000000000000000052 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585e4000 cost 6003 (1 ios * 4000 + 2003 bytes) 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585e4000 onodes 0x558c57bd5400 shared_blobs 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore.extentmap(0x558c57bd5508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 596 (594 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585e4000 allocated 0x[] released 0x[] 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4000 prepare 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585e4000 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4000 io_done 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 lua=60'51 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] op_applied version 60'52 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] eval_repop repgather(0x558c576b70e0 60'52 rep_tid=57 committed?=0 r=0) 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c54e78580 finish 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.629+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57f3ef20 prio 196 cost 0 latency 0.001531 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.629+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 517 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332c00 con 0x558c54c64000 2019-12-07T19:34:37.629+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57f3f4a0 prio 196 cost 0 latency 0.000012 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.629+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.629+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.629+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:37.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.629+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c581cdce0 prio 196 cost 0 latency 0.001172 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.629+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.629+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c581cdce0 finish 2019-12-07T19:34:37.629+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57f3f4a0 prio 196 cost 0 latency 0.000096 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.629+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57f3f4a0 finish 2019-12-07T19:34:37.629+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 297 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58114e00 con 0x558c54c5bc00 2019-12-07T19:34:37.630+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c581cda20 prio 196 cost 0 latency 0.000019 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 319 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54dfa400 con 0x558c54c5b400 2019-12-07T19:34:37.630+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c581cd600 prio 196 cost 0 latency 0.000009 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4cc976 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001379368s (0.000000125s flush + 0.001379243s kv commit) 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1732 < min_flush_size 524288 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd5400 had 1 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1755 < min_flush_size 524288 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1755 < min_flush_size 524288 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4cc976~6db to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4cc976 0x4cc976~6db to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.630+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c581cda20 prio 196 cost 0 latency 0.000158 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x49c976 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x976 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x51 and padding block with 0xfaf 2019-12-07T19:34:37.630+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c581cda20 finish 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:37.630+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c581cd600 prio 196 cost 0 latency 0.000156 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58749200> 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749200 kv_submitted 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58749200 2019-12-07T19:34:37.630+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749200 finishing 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58749200 onodes 0x558c57bd5400,0x558c58295400 2019-12-07T19:34:37.630+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c581cd600 finish 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58749200 done 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4000 kv_submitted 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58749200 [] 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.630+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:37.630+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4cd051 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:37.630+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000170574s (0.000000080s flush + 0.000170494s kv commit) 2019-12-07T19:34:37.630+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585e4000> 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4000 kv_submitted 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585e4000 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4000 finishing 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585e4000 onodes 0x558c57bd5400 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4000 done 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585e4000 [] 2019-12-07T19:34:37.630+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.630+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:37.630+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 518 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58115200 con 0x558c54c64000 2019-12-07T19:34:37.630+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57f3f8c0 prio 196 cost 0 latency 0.000025 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 209 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58115c00 con 0x558c54c19800 2019-12-07T19:34:37.630+0000 7f8f24c58700 15 osd.7 60 enqueue_op 0x558c5511f4a0 prio 196 cost 0 latency 0.000027 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 210 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b9c00 con 0x558c54c19800 2019-12-07T19:34:37.630+0000 7f8f24c58700 15 osd.7 60 enqueue_op 0x558c5511f760 prio 196 cost 0 latency 0.000012 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.630+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 211 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f800 con 0x558c54c19800 2019-12-07T19:34:37.630+0000 7f8f24c58700 15 osd.7 60 enqueue_op 0x558c5511fa20 prio 196 cost 0 latency 0.000007 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 298 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57613000 con 0x558c54c5bc00 2019-12-07T19:34:37.630+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c581cd080 prio 196 cost 0 latency 0.000024 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.630+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 320 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58513200 con 0x558c54c5b400 2019-12-07T19:34:37.630+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000011 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.630+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.631+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 321 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57612c00 con 0x558c54c5b400 2019-12-07T19:34:37.631+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c581ccc60 prio 196 cost 0 latency 0.000015 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57f3f8c0 prio 196 cost 0 latency 0.000571 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57f3f8c0 finish 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c5511f4a0 prio 196 cost 0 latency 0.000591 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c5511f4a0 finish 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c581cd080 prio 196 cost 0 latency 0.000436 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c581cd080 finish 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c581ccb00 prio 196 cost 0 latency 0.000384 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c581ccb00 finish 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c5511f760 prio 196 cost 0 latency 0.000657 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c5511f760 finish 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c581ccc60 prio 196 cost 0 latency 0.000209 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c581ccc60 finish 2019-12-07T19:34:37.631+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c5511fa20 prio 196 cost 0 latency 0.000746 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.631+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c5511fa20 finish 2019-12-07T19:34:37.631+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 117 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c58a00 con 0x558c54c19c00 2019-12-07T19:34:37.632+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57efe420 prio 196 cost 0 latency 0.000015 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.632+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 118 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518000 con 0x558c54c19c00 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.632+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57effe40 prio 196 cost 0 latency 0.000008 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.632+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57efe420 prio 196 cost 0 latency 0.000048 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=55, last_complete=60'50, committed=1, applied=1)) v2 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=60'50 tt=0'0 tid=55 reqid=client.4540.0:449 client_op=osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=59'48 crt=59'48 lcod 58'46 mlcod 58'46 active+clean ps=[e~1,15~1,1a~1,23~1]] repop_all_committed: repop tid 55 all committed 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] eval_repop repgather(0x558c576b6fc0 60'50 rep_tid=55 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] commit: repgather(0x558c576b6fc0 60'50 rep_tid=55 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] log_op_stats osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 inb 0 outb 0 lat 0.008078 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] sending reply on osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 0x558c57bd4780 2019-12-07T19:34:37.632+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(449 smithi13140796-2 [append 0~0] v60'50 uv50 ondisk = 0) v8 -- 0x558c57bd4780 con 0x558c54c9f000 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:37.632+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 119 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57612800 con 0x558c54c19c00 2019-12-07T19:34:37.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 59:204 2019-12-07T19:34:37.632+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57eff1e0 prio 196 cost 0 latency 0.000011 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] removing repgather(0x558c576b6fc0 60'50 rep_tid=55 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] q front is repgather(0x558c576b6fc0 60'50 rep_tid=55 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] remove_repop repgather(0x558c576b6fc0 60'50 rep_tid=55 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=60'50 tt=0'0 tid=55 reqid=client.4540.0:449 client_op=osd_op(client.4540.0:449 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57efe420 finish 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57effe40 prio 196 cost 0 latency 0.000232 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=56, last_complete=60'51, committed=1, applied=1)) v2 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=60'51 tt=0'0 tid=56 reqid=client.4540.0:450 client_op=osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'50 crt=59'48 lcod 59'48 mlcod 59'48 active+clean ps=[e~1,15~1,1a~1,23~1]] repop_all_committed: repop tid 56 all committed 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] eval_repop repgather(0x558c576b6ea0 60'51 rep_tid=56 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] commit: repgather(0x558c576b6ea0 60'51 rep_tid=56 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f06986700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] log_op_stats osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 inb 0 outb 0 lat 0.008281 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] sending reply on osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 0x558c57bd4a00 2019-12-07T19:34:37.632+0000 7f8f06986700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(450 smithi13140796-2 [rollback 20] v60'51 uv51 ondisk = 0) v8 -- 0x558c57bd4a00 con 0x558c54c9f000 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:37.632+0000 7f8f06986700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 60:205 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] removing repgather(0x558c576b6ea0 60'51 rep_tid=56 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] q front is repgather(0x558c576b6ea0 60'51 rep_tid=56 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f06986700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] remove_repop repgather(0x558c576b6ea0 60'51 rep_tid=56 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=60'51 tt=0'0 tid=56 reqid=client.4540.0:450 client_op=osd_op(client.4540.0:450 1.cs0 1:33f5bf01:::smithi13140796-2:head [rollback 20] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:37.632+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57effe40 finish 2019-12-07T19:34:37.632+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57eff1e0 prio 196 cost 0 latency 0.000413 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=57, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=60'52 tt=0'0 tid=57 reqid=client.4540.0:451 client_op=osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'51 crt=59'48 lcod 60'50 mlcod 60'50 active+clean ps=[e~1,15~1,1a~1,23~1]] repop_all_committed: repop tid 57 all committed 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] eval_repop repgather(0x558c576b70e0 60'52 rep_tid=57 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] commit: repgather(0x558c576b70e0 60'52 rep_tid=57 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] log_op_stats osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 inb 0 outb 0 lat 0.008491 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] sending reply on osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) v8 0x558c57572780 2019-12-07T19:34:37.632+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(451 smithi13140796-2 [append 0~0] v60'52 uv52 ondisk = 0) v8 -- 0x558c57572780 con 0x558c54c9f000 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] prepare_stats_for_publish reporting purged_snaps [e~1,15~1,1a~1,23~1] 2019-12-07T19:34:37.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] publish_stats_to_osd 60:206 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] removing repgather(0x558c576b70e0 60'52 rep_tid=57 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] q front is repgather(0x558c576b70e0 60'52 rep_tid=57 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 15 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] remove_repop repgather(0x558c576b70e0 60'52 rep_tid=57 committed?=1 r=0) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=60'52 tt=0'0 tid=57 reqid=client.4540.0:451 client_op=osd_op(client.4540.0:451 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~0] snapc 25=[25,21,20,1b] ondisk+write+known_if_redirected e60) roll_forward_to=59'48 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=58 reqid=client.4540.0:451 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=58 reqid=client.4540.0:451 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:37.632+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 60/28 ECSubWrite(tid=58, reqid=client.4540.0:451, at_version=0'0, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c58231000 con 0x558c54c64000 2019-12-07T19:34:37.632+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 60/28 ECSubWrite(tid=58, reqid=client.4540.0:451, at_version=0'0, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c58231800 con 0x558c54c19800 2019-12-07T19:34:37.632+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 60/28 ECSubWrite(tid=58, reqid=client.4540.0:451, at_version=0'0, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c574a3000 con 0x558c54c5bc00 2019-12-07T19:34:37.632+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 60/28 ECSubWrite(tid=58, reqid=client.4540.0:451, at_version=0'0, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c574a4800 con 0x558c54c5b400 2019-12-07T19:34:37.632+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 60/28 ECSubWrite(tid=58, reqid=client.4540.0:451, at_version=0'0, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c574a2000 con 0x558c54c19c00 2019-12-07T19:34:37.632+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=59'48 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log log((0'0,60'52], crt=59'48) [] 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] rollforward: entry=60'49 (57'42) clone 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:21.628654+0000 0 snaps [25,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] rollforward: entry=60'50 (57'42) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:449 2019-12-07T19:34:37.625406+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] rollforward: entry=60'51 (60'50) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:450 2019-12-07T19:34:37.625424+0000 0 ObjectCleanRegions clean_offsets: [1622016~18446744073707929599], clean_omap: 0, new_object: 0 2019-12-07T19:34:37.632+0000 7f8f0297e700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] rollforward: entry=60'52 (60'51) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:451 2019-12-07T19:34:37.625428+0000 0 ObjectCleanRegions clean_offsets: [0~1622016,1622016~18446744073707929599], clean_omap: 1, new_object: 0 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log approx pg log length = 52 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:37.633+0000 7f8f0297e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585e4300 seq 71 2019-12-07T19:34:37.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head#33 onode 0x558c58295400 txc 0x558c585e4300 2019-12-07T19:34:37.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head#33 0x0 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b792d0) put_ref 0x0~2000 Blob(0x558c58b792d0 blob([0xa47000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b793b0) put_ref 0x0~2000 Blob(0x558c58b793b0 blob([0xa49000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79490) put_ref 0x0~2000 Blob(0x558c58b79490 blob([0xa4b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79570) put_ref 0x0~2000 Blob(0x558c58b79570 blob([0xa4d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79650) put_ref 0x0~2000 Blob(0x558c58b79650 blob([0xa4f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79730) put_ref 0x0~2000 Blob(0x558c58b79730 blob([0xa51000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79810) put_ref 0x0~2000 Blob(0x558c58b79810 blob([0xa53000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b798f0) put_ref 0x0~2000 Blob(0x558c58b798f0 blob([0xa55000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b799d0) put_ref 0x0~2000 Blob(0x558c58b799d0 blob([0xa57000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79ab0) put_ref 0x0~2000 Blob(0x558c58b79ab0 blob([0xa59000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79b90) put_ref 0x0~2000 Blob(0x558c58b79b90 blob([0xa5b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79c70) put_ref 0x0~2000 Blob(0x558c58b79c70 blob([0xa5d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79d50) put_ref 0x0~2000 Blob(0x558c58b79d50 blob([0xa5f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79e30) put_ref 0x0~2000 Blob(0x558c58b79e30 blob([0xa61000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b79f10) put_ref 0x0~2000 Blob(0x558c58b79f10 blob([0xa63000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80000) put_ref 0x0~2000 Blob(0x558c58b80000 blob([0xa65000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b800e0) put_ref 0x0~2000 Blob(0x558c58b800e0 blob([0xa67000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b801c0) put_ref 0x0~2000 Blob(0x558c58b801c0 blob([0xa69000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b802a0) put_ref 0x0~2000 Blob(0x558c58b802a0 blob([0xa6b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80380) put_ref 0x0~2000 Blob(0x558c58b80380 blob([0xa6d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80460) put_ref 0x0~2000 Blob(0x558c58b80460 blob([0xa6f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80540) put_ref 0x0~2000 Blob(0x558c58b80540 blob([0xa71000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80620) put_ref 0x0~2000 Blob(0x558c58b80620 blob([0xa73000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80700) put_ref 0x0~2000 Blob(0x558c58b80700 blob([0xa75000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b807e0) put_ref 0x0~2000 Blob(0x558c58b807e0 blob([0xa77000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b808c0) put_ref 0x0~2000 Blob(0x558c58b808c0 blob([0xa79000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b809a0) put_ref 0x0~2000 Blob(0x558c58b809a0 blob([0xa7b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80a80) put_ref 0x0~2000 Blob(0x558c58b80a80 blob([0xa7d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80b60) put_ref 0x0~2000 Blob(0x558c58b80b60 blob([0xa7f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80c40) put_ref 0x0~2000 Blob(0x558c58b80c40 blob([0xa81000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80d20) put_ref 0x0~2000 Blob(0x558c58b80d20 blob([0xa83000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80e00) put_ref 0x0~2000 Blob(0x558c58b80e00 blob([0xa85000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80ee0) put_ref 0x0~2000 Blob(0x558c58b80ee0 blob([0xa87000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b80fc0) put_ref 0x0~2000 Blob(0x558c58b80fc0 blob([0xa89000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b810a0) put_ref 0x0~2000 Blob(0x558c58b810a0 blob([0xa8b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81180) put_ref 0x0~2000 Blob(0x558c58b81180 blob([0xa8d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81260) put_ref 0x0~2000 Blob(0x558c58b81260 blob([0xa8f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81340) put_ref 0x0~2000 Blob(0x558c58b81340 blob([0xa91000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81420) put_ref 0x0~2000 Blob(0x558c58b81420 blob([0xa93000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81500) put_ref 0x0~2000 Blob(0x558c58b81500 blob([0xa95000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b815e0) put_ref 0x0~2000 Blob(0x558c58b815e0 blob([0xa97000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b816c0) put_ref 0x0~2000 Blob(0x558c58b816c0 blob([0xa99000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b817a0) put_ref 0x0~2000 Blob(0x558c58b817a0 blob([0xa9b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81880) put_ref 0x0~2000 Blob(0x558c58b81880 blob([0xa9d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81960) put_ref 0x0~2000 Blob(0x558c58b81960 blob([0xa9f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81a40) put_ref 0x0~2000 Blob(0x558c58b81a40 blob([0xaa1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81b20) put_ref 0x0~2000 Blob(0x558c58b81b20 blob([0xaa3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81c00) put_ref 0x0~2000 Blob(0x558c58b81c00 blob([0xaa5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81ce0) put_ref 0x0~2000 Blob(0x558c58b81ce0 blob([0xaa7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.blob(0x558c58b81dc0) put_ref 0x0~1000 Blob(0x558c58b81dc0 blob([0xaa9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295508) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295508) dirty_range mark shard 0x24000 dirty 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore.extentmap(0x558c58295508) dirty_range mark shard 0x48000 dirty 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c58b792d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa47000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c58b793b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa49000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c58b79490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa4b000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c58b79570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa4d000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c58b79650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa4f000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c58b79730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa51000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c58b79810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa53000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c58b798f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa55000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c58b799d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa57000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c58b79ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa59000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c58b79b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa5b000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c58b79c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa5d000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c58b79d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa5f000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c58b79e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa61000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c58b79f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa63000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c58b80000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa65000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c58b800e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa67000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c58b801c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa69000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c58b802a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa6b000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c58b80380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa6d000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c58b80460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa6f000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c58b80540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa71000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c58b80620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa73000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c58b80700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa75000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c58b807e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa77000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c58b808c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa79000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c58b809a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa7b000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~2000: 0x0~2000 Blob(0x558c58b80a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa7d000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x38000~2000: 0x0~2000 Blob(0x558c58b80b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa7f000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3a000~2000: 0x0~2000 Blob(0x558c58b80c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa81000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3c000~2000: 0x0~2000 Blob(0x558c58b80d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa83000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3e000~2000: 0x0~2000 Blob(0x558c58b80e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa85000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x40000~2000: 0x0~2000 Blob(0x558c58b80ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa87000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x42000~2000: 0x0~2000 Blob(0x558c58b80fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa89000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x44000~2000: 0x0~2000 Blob(0x558c58b810a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa8b000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x46000~2000: 0x0~2000 Blob(0x558c58b81180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa8d000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x48000~2000: 0x0~2000 Blob(0x558c58b81260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa8f000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4a000~2000: 0x0~2000 Blob(0x558c58b81340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa91000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~2000: 0x0~2000 Blob(0x558c58b81420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa93000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4e000~2000: 0x0~2000 Blob(0x558c58b81500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa95000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x50000~2000: 0x0~2000 Blob(0x558c58b815e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa97000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x52000~2000: 0x0~2000 Blob(0x558c58b816c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa99000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x54000~2000: 0x0~2000 Blob(0x558c58b817a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa9b000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x56000~2000: 0x0~2000 Blob(0x558c58b81880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa9d000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x58000~2000: 0x0~2000 Blob(0x558c58b81960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xa9f000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5a000~2000: 0x0~2000 Blob(0x558c58b81a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaa1000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5c000~2000: 0x0~2000 Blob(0x558c58b81b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaa3000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5e000~2000: 0x0~2000 Blob(0x558c58b81c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaa5000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x60000~2000: 0x0~2000 Blob(0x558c58b81ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaa7000~2000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x62000~1000: 0x0~1000 Blob(0x558c58b81dc0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=3)))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaa9000~1000] 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x24000 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x48000 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head#33 = 0 2019-12-07T19:34:37.633+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585e4300 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585e4300 onodes shared_blobs 0x558c58b79340,0x558c58b79420,0x558c58b79500,0x558c58b795e0,0x558c58b796c0,0x558c58b797a0,0x558c58b79880,0x558c58b79960,0x558c58b79a40,0x558c58b79b20,0x558c58b79c00,0x558c58b79ce0,0x558c58b79dc0,0x558c58b79ea0,0x558c58b79f80,0x558c58b80070,0x558c58b80150,0x558c58b80230,0x558c58b80310,0x558c58b803f0,0x558c58b804d0,0x558c58b805b0,0x558c58b80690,0x558c58b80770,0x558c58b80850,0x558c58b80930,0x558c58b80a10,0x558c58b80af0,0x558c58b80bd0,0x558c58b80cb0,0x558c58b80d90,0x558c58b80e70,0x558c58b80f50,0x558c58b81030,0x558c58b81110,0x558c58b811f0,0x558c58b812d0,0x558c58b813b0,0x558c58b81490,0x558c58b81570,0x558c58b81650,0x558c58b81730,0x558c58b81810,0x558c58b818f0,0x558c58b819d0,0x558c58b81ab0,0x558c58b81b90,0x558c58b81c70,0x558c58b81d50,0x558c58b81e30 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a78 is 11 SharedBlob(0x558c58b79340 loaded (sbid 0x2a78 ref_map(0xa47000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a79 is 11 SharedBlob(0x558c58b79420 loaded (sbid 0x2a79 ref_map(0xa49000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7a is 11 SharedBlob(0x558c58b79500 loaded (sbid 0x2a7a ref_map(0xa4b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7b is 11 SharedBlob(0x558c58b795e0 loaded (sbid 0x2a7b ref_map(0xa4d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7c is 11 SharedBlob(0x558c58b796c0 loaded (sbid 0x2a7c ref_map(0xa4f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7d is 11 SharedBlob(0x558c58b797a0 loaded (sbid 0x2a7d ref_map(0xa51000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7e is 11 SharedBlob(0x558c58b79880 loaded (sbid 0x2a7e ref_map(0xa53000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a7f is 11 SharedBlob(0x558c58b79960 loaded (sbid 0x2a7f ref_map(0xa55000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a80 is 11 SharedBlob(0x558c58b79a40 loaded (sbid 0x2a80 ref_map(0xa57000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a81 is 11 SharedBlob(0x558c58b79b20 loaded (sbid 0x2a81 ref_map(0xa59000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a82 is 11 SharedBlob(0x558c58b79c00 loaded (sbid 0x2a82 ref_map(0xa5b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a83 is 11 SharedBlob(0x558c58b79ce0 loaded (sbid 0x2a83 ref_map(0xa5d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a84 is 11 SharedBlob(0x558c58b79dc0 loaded (sbid 0x2a84 ref_map(0xa5f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a85 is 11 SharedBlob(0x558c58b79ea0 loaded (sbid 0x2a85 ref_map(0xa61000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a86 is 11 SharedBlob(0x558c58b79f80 loaded (sbid 0x2a86 ref_map(0xa63000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a87 is 11 SharedBlob(0x558c58b80070 loaded (sbid 0x2a87 ref_map(0xa65000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a88 is 11 SharedBlob(0x558c58b80150 loaded (sbid 0x2a88 ref_map(0xa67000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a89 is 11 SharedBlob(0x558c58b80230 loaded (sbid 0x2a89 ref_map(0xa69000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8a is 11 SharedBlob(0x558c58b80310 loaded (sbid 0x2a8a ref_map(0xa6b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8b is 11 SharedBlob(0x558c58b803f0 loaded (sbid 0x2a8b ref_map(0xa6d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8c is 11 SharedBlob(0x558c58b804d0 loaded (sbid 0x2a8c ref_map(0xa6f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8d is 11 SharedBlob(0x558c58b805b0 loaded (sbid 0x2a8d ref_map(0xa71000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8e is 11 SharedBlob(0x558c58b80690 loaded (sbid 0x2a8e ref_map(0xa73000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a8f is 11 SharedBlob(0x558c58b80770 loaded (sbid 0x2a8f ref_map(0xa75000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a90 is 11 SharedBlob(0x558c58b80850 loaded (sbid 0x2a90 ref_map(0xa77000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a91 is 11 SharedBlob(0x558c58b80930 loaded (sbid 0x2a91 ref_map(0xa79000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a92 is 11 SharedBlob(0x558c58b80a10 loaded (sbid 0x2a92 ref_map(0xa7b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a93 is 11 SharedBlob(0x558c58b80af0 loaded (sbid 0x2a93 ref_map(0xa7d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a94 is 11 SharedBlob(0x558c58b80bd0 loaded (sbid 0x2a94 ref_map(0xa7f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a95 is 11 SharedBlob(0x558c58b80cb0 loaded (sbid 0x2a95 ref_map(0xa81000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a96 is 11 SharedBlob(0x558c58b80d90 loaded (sbid 0x2a96 ref_map(0xa83000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a97 is 11 SharedBlob(0x558c58b80e70 loaded (sbid 0x2a97 ref_map(0xa85000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a98 is 11 SharedBlob(0x558c58b80f50 loaded (sbid 0x2a98 ref_map(0xa87000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a99 is 11 SharedBlob(0x558c58b81030 loaded (sbid 0x2a99 ref_map(0xa89000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9a is 11 SharedBlob(0x558c58b81110 loaded (sbid 0x2a9a ref_map(0xa8b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9b is 11 SharedBlob(0x558c58b811f0 loaded (sbid 0x2a9b ref_map(0xa8d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9c is 11 SharedBlob(0x558c58b812d0 loaded (sbid 0x2a9c ref_map(0xa8f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9d is 11 SharedBlob(0x558c58b813b0 loaded (sbid 0x2a9d ref_map(0xa91000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9e is 11 SharedBlob(0x558c58b81490 loaded (sbid 0x2a9e ref_map(0xa93000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a9f is 11 SharedBlob(0x558c58b81570 loaded (sbid 0x2a9f ref_map(0xa95000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa0 is 11 SharedBlob(0x558c58b81650 loaded (sbid 0x2aa0 ref_map(0xa97000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa1 is 11 SharedBlob(0x558c58b81730 loaded (sbid 0x2aa1 ref_map(0xa99000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa2 is 11 SharedBlob(0x558c58b81810 loaded (sbid 0x2aa2 ref_map(0xa9b000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa3 is 11 SharedBlob(0x558c58b818f0 loaded (sbid 0x2aa3 ref_map(0xa9d000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa4 is 11 SharedBlob(0x558c58b819d0 loaded (sbid 0x2aa4 ref_map(0xa9f000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa5 is 11 SharedBlob(0x558c58b81ab0 loaded (sbid 0x2aa5 ref_map(0xaa1000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa6 is 11 SharedBlob(0x558c58b81b90 loaded (sbid 0x2aa6 ref_map(0xaa3000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa7 is 11 SharedBlob(0x558c58b81c70 loaded (sbid 0x2aa7 ref_map(0xaa5000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa8 is 11 SharedBlob(0x558c58b81d50 loaded (sbid 0x2aa8 ref_map(0xaa7000~2000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aa9 is 11 SharedBlob(0x558c58b81e30 loaded (sbid 0x2aa9 ref_map(0xaa9000~1000=2))) 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585e4300 allocated 0x[] released 0x[] 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4300 prepare 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585e4300 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4300 io_done 2019-12-07T19:34:37.633+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57eff1e0 finish 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2594 < min_flush_size 524288 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58295400 had 1 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2617 < min_flush_size 524288 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2617 < min_flush_size 524288 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4cd051~a39 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4cd051 0x4cd051~a39 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x49d051 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x51 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa8a and padding block with 0x576 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4cda8a 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:37.633+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000206886s (0.000000141s flush + 0.000206745s kv commit) 2019-12-07T19:34:37.633+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:37.633+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 120 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57519a00 con 0x558c54c19c00 2019-12-07T19:34:37.633+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57eff340 prio 196 cost 0 latency 0.000015 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.633+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.633+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57eff340 prio 196 cost 0 latency 0.000047 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.633+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.633+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57eff340 finish 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:37.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585e4300> 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:37.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4300 kv_submitted 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585e4300 2019-12-07T19:34:37.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4300 finishing 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585e4300 onodes 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4300 done 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:37.633+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585e4300 [] 2019-12-07T19:34:37.633+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:37.633+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.633+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.633+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:37.634+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 212 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58332400 con 0x558c54c19800 2019-12-07T19:34:37.634+0000 7f8f24c58700 15 osd.7 60 enqueue_op 0x558c5511fce0 prio 196 cost 0 latency 0.000016 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.634+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c5511fce0 prio 196 cost 0 latency 0.000049 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c5511fce0 finish 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.634+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 322 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c5763c600 con 0x558c54c5b400 2019-12-07T19:34:37.634+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c5511e420 prio 196 cost 0 latency 0.000012 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.634+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 299 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5763ce00 con 0x558c54c5bc00 2019-12-07T19:34:37.634+0000 7f8f23c56700 15 osd.7 60 enqueue_op 0x558c57335b80 prio 196 cost 0 latency 0.000010 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c5511e420 prio 196 cost 0 latency 0.000041 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.634+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.634+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c5511e420 finish 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.634+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57335b80 prio 196 cost 0 latency 0.000043 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.634+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f0297e700 10 osd.7 60 dequeue_op 0x558c57335b80 finish 2019-12-07T19:34:37.634+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.634+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 519 ==== MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57613600 con 0x558c54c64000 2019-12-07T19:34:37.634+0000 7f8f24457700 15 osd.7 60 enqueue_op 0x558c57efeb00 prio 196 cost 0 latency 0.000010 epoch 60 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) queued 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2) prio 196 cost 0 e60) pg 0x558c57676000 2019-12-07T19:34:37.634+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57efeb00 prio 196 cost 0 latency 0.000041 MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 pg pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] 2019-12-07T19:34:37.634+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 60/28 ECSubWriteReply(tid=58, last_complete=60'52, committed=1, applied=1)) v2 2019-12-07T19:34:37.634+0000 7f8f06986700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=58 reqid=client.4540.0:451 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:37.634+0000 7f8f06986700 10 osd.7 60 dequeue_op 0x558c57efeb00 finish 2019-12-07T19:34:37.634+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:37.645+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 19 ==== osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:37.646713+0000/61.118432011s send_stamp 61.118432011s delta_ub 0.036479118s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8c00 con 0x558c552d1c00 2019-12-07T19:34:37.645+0000 7f8f24457700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036167958s,0.036479118s]) 2019-12-07T19:34:37.645+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 19 ==== osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:37.646713+0000/61.118432011s send_stamp 61.118432011s delta_ub 0.036479118s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58115000 con 0x558c552d1800 2019-12-07T19:34:37.645+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:37.646713+0000/61.118432011s send_stamp 61.082264053s delta_ub 18446744073.673383658s) v5 -- 0x558c55044800 con 0x558c552d1c00 2019-12-07T19:34:37.645+0000 7f8f24c58700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036148101s,0.036479118s]) 2019-12-07T19:34:37.645+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:37.646713+0000/61.118432011s send_stamp 61.082283910s delta_ub 18446744073.673403515s) v5 -- 0x558c584fa200 con 0x558c552d1800 2019-12-07T19:34:37.647+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 14 ==== osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:37.664049+0000/61.113275133s send_stamp 61.113275133s delta_ub 0.013889716s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c584fa000 con 0x558c54c65400 2019-12-07T19:34:37.663+0000 7f8f24c58700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013591486s,0.013889716s]) 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:37.664049+0000/61.113275133s send_stamp 61.099683647s delta_ub 18446744073.695960130s) v5 -- 0x558c584fa200 con 0x558c54c65400 2019-12-07T19:34:37.663+0000 7f8f24c58700 0 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65000 msgr2=0x558c54c71180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 14 ==== osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:37.664049+0000/61.113275133s send_stamp 61.113275133s delta_ub 0.013889716s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58115000 con 0x558c54c65000 2019-12-07T19:34:37.663+0000 7f8f24c58700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013515646s,0.013889716s]) 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:37.664049+0000/61.113275133s send_stamp 61.099759487s delta_ub 18446744073.696035970s) v5 -- 0x558c584fa200 con 0x558c54c65000 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65000 msgr2=0x558c54c71180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 80 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65000 msgr2=0x558c54c71180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65000 0x558c54c71180 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65000 0x558c54c71180 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:37.663+0000 7f8f0e996700 20 osd.7 60 heartbeat_reset con 0x558c54c65000 s 0x558c54c0ba00 2019-12-07T19:34:37.663+0000 7f8f0e996700 10 osd.7 60 heartbeat_reset closing (old) failed hb con 0x558c54c65000 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65400 msgr2=0x558c54c6f080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 75 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65400 msgr2=0x558c54c6f080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65400 0x558c54c6f080 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c54c65400 0x558c54c6f080 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:37.663+0000 7f8f0d994700 20 osd.7 60 heartbeat_reset con 0x558c54c65400 s 0x558c54c5d1e0 2019-12-07T19:34:37.663+0000 7f8f0d994700 10 osd.7 60 heartbeat_reset closing (old) failed hb con 0x558c54c65400 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c58147400 0x558c552d6000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> conn(0x558c58147400 0x558c552d6000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c548d3000 0x558c552d6b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> conn(0x558c548d3000 0x558c552d6b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:37.663+0000 7f8f24c58700 10 osd.7 60 ms_handle_authentication new session 0x558c54c5c4e0 con 0x558c58147400 entity osd.4 addr 2019-12-07T19:34:37.663+0000 7f8f24c58700 10 osd.7 60 ms_handle_authentication session 0x558c54c5c4e0 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:37.663+0000 7f8f24c58700 10 osd.7 60 ms_handle_authentication new session 0x558c57bfb040 con 0x558c548d3000 entity osd.4 addr 2019-12-07T19:34:37.663+0000 7f8f24c58700 10 osd.7 60 ms_handle_authentication session 0x558c57bfb040 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] >> 172.21.15.109:0/36938 conn(0x558c58147400 0x558c552d6000 crc :-1 s=READY pgs=19 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:37.663+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] >> 172.21.15.109:0/36938 conn(0x558c548d3000 0x558c552d6b00 crc :-1 s=READY pgs=15 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:37.697+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.747+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 10 osd.7 60 tick_without_osd_lock 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f435000/0x40000000/0x1680000000, data 0xda1226/0xbc9000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 25 osd.7 60 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:32.576231+0000 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 25 osd.7 60 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576218+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 25 osd.7 60 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 25 osd.7 60 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576154+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 25 osd.7 60 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:32.575839+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 20 osd.7 60 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 10 osd.7 60 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 20 osd.7 60 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:37.769+0000 7f8f1fbbe700 10 osd.7 60 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:37.797+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.848+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.898+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.948+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:37.998+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.046+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:38.048+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.098+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.145+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 15 ==== osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.146692+0000/61.613015596s send_stamp 61.613015596s delta_ub 0.031047485s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57951800 con 0x558c54c5ac00 2019-12-07T19:34:38.145+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 15 ==== osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.146692+0000/61.613015596s send_stamp 61.613015596s delta_ub 0.031047485s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c54c5a400 2019-12-07T19:34:38.145+0000 7f8f23c56700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030707100s,0.031047485s]) 2019-12-07T19:34:38.145+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.146692+0000/61.613015596s send_stamp 61.582308496s delta_ub 18446744073.678844516s) v5 -- 0x558c58513600 con 0x558c54c5ac00 2019-12-07T19:34:38.145+0000 7f8f24c58700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030688206s,0.031047485s]) 2019-12-07T19:34:38.145+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.146692+0000/61.613015596s send_stamp 61.582327390s delta_ub 18446744073.678863410s) v5 -- 0x558c58115000 con 0x558c54c5a400 2019-12-07T19:34:38.148+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.198+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510368 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.203+0000 7f8f21451700 10 osd.7 60 tick 2019-12-07T19:34:38.203+0000 7f8f21451700 10 osd.7 60 do_waiters -- start 2019-12-07T19:34:38.203+0000 7f8f21451700 10 osd.7 60 do_waiters -- finish 2019-12-07T19:34:38.203+0000 7f8f21451700 20 osd.7 60 tick last_purged_snaps_scrub 2019-12-07T19:33:37.591458+0000 next 2019-12-07T19:34:37.591458+0000 ... now 2019-12-07T19:34:38.203+0000 7f8f21451700 10 osd.7 60 scrub_purged_snaps 2019-12-07T19:34:38.203+0000 7f8f21451700 0 log_channel(cluster) log [DBG] : purged_snaps scrub starts 2019-12-07T19:34:38.203+0000 7f8f21451700 10 snap_mapper.run 2019-12-07T19:34:38.203+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:38.203+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) upper_bound after PSN_ key 0x00000000000000000000000000000403'.PSN_' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000002' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [2,3) 2019-12-07T19:34:38.203+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:38.203+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630140) upper_bound after SNA_ key 0x00000000000000000000000000000402'.SNA_' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630140) valid is at 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_m mapping pool 1 snap 1b shard 255 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000004' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [4,5) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000007' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [6,8) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_000000000000000a' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [a,b) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_000000000000000d' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [c,e) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000011' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [10,12) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000013' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [13,14) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000017' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [17,18) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000019' 2019-12-07T19:34:38.203+0000 7f8f21451700 20 snap_mapper._parse_p purged_snaps pool 1 [19,1a) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.OmapIteratorImpl(0x558c57630040) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:38.203+0000 7f8f21451700 10 snap_mapper.run passed final purged_snaps interval, rest ok 2019-12-07T19:34:38.203+0000 7f8f21451700 10 snap_mapper.run end, found 0 stray 2019-12-07T19:34:38.203+0000 7f8f21451700 0 log_channel(cluster) log [DBG] : purged_snaps scrub ok 2019-12-07T19:34:38.203+0000 7f8f21451700 10 osd.7 60 scrub_purged_snaps done queueing pgs, updating superblock 2019-12-07T19:34:38.203+0000 7f8f21451700 10 osd.7 60 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e60 [1,60] lci=[13,60]) 2019-12-07T19:34:38.203+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c584fc000 seq 52 2019-12-07T19:34:38.203+0000 7f8f21451700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:38.203+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c584fc000 0 blobs 2019-12-07T19:34:38.203+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:38.204+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:38.204+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584fc000 cost 4803 (1 ios * 4000 + 803 bytes) 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584fc000 onodes 0x558c53a91900 shared_blobs 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584fc000 allocated 0x[] released 0x[] 2019-12-07T19:34:38.204+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc000 prepare 2019-12-07T19:34:38.204+0000 7f8f21451700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584fc000 2019-12-07T19:34:38.204+0000 7f8f21451700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc000 io_done 2019-12-07T19:34:38.204+0000 7f8f21451700 20 osd.7 60 send_beacon sending 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.204+0000 7f8f21451700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_beacon(pgs [1.2,1.c,1.4] lec 57 last_purged_snaps_scrub 2019-12-07T19:34:38.205126+0000 v60) v2 -- 0x558c584fe000 con 0x558c5476d000 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4270 < min_flush_size 524288 2019-12-07T19:34:38.204+0000 7f8f21451700 10 osd.7 60 scrub_purged_snaps done 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4cda8a~10c5 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4cda8a 0x4cda8a~10c5 to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4a0000 x_off 0x49da8a 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa8a 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb4f and padding block with 0x4b1 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4ceb4f 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.204+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000128669s (0.000000207s flush + 0.000128462s kv commit) 2019-12-07T19:34:38.204+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584fc000> 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.204+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc000 kv_submitted 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584fc000 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c584fc000 osr 0x558c54ea4b40 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 49 0x2000~1000 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 50 0x2000~1000 crc 142e93d 2019-12-07T19:34:38.204+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.237+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:38.248+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.298+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.348+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:38.348+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.399+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.449+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.474+0000 7f8f0097a700 5 osd.7 60 heartbeat osd_stat(store_statfs(0x163f435000/0x40000000/0x1680000000, data 0xda1226/0xbc9000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5] op hist []) 2019-12-07T19:34:38.474+0000 7f8f0097a700 20 osd.7 60 check_full_status cur ratio 0.000127962, physical ratio 0.000127962, new state none 2019-12-07T19:34:38.474+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.688386373s) v5 -- 0x558c54c1f800 con 0x558c54c9e000 2019-12-07T19:34:38.474+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.688386373s) v5 -- 0x558c58512200 con 0x558c54c9e400 2019-12-07T19:34:38.474+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.673403515s) v5 -- 0x558c58512a00 con 0x558c551b7800 2019-12-07T19:34:38.474+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.673403515s) v5 -- 0x558c58512000 con 0x558c551b7c00 2019-12-07T19:34:38.474+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.687628620s) v5 -- 0x558c584fe400 con 0x558c552c2000 2019-12-07T19:34:38.474+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.687628620s) v5 -- 0x558c584fe600 con 0x558c552c2400 2019-12-07T19:34:38.474+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.696035970s) v5 -- 0x558c584fe800 con 0x558c552c2800 2019-12-07T19:34:38.475+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.696035970s) v5 -- 0x558c584fea00 con 0x558c552c2c00 2019-12-07T19:34:38.475+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.678863410s) v5 -- 0x558c584fec00 con 0x558c54c65800 2019-12-07T19:34:38.475+0000 7f8f0097a700 1 -- 172.21.15.109:0/36939 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.911542687s delta_ub 18446744073.678863410s) v5 -- 0x558c584fee00 con 0x558c54c65c00 2019-12-07T19:34:38.475+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6808/37077 17 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.948075332s delta_ub 0.036532645s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513400 con 0x558c551b7800 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:32.576218+0000 -> 2019-12-07T19:34:38.476352+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 2019-12-07T19:34:38.475+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6814/36938 17 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.925499150s delta_ub 0.013956463s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c584fa000 con 0x558c552c2800 2019-12-07T19:34:38.475+0000 7f8f23c56700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036283533s,0.036532645s]) 2019-12-07T19:34:38.475+0000 7f8f24c58700 25 osd.7 60 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:32.576154+0000 -> 2019-12-07T19:34:38.476367+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 2019-12-07T19:34:38.475+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6816/37076 17 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.933885852s delta_ub 0.022343165s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8c00 con 0x558c552c2000 2019-12-07T19:34:38.475+0000 7f8f24c58700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013692546s,0.013956463s]) 2019-12-07T19:34:38.475+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6806/36936 13 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.942601437s delta_ub 0.031058750s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513400 con 0x558c54c65800 2019-12-07T19:34:38.475+0000 7f8f24457700 25 osd.7 60 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:32.576173+0000 -> 2019-12-07T19:34:38.476378+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 2019-12-07T19:34:38.475+0000 7f8f24457700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.022068569s,0.022343165s]) 2019-12-07T19:34:38.475+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.2 v2:172.21.15.131:6806/37077 17 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.948094466s delta_ub 0.036551779s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c584fa000 con 0x558c551b7c00 2019-12-07T19:34:38.475+0000 7f8f24c58700 25 osd.7 60 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476352+0000 last_rx_front 2019-12-07T19:34:32.576207+0000 -> 2019-12-07T19:34:38.476394+0000 2019-12-07T19:34:38.475+0000 7f8f24c58700 25 osd.7 60 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:38.476103+0000) and older pending ping(s) 2019-12-07T19:34:38.475+0000 7f8f24c58700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036260961s,0.036551779s]) 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:32.576059+0000 -> 2019-12-07T19:34:38.476409+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 2019-12-07T19:34:38.475+0000 7f8f23c56700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030752447s,0.031058750s]) 2019-12-07T19:34:38.475+0000 7f8f24c58700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6827/37078 13 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.933312996s delta_ub 0.021770309s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c584fa000 con 0x558c54c9e400 2019-12-07T19:34:38.475+0000 7f8f24c58700 25 osd.7 60 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 last_rx_front 2019-12-07T19:34:32.576231+0000 -> 2019-12-07T19:34:38.476418+0000 2019-12-07T19:34:38.475+0000 7f8f24c58700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021455238s,0.021770309s]) 2019-12-07T19:34:38.475+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.5 v2:172.21.15.109:6804/36936 13 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.942622617s delta_ub 0.031079930s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513400 con 0x558c54c65c00 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476409+0000 last_rx_front 2019-12-07T19:34:32.576084+0000 -> 2019-12-07T19:34:38.476432+0000 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:38.476103+0000) and older pending ping(s) 2019-12-07T19:34:38.475+0000 7f8f23c56700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030751352s,0.031079930s]) 2019-12-07T19:34:38.475+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.1 v2:172.21.15.131:6831/37078 13 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.933292495s delta_ub 0.021749808s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513400 con 0x558c54c9e000 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:32.576139+0000 -> 2019-12-07T19:34:38.476453+0000 last_rx_front 2019-12-07T19:34:38.476418+0000 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:38.476103+0000) and older pending ping(s) 2019-12-07T19:34:38.475+0000 7f8f23c56700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.021400022s,0.021749808s]) 2019-12-07T19:34:38.475+0000 7f8f23c56700 1 -- 172.21.15.109:0/36939 <== osd.3 v2:172.21.15.131:6814/37076 17 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.933906299s delta_ub 0.022363612s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58513400 con 0x558c552c2400 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476378+0000 last_rx_front 2019-12-07T19:34:32.576184+0000 -> 2019-12-07T19:34:38.476470+0000 2019-12-07T19:34:38.475+0000 7f8f23c56700 25 osd.7 60 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:38.476103+0000) and older pending ping(s) 2019-12-07T19:34:38.475+0000 7f8f23c56700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021996890s,0.022363612s]) 2019-12-07T19:34:38.475+0000 7f8f24457700 1 -- 172.21.15.109:0/36939 <== osd.4 v2:172.21.15.109:6812/36938 17 ==== osd_ping(ping_reply e60 up_from 13 ping_stamp 2019-12-07T19:34:38.476103+0000/61.911542687s send_stamp 61.925676184s delta_ub 0.014133497s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5768b200 con 0x558c552c2c00 2019-12-07T19:34:38.475+0000 7f8f24457700 25 osd.7 60 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476367+0000 last_rx_front 2019-12-07T19:34:32.576108+0000 -> 2019-12-07T19:34:38.476505+0000 2019-12-07T19:34:38.475+0000 7f8f24457700 25 osd.7 60 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:38.476103+0000) and older pending ping(s) 2019-12-07T19:34:38.475+0000 7f8f24457700 20 osd.7 60 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013731992s,0.014133497s]) 2019-12-07T19:34:38.500+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.549+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.599+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1510384 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.601+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9f400 0x558c54e34580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:38.601+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c54c9f400 0x558c54e34580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:38.601+0000 7f8f24c58700 10 osd.7 60 ms_handle_authentication new session 0x558c57bfa9c0 con 0x558c54c9f400 entity client.admin addr 2019-12-07T19:34:38.601+0000 7f8f24c58700 10 osd.7 60 ms_handle_authentication session 0x558c57bfa9c0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:38.601+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2305695222 conn(0x558c54c9f400 0x558c54e34580 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4547 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:38.601+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4547 172.21.15.131:0/2305695222 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c585f9c20 con 0x558c54c9f400 2019-12-07T19:34:38.601+0000 7f8f164bb700 20 osd.7 60 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:38.601+0000 7f8f164bb700 10 osd.7 60 do_waiters -- start 2019-12-07T19:34:38.601+0000 7f8f164bb700 10 osd.7 60 do_waiters -- finish 2019-12-07T19:34:38.601+0000 7f8f164bb700 20 osd.7 60 _dispatch 0x558c585f9c20 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:38.602+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2305695222 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce0e00 con 0x558c54c9f400 2019-12-07T19:34:38.607+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 34 ==== osd_map(60..61 src has 1..61) v4 ==== 1590+0+0 (crc 0 0 0) 0x558c585e2240 con 0x558c5476d000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 osd.7 60 OSD::ms_dispatch: osd_map(60..61 src has 1..61) v4 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 osd.7 60 do_waiters -- start 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 osd.7 60 do_waiters -- finish 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 osd.7 60 _dispatch 0x558c585e2240 osd_map(60..61 src has 1..61) v4 2019-12-07T19:34:38.607+0000 7f8f164bb700 3 osd.7 60 handle_osd_map epochs [60,61], i have 60, src has [1,61] 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 osd.7 60 handle_osd_map got inc map for epoch 61 2019-12-07T19:34:38.607+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:33c4e826:::osdmap.60:0# 0x0~0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1458 size 0x1458 (5208) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c574f1f80 blob([0xbc9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579d7960 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580f7030 blob([!~1000,0xbca000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,458]) SharedBlob(0x558c580f70a0 sbid 0x0)) need 0x1000~458 cache has 0x[1000~458] 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:33c4e826:::osdmap.60:0# 0x0~1458 = 5208 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 osd.7 60 add_map_bl 60 5208 bytes 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 osd.7 60 got_full_map 61, nothing requested 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 osd.7 60 handle_osd_map pg_num_history pg_num_history(e61 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {61={1=[e~1,15~1,1a~1]}} 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) lower_bound to PSN__1_000000000000000d key 0x00000000000000000000000000000403'.PSN__1_000000000000000d' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) valid is at 0x00000000000000000000000000000403'.PSN__1_000000000000000d' 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) lower_bound to PSN__1_000000000000000f key 0x00000000000000000000000000000403'.PSN__1_000000000000000f' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000011' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap f found [10,12), no overlap 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps [e,f) - join with earlier [c,e) 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) lower_bound to PSN__1_0000000000000014 key 0x00000000000000000000000000000403'.PSN__1_0000000000000014' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000017' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 14 found [17,18), no overlap 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) lower_bound to PSN__1_0000000000000016 key 0x00000000000000000000000000000403'.PSN__1_0000000000000016' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000017' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 16 found [17,18), no overlap 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) lower_bound to PSN__1_0000000000000019 key 0x00000000000000000000000000000403'.PSN__1_0000000000000019' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) valid is at 0x00000000000000000000000000000403'.PSN__1_0000000000000019' 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.onode(0x558c548f2500).flush flush done 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) lower_bound to PSN__1_000000000000001b key 0x00000000000000000000000000000403'.PSN__1_000000000000001b' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.OmapIteratorImpl(0x558c5517b800) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 snap_mapper._lookup_purged_snap pool 1 snap 1b key 'PSN__1_000000000000001b' lower_bound not found 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps [1a,1b) - join with earlier [19,1a) 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 2 keys, set 3 keys 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 osd.7 60 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e61 [1,61] lci=[13,61]) 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c582b6f00 seq 53 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c1df972f:::inc_osdmap.61:0# key 0x7f7fffffffffffffffc1df97'/!inc_osdmap.61!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c1df972f:::inc_osdmap.61:0# 0x0~383 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1284 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c1df972f:::inc_osdmap.61:0# 0x0~383 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~383 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xc7d on front/back, now 0x0~1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6f00 1 blobs 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbcb000~1000/1000,1000,0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbcb000~1000] 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580d7c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f7c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580d7c00 blob([0xbcb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7c70 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7c00) get_ref 0x0~383 Blob(0x558c580d7c00 blob([0xbcb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7c70 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c580d7c00) get_ref init 0x1000, 1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~383: 0x0~383 Blob(0x558c580d7c00 blob([0xbcb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x383) SharedBlob(0x558c580f7c70 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c580f7c88 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x383 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c579df008) dirty_range mark inline shard dirty 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c1df972f:::inc_osdmap.61:0# 0x0~383 = 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3904e826:::osdmap.61:0# key 0x7f7fffffffffffffff3904e8'&!osdmap.61!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3904e826:::osdmap.61:0# 0x0~1464 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1285 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3904e826:::osdmap.61:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~464 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6f00 2 blobs 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbcc000~2000/1000,2000,0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbcc000~2000] 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580f71f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f7260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580f71f0 blob([0xbcc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7260 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c580f71f0) get_ref 0x0~1000 Blob(0x558c580f71f0 blob([0xbcc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7260 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c580f71f0) get_ref init 0x1000, 1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c580f71f0 blob([0xbcc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580f7260 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c580f7278 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580f72d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f7340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580f72d0 blob([!~1000,0xbcd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7340 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c580f72d0) get_ref 0x1000~464 Blob(0x558c580f72d0 blob([!~1000,0xbcd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c580f7340 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c580f72d0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558c580f72d0 blob([!~1000,0xbcd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c580f7340 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c580f7358 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1464 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c579ded88) dirty_range mark inline shard dirty 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3904e826:::osdmap.61:0# 0x0~1464 = 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6f00 0 blobs 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000403'.PSN__1_000000000000000d' <- PSN__1_000000000000000d 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000403'.PSN__1_0000000000000019' <- PSN__1_0000000000000019 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_000000000000000e' <- PSN__1_000000000000000e 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000015' <- PSN__1_0000000000000015 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_000000000000001a' <- PSN__1_000000000000001a 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c582b6f00 0 blobs 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c582b6f00 cost 23819 (4 ios * 4000 + 7819 bytes) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c582b6f00 onodes 0x558c53a91900,0x558c55276c80,0x558c579dec80,0x558c579def00 shared_blobs 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c579ded88) update #-1:3904e826:::osdmap.61:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c579ded88) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3904e826:::osdmap.61:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c579df008) update #-1:c1df972f:::inc_osdmap.61:0# 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore.extentmap(0x558c579df008) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c1df972f:::inc_osdmap.61:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c582b6f00 allocated 0x[bcb000~3000] released 0x[] 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 freelist allocate 0xbcb000~3000 2019-12-07T19:34:38.607+0000 7f8f164bb700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6f00 prepare 2019-12-07T19:34:38.607+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c582b6f00 2019-12-07T19:34:38.607+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6f00 aio_wait 2019-12-07T19:34:38.607+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c582b6f00 2019-12-07T19:34:38.607+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6f00 io_done 2019-12-07T19:34:38.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.607+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:38.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 520 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:38.609+0000 7f8f24457700 15 osd.7 60 enqueue_peering_evt 1.8s5 epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s) 2019-12-07T19:34:38.609+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 521 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:38.609+0000 7f8f24457700 15 osd.7 60 enqueue_peering_evt 1.9s4 epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s) 2019-12-07T19:34:38.609+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e61) pg 0x558c57380000 2019-12-07T19:34:38.609+0000 7f8f0898a700 20 osd.7 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 61, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:38.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 522 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:38.609+0000 7f8f24457700 15 osd.7 60 enqueue_peering_evt 1.0s3 epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s) 2019-12-07T19:34:38.609+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 523 ==== pg_info2(1.es4 1.es4 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:38.609+0000 7f8f24457700 15 osd.7 60 enqueue_peering_evt 1.es4 epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s) 2019-12-07T19:34:38.609+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e61) pg 0x558c5500d000 2019-12-07T19:34:38.609+0000 7f8f04982700 20 osd.7 op_wq _add_slot_waiter 1.0s3 peering, item epoch is 61, will wait on OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:38.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e61) pg 0x558c54d98000 2019-12-07T19:34:38.609+0000 7f8f05984700 20 osd.7 op_wq _add_slot_waiter 1.es4 peering, item epoch is 61, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e61) pg 0x558c5500a000 2019-12-07T19:34:38.609+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.9s4 peering, item epoch is 61, will wait on OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 524 ==== pg_info2(1.as5 1.as5 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:38.609+0000 7f8f24457700 15 osd.7 60 enqueue_peering_evt 1.as5 epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s) 2019-12-07T19:34:38.609+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e61) pg 0x558c57e8b000 2019-12-07T19:34:38.609+0000 7f8f03980700 20 osd.7 op_wq _add_slot_waiter 1.as5 peering, item epoch is 61, will wait on OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:38.609+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 525 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c58122000 con 0x558c54c64000 2019-12-07T19:34:38.609+0000 7f8f24457700 15 osd.7 60 enqueue_peering_evt 1.7s4 epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s) 2019-12-07T19:34:38.609+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.609+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.609+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e61) pg 0x558c54d5f000 2019-12-07T19:34:38.609+0000 7f8f0117b700 20 osd.7 op_wq _add_slot_waiter 1.7s4 peering, item epoch is 61, will wait on OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.609+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5297 < min_flush_size 524288 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9095 < min_flush_size 524288 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579dec80 had 1 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579def00 had 1 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9118 < min_flush_size 524288 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9118 < min_flush_size 524288 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4ceb4f~239e to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4ceb4f 0x4ceb4f~239e to file(ino 16 size 0x4d0000 mtime 2019-12-07T19:34:36.626884+0000 allocated 4d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4a0000]) 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _allocate len 0xeed from 0 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _allocate len 0xeed from 1 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd600000 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd600000~10000/10000,10000,acd600000 2019-12-07T19:34:38.609+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x4e0000 to include allocated 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 76 (was clean) 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x49eb4f 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb4f 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xeed and padding block with 0x113 2019-12-07T19:34:38.609+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x4e0000 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d0eed 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (76) on file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]), flushing log 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 76 len 0x4a crc 0xc0427b4b) 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x4b000~1000 to file(ino 1 size 0x4b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x4b000 0x4b000~1000 to file(ino 1 size 0x4b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x4c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4b000 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x4c000 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000022 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 76 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.002365185s (0.001968440s flush + 0.000396745s kv commit) 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c582b6f00> 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.610+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6f00 kv_submitted 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c582b6f00 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c582b6f00 osr 0x558c54ea4b40 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 49 0x1b000~1000 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7 60 _committed_osd_maps 61..61 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 51 0x1b000~1000 crc 82c9c7cd 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7 60 advance to epoch 61 (<= last 61 <= newest_map 61) 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 50 0x2000~1000 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 51 0x2000~1000 crc ba79fd00 2019-12-07T19:34:38.610+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.610+0000 7f8f141a1700 7 osd.7 61 consume_map version 61 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.8s5 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.0s3 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.1s1 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.9s4 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.as5 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.2s0 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.bs1 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.3s4 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.cs0 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.4s0 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.5s4 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.ds2 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.es4 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.6s2 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.fs5 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 identify_splits_and_merges 1.7s4 e60 to e61 pg_nums {18=16} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 61 prune_pg_created 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:0.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 pending_peering first epoch 61 <= 61, requeueing 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {61=} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 pending_peering first epoch 61 <= 61, requeueing 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:0._wake_pg_slot _wake_pg_slot 1.0s3 to_process <> waiting <> waiting_peering {61=} 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:1.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 pending_peering first epoch 61 <= 61, requeueing 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.9s4 to_process <> waiting <> waiting_peering {61=} 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:2.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 pending_peering first epoch 61 <= 61, requeueing 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:2._wake_pg_slot _wake_pg_slot 1.as5 to_process <> waiting <> waiting_peering {61=} 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:3.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e61) pg 0x558c5500a000 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:4.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e61) pg 0x558c5500d000 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:5.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:6.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 pending_peering first epoch 61 <= 61, requeueing 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {61=} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7:7.consume_map 61 (was 60) 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e61) pg 0x558c57e8b000 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 pending_peering first epoch 61 <= 61, requeueing 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.7s4 to_process <> waiting <> waiting_peering {61=} 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e61) pg 0x558c54d98000 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7:1.update_pg_epoch 1.9s4 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f141a1700 10 osd.7 61 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 60 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.8s5 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.0s3 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.1s1 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.9s4 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.as5 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:38.610+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.2s0 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.bs1 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.3s4 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.cs0 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.4s0 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.5s4 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.ds2 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.es4 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.6s2 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.fs5 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f141a1700 15 osd.7 61 enqueue_peering_evt 1.7s4 epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.es0( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s) 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.9s0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s) 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s) 2019-12-07T19:34:38.610+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s) 2019-12-07T19:34:38.610+0000 7f8f141a1700 7 osd.7 61 activate_map version 61 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c575ef000 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58111800 seq 30 2019-12-07T19:34:38.610+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c584fc600 seq 91 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c57522000 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c578a7b00 seq 51 2019-12-07T19:34:38.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:38.610+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:38.610+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58111800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58111800 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 60 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 60 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:38.610+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584fc600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584fc600 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584fc600 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58111800 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c5736a000 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:38.610+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc600 prepare 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584fc600 2019-12-07T19:34:38.610+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:38.610+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc600 io_done 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e61) pg 0x558c54d5f000 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c5500a000 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.7s4 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 60 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c57676000 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c55141000 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c5513e000 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active advmap 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 60 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] on_active_advmap new purged_snaps [e~1,15~1,1a~1], now [23~1] 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 60 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] share_pg_info 2019-12-07T19:34:38.610+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.7s0( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s) 2019-12-07T19:34:38.610+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.610+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e61/61) v1 -- 0x558c58228e00 con 0x558c54c64000 2019-12-07T19:34:38.610+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578a7b00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578a7b00 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578a7b00 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c5853f800 seq 30 2019-12-07T19:34:38.610+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:38.610+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c583b6f00 seq 90 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7b00 prepare 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578a7b00 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7b00 io_done 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e61/61) v1 -- 0x558c58236e00 con 0x558c54c19800 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853f800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853f800 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b6f00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e61) pg 0x558c57380000 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853f800 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b6f00 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e61/61) v1 -- 0x558c58229500 con 0x558c54c5bc00 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b6f00 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6f00 prepare 2019-12-07T19:34:38.610+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b6f00 2019-12-07T19:34:38.610+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6f00 io_done 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e61/61) v1 -- 0x558c58229c00 con 0x558c54c5b400 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f0918b700 20 osd.7 61 agent_entry empty queue 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c57383000 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 60 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e61/61) v1 -- 0x558c5822a300 con 0x558c54c19c00 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7:7.update_pg_epoch 1.fs5 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 60 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:38.610+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f0297e700 15 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] publish_stats_to_osd 60:207 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f800 prepare 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853f800 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f800 io_done 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c54d5c000 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c54d5f000 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s) 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f05183700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f05183700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111800 prepare 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58111800 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111800 io_done 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] state: Active: handling ActMap 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c57e8b000 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] Active: kicking snap trim 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 60 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0297e700 7 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] state: handle ActMap primary 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c57e88000 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f0297e700 15 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] publish_stats_to_osd 60:208 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58500000 seq 55 2019-12-07T19:34:38.610+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s0 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] activate_map: Not dirtying info: last_persisted is 60 while current is 61 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 60 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active advmap 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58500000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58500000 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58500000 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] on_active_advmap new purged_snaps [e~1,15~1,1a~1], now [23~1] 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] share_pg_info 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500000 prepare 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58500000 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500000 io_done 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585e4600 seq 72 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c57392a00 con 0x558c54c64000 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c57380000 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c57ed2000 con 0x558c54c19800 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585e4600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585e4600 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585e4600 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4600 prepare 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585e4600 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c5500d000 2019-12-07T19:34:38.610+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c57ed5800 con 0x558c54c5bc00 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4600 io_done 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c54e06300 con 0x558c54c5b400 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.610+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c548ab000 2019-12-07T19:34:38.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.610+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c54e05c00 con 0x558c54c19c00 2019-12-07T19:34:38.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 NullEvt) prio 255 cost 10 e61) pg 0x558c54d98000 2019-12-07T19:34:38.610+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:38.610+0000 7f8f07988700 15 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] publish_stats_to_osd 59:173 2019-12-07T19:34:38.610+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] state: Started advmap 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map: 61 2019-12-07T19:34:38.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:38.610+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 60 -> 61 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] hit_set_clear 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 60 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] agent_stop 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] handle_activate_map: 61 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] activate_map 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[e~1,15~1,1a~1,23~1]] state: Active advmap 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] state: Active: handling ActMap 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] Active: kicking snap trim 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] on_active_advmap new purged_snaps [e~1,15~1,1a~1], now [23~1] 2019-12-07T19:34:38.610+0000 7f8f07988700 7 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] state: handle ActMap primary 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:38.610+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] share_pg_info 2019-12-07T19:34:38.610+0000 7f8f07988700 15 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] publish_stats_to_osd 60:174 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07988700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.610+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c58111b00 seq 32 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c5822aa00 con 0x558c54c64000 2019-12-07T19:34:38.610+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58111b00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58111b00 onodes shared_blobs 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58111b00 allocated 0x[] released 0x[] 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c5822b100 con 0x558c54c19800 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111b00 prepare 2019-12-07T19:34:38.610+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58111b00 2019-12-07T19:34:38.610+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111b00 io_done 2019-12-07T19:34:38.610+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.610+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 8 submitting 8 deferred done 0 stable 0 2019-12-07T19:34:38.610+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c5822b800 con 0x558c54c5bc00 2019-12-07T19:34:38.610+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c5509c000 con 0x558c54c5b400 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 -- 0x558c5509c700 con 0x558c54c19c00 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4472 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5590 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:38.611+0000 7f8f0297e700 15 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] publish_stats_to_osd 59:161 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6708 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] state: Started advmap 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7826 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8944 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8967 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8967 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] hit_set_clear 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d0eed~2307 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d0eed 0x4d0eed~2307 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] agent_stop 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a0eed 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 61 advance_pg new pool opts old pool opts 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xeed 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1f4 and padding block with 0xe0c 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] handle_activate_map: 61 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] activate_map 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] state: Active: handling ActMap 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] Active: kicking snap trim 2019-12-07T19:34:38.611+0000 7f8f0297e700 7 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] state: handle ActMap primary 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:38.611+0000 7f8f0297e700 15 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] publish_stats_to_osd 60:162 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] activate_map: Not dirtying info: last_persisted is 59 while current is 61 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d31f4 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 NullEvt 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 8 cleaned 0 in 0.000152240s (0.000000072s flush + 0.000152168s kv commit) 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584fc600,0x558c578a7b00,0x558c583b6f00,0x558c5853f800,0x558c58111800,0x558c58500000,0x558c585e4600,0x558c58111b00> 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c585e4900 seq 49 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc600 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584fc600 2019-12-07T19:34:38.611+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc600 finishing 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584fc600 onodes 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:38.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584fc600 done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584fc600 [] 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7b00 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578a7b00 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585e4900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585e4900 onodes shared_blobs 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578a7b00 finishing 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585e4900 allocated 0x[] released 0x[] 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578a7b00 onodes 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578a7b00 done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578a7b00 [] 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6f00 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b6f00 2019-12-07T19:34:38.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b6f00 finishing 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4900 prepare 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b6f00 onodes 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b6f00 done 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585e4900 2019-12-07T19:34:38.611+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:38.611+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4900 io_done 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b6f00 [] 2019-12-07T19:34:38.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f800 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853f800 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853f800 finishing 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853f800 onodes 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853f800 done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853f800 [] 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111800 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58111800 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111800 finishing 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58111800 onodes 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58111800 done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58111800 [] 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500000 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58500000 2019-12-07T19:34:38.611+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500000 finishing 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58500000 onodes 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58500000 done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:38.611+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58500000 [] 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d31f4~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4600 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585e4600 2019-12-07T19:34:38.611+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d31f4 0x4d31f4~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4600 finishing 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a31f4 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585e4600 onodes 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1f4 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4600 done 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x669 and padding block with 0x997 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585e4600 [] 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111b00 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58111b00 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58111b00 finishing 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58111b00 onodes 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58111b00 done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58111b00 [] 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.611+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d3669 2019-12-07T19:34:38.611+0000 7f8f24457700 0 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.611+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000223233s (0.000000073s flush + 0.000223160s kv commit) 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.611+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585e4900> 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4900 kv_submitted 2019-12-07T19:34:38.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 61 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585e4900 2019-12-07T19:34:38.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 msgr2=0x558c54c38680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4900 finishing 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585e4900 onodes 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4900 done 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585e4900 [] 2019-12-07T19:34:38.611+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.611+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 crc :-1 s=READY pgs=11 cs=2 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:38.611+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.611+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 crc :-1 s=READY pgs=11 cs=2 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:38.611+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.611+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 300 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c5509c000 con 0x558c54c5bc00 2019-12-07T19:34:38.612+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.6s2 epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:38.612+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.612+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 301 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c5822b800 con 0x558c54c5bc00 2019-12-07T19:34:38.612+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.1s1 epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s) 2019-12-07T19:34:38.612+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 302 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c5822b800 con 0x558c54c5bc00 2019-12-07T19:34:38.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e61) pg 0x558c5736a000 2019-12-07T19:34:38.612+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.fs5 epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s) 2019-12-07T19:34:38.612+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s) 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e61) pg 0x558c57383000 2019-12-07T19:34:38.612+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s) 2019-12-07T19:34:38.612+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:38.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c584fc900 seq 26 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:38.612+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:38.612+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 304 ==== MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=157, reqid=client.4540.0:453, at_version=61'73, trim_to=0'0, roll_forward_to=57'71)) v2 ==== 3191+0+0 (crc 0 0 0) 0x558c574a3000 con 0x558c54c5bc00 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:38.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c5875d200 seq 33 2019-12-07T19:34:38.612+0000 7f8f23c56700 15 osd.7 61 enqueue_op 0x558c57334f20 prio 127 cost 0 latency 0.000024 epoch 61 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=157, reqid=client.4540.0:453, at_version=61'73, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:38.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:38.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:38.612+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=157, reqid=client.4540.0:453, at_version=61'73, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) 2019-12-07T19:34:38.612+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:38.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:38.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584fc900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584fc900 onodes shared_blobs 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:38.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584fc900 allocated 0x[] released 0x[] 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875d200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875d200 onodes shared_blobs 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875d200 allocated 0x[] released 0x[] 2019-12-07T19:34:38.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc900 prepare 2019-12-07T19:34:38.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584fc900 2019-12-07T19:34:38.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc900 io_done 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d200 prepare 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875d200 2019-12-07T19:34:38.612+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d200 io_done 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s)) prio 255 cost 10 e61) pg 0x558c54d5c000 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.612+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 303 ==== osd_map(61..61 src has 1..61) v4 ==== 943+0+0 (crc 0 0 0) 0x558c58be2240 con 0x558c54c5bc00 2019-12-07T19:34:38.612+0000 7f8f1199c700 20 osd.7 61 OSD::ms_dispatch: osd_map(61..61 src has 1..61) v4 2019-12-07T19:34:38.612+0000 7f8f1199c700 10 osd.7 61 do_waiters -- start 2019-12-07T19:34:38.612+0000 7f8f1199c700 10 osd.7 61 do_waiters -- finish 2019-12-07T19:34:38.612+0000 7f8f1199c700 20 osd.7 61 _dispatch 0x558c58be2240 osd_map(61..61 src has 1..61) v4 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 3(0) info: 1.6s0( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 70.116514081s ub 70.116514081s int 16s) 2019-12-07T19:34:38.612+0000 7f8f1199c700 3 osd.7 61 handle_osd_map epochs [61,61], i have 61, src has [1,61] 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:38.612+0000 7f8f1199c700 10 osd.7 61 no new maps here, dropping 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d3669~8d3 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5853fb00 seq 111 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d3669 0x4d3669~8d3 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a3669 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x669 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf3c and padding block with 0xc4 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853fb00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853fb00 onodes shared_blobs 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853fb00 allocated 0x[] released 0x[] 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853fb00 prepare 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853fb00 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853fb00 io_done 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=157, reqid=client.4540.0:453, at_version=61'73, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) queued 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=157, reqid=client.4540.0:453, at_version=61'73, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) pg 0x558c54d5c000 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 61 dequeue_op 0x558c57334f20 prio 127 cost 0 latency 0.000144 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=157, reqid=client.4540.0:453, at_version=61'73, trim_to=0'0, roll_forward_to=57'71)) v2 pg pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[23~1]] 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 61 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 60 -> 61 (as per caller) 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=157, reqid=client.4540.0:453, at_version=61'73, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.add_oid 1:6c205177:::smithi13140796-8:26 1b,20,21,25,26 2019-12-07T19:34:38.612+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d3f3c 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:26 got.empty() 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6c205177:::smithi13140796-8:26 1b,20,21,25,26 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000150099s (0.000000083s flush + 0.000150016s kv commit) 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_000000000000001B_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000020_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000021_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000025_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 snap_mapper.add_oid set SNA_1_0000000000000026_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d3f3c~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d3f3c 0x4d3f3c~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a3f3c 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf3c 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3b1 and padding block with 0xc4f 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 57'71 (0'0,57'71] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[23~1]] append_log log((0'0,57'71], crt=57'71) [61'72 (53'63) clone 1:6c205177:::smithi13140796-8:26 by unknown.0.0:0 2019-12-07T19:34:16.641390+0000 0 snaps [26,25,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,61'73 (53'63) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:453 2019-12-07T19:34:38.611624+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0] 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'72 (0'0,61'72] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=57'71 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] add_log_entry 61'72 (53'63) clone 1:6c205177:::smithi13140796-8:26 by unknown.0.0:0 2019-12-07T19:34:16.641390+0000 0 snaps [26,25,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584fc900,0x558c5875d200> 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc900 kv_submitted 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584fc900 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc900 finishing 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584fc900 onodes 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584fc900 done 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'73 (0'0,61'73] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=57'71 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] add_log_entry 61'73 (53'63) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:453 2019-12-07T19:34:38.611624+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584fc900 [] 2019-12-07T19:34:38.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d200 kv_submitted 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875d200 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'73 (0'0,61'73] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=57'71 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log approx pg log length = 73 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d200 finishing 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875d200 onodes 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875d200 done 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875d200 [] 2019-12-07T19:34:38.612+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.612+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'73 (0'0,61'73] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=57'71 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:38.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 61'72, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d43b1 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116389s (0.000000066s flush + 0.000116323s kv commit) 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853fb00> 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853fb00 kv_submitted 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853fb00 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853fb00 finishing 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853fb00 onodes 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853fb00 done 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:38.612+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853fb00 [] 2019-12-07T19:34:38.612+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c550a2000 seq 112 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:26# key 0x8280000000000000016c2051'w!smithi13140796-8!='0x0000000000000026ffffffffffffffff'o' 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:38.612+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:26# 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1286 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.onode(0x558c575ba280).flush flush done 2019-12-07T19:34:38.612+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# 0x0 2019-12-07T19:34:38.612+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:26# 0x0~5f000 -> 0x0~5f000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x0~2000: 0x0~2000 Blob(0x558c575bd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57896380 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57896380) get_ref 0x0~2000 Blob(0x558c57896380 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57896380) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57896380 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x2000~2000: 0x0~2000 Blob(0x558c575bd420 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c578962a0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c578962a0) get_ref 0x0~2000 Blob(0x558c578962a0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c578962a0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c578962a0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57314230 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57f51570 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57f51570) get_ref 0x0~2000 Blob(0x558c57f51570 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57f51570) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c57f51570 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x6000~2000: 0x0~2000 Blob(0x558c57314150 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c55278a10 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55278a10) get_ref 0x0~2000 Blob(0x558c55278a10 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55278a10) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c55278a10 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x8000~2000: 0x0~2000 Blob(0x558c57314070 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c550607e0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c550607e0) get_ref 0x0~2000 Blob(0x558c550607e0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c550607e0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c550607e0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0xa000~2000: 0x0~2000 Blob(0x558c573145b0 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c55060770 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55060770) get_ref 0x0~2000 Blob(0x558c55060770 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55060770) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c55060770 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57314700 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd180) get_ref 0x0~2000 Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd180) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0xe000~2000: 0x0~2000 Blob(0x558c573147e0 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c550b2700 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c550b2700) get_ref 0x0~2000 Blob(0x558c550b2700 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c550b2700) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c550b2700 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x10000~2000: 0x0~2000 Blob(0x558c573148c0 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c579d6e70 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c579d6e70) get_ref 0x0~2000 Blob(0x558c579d6e70 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c579d6e70) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c579d6e70 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x12000~1000: 0x0~1000 Blob(0x558c573149a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c55060690 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55060690) get_ref 0x0~1000 Blob(0x558c55060690 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55060690) get_ref init 0x1000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x12000~1000: 0x0~1000 Blob(0x558c55060690 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x13000~2000: 0x0~2000 Blob(0x558c57efd570 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10922 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efd570 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efd570 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c5898b180 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c5898b180) get_ref 0x0~2000 Blob(0x558c5898b180 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c5898b180) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x13000~2000: 0x0~2000 Blob(0x558c5898b180 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x15000~2000: 0x0~2000 Blob(0x558c57efd650 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10923 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efd650 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efd650 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57654f50 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57654f50) get_ref 0x0~2000 Blob(0x558c57654f50 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57654f50) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x15000~2000: 0x0~2000 Blob(0x558c57654f50 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x17000~2000: 0x0~2000 Blob(0x558c57efd730 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10924 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efd730 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efd730 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c579d6fc0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c579d6fc0) get_ref 0x0~2000 Blob(0x558c579d6fc0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c579d6fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x17000~2000: 0x0~2000 Blob(0x558c579d6fc0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x19000~2000: 0x0~2000 Blob(0x558c57efd810 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10925 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efd810 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efd810 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575bc620 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) get_ref 0x0~2000 Blob(0x558c575bc620 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc620) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x19000~2000: 0x0~2000 Blob(0x558c575bc620 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x1b000~2000: 0x0~2000 Blob(0x558c57efd8f0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10926 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efd8f0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efd8f0 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c54677880 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c54677880) get_ref 0x0~2000 Blob(0x558c54677880 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c54677880) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x1b000~2000: 0x0~2000 Blob(0x558c54677880 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x1d000~2000: 0x0~2000 Blob(0x558c57efd9d0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10927 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efd9d0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efd9d0 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575bc700 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) get_ref 0x0~2000 Blob(0x558c575bc700 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c575bc700) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x1d000~2000: 0x0~2000 Blob(0x558c575bc700 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x1f000~2000: 0x0~2000 Blob(0x558c57efdab0 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10928 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efdab0 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efdab0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd340 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd340) get_ref 0x0~2000 Blob(0x558c57efd340 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd340) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x1f000~2000: 0x0~2000 Blob(0x558c57efd340 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x21000~2000: 0x0~2000 Blob(0x558c57efdb90 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10929 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efdb90 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efdb90 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57396cb0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57396cb0) get_ref 0x0~2000 Blob(0x558c57396cb0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c57396cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x21000~2000: 0x0~2000 Blob(0x558c57396cb0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x23000~2000: 0x0~2000 Blob(0x558c57efdc70 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10930 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efdc70 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efdc70 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c5523a1c0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c5523a1c0) get_ref 0x0~2000 Blob(0x558c5523a1c0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c5523a1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c5523a1c0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x25000~2000: 0x0~2000 Blob(0x558c580a4930 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10931 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c580a4930 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c580a4930 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c5742ae70 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) get_ref 0x0~2000 Blob(0x558c5742ae70 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ae70) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x27000~2000: 0x0~2000 Blob(0x558c54e57420 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10932 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54e57420 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54e57420 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c581d6c40 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6c40) get_ref 0x0~2000 Blob(0x558c581d6c40 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6c40) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c581d6c40 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x29000~2000: 0x0~2000 Blob(0x558c57a070a0 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10933 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57a070a0 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57a070a0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c574d3a40 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3a40) get_ref 0x0~2000 Blob(0x558c574d3a40 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3a40) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c574d3a40 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c5792d810 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10934 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5792d810 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5792d810 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c55264070 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55264070) get_ref 0x0~2000 Blob(0x558c55264070 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c55264070) get_ref init 0x2000, 1000 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c55264070 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c5809b5e0 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10935 2019-12-07T19:34:38.612+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5809b5e0 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 sbid 0x0)) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5809b5e0 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=1)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c5742ad90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2)))) 2019-12-07T19:34:38.612+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) get_ref 0x0~2000 Blob(0x558c5742ad90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c5742ad90) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x2f000~1000: 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x5c2000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10936 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5792d8f0 blob([!~1000,0x5c2000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5792d8f0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57efd2d0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd2d0) get_ref 0x1000~1000 Blob(0x558c57efd2d0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x2f000~1000: 0x1000~1000 Blob(0x558c57efd2d0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x30000~2000: 0x0~2000 Blob(0x558c57efd3b0 blob([0x5c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10937 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c57efd3b0 blob([0x5c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c57efd3b0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575bd6c0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) get_ref 0x0~2000 Blob(0x558c575bd6c0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x32000~2000: 0x0~2000 Blob(0x558c577f3e30 blob([0x5c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10938 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c577f3e30 blob([0x5c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c577f3e30 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c579d6e00 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c579d6e00) get_ref 0x0~2000 Blob(0x558c579d6e00 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c579d6e00) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c579d6e00 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x34000~2000: 0x0~2000 Blob(0x558c5792d730 blob([0x5c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10939 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5792d730 blob([0x5c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5792d730 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c55278f50 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c55278f50) get_ref 0x0~2000 Blob(0x558c55278f50 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c55278f50) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c55278f50 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x36000~2000: 0x0~2000 Blob(0x558c5792d650 blob([0x5c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10940 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5792d650 blob([0x5c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5792d650 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575f6070 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6070) get_ref 0x0~2000 Blob(0x558c575f6070 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6070) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c575f6070 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x38000~2000: 0x0~2000 Blob(0x558c54df8a80 blob([0x5cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10941 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df8a80 blob([0x5cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df8a80 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c574d3030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) get_ref 0x0~2000 Blob(0x558c574d3030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d3030) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c574d3030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c54df8b60 blob([0x5cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10942 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df8b60 blob([0x5cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df8b60 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c581d6b60 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6b60) get_ref 0x0~2000 Blob(0x558c581d6b60 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6b60) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c581d6b60 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c54df8c40 blob([0x5cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10943 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df8c40 blob([0x5cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df8c40 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c54cf82a0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf82a0) get_ref 0x0~2000 Blob(0x558c54cf82a0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c54cf82a0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c54cf82a0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c54df8d20 blob([0x5d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10944 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df8d20 blob([0x5d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df8d20 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575f6000 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6000) get_ref 0x0~2000 Blob(0x558c575f6000 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6000) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c575f6000 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x40000~2000: 0x0~2000 Blob(0x558c54df8e00 blob([0x5d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10945 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df8e00 blob([0x5d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df8e00 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c574d30a0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) get_ref 0x0~2000 Blob(0x558c574d30a0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d30a0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x42000~2000: 0x0~2000 Blob(0x558c54df8ee0 blob([0x5d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10946 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df8ee0 blob([0x5d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df8ee0 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c577f3ab0 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3ab0) get_ref 0x0~2000 Blob(0x558c577f3ab0 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c577f3ab0 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x44000~2000: 0x0~2000 Blob(0x558c54df8fc0 blob([0x5d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10947 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df8fc0 blob([0x5d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df8fc0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c574e0690 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574e0690) get_ref 0x0~2000 Blob(0x558c574e0690 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574e0690) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c574e0690 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x46000~2000: 0x0~2000 Blob(0x558c54df90a0 blob([0x5d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10948 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df90a0 blob([0x5d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df90a0 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575f6380 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6380) get_ref 0x0~2000 Blob(0x558c575f6380 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f6380) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c575f6380 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x48000~2000: 0x0~2000 Blob(0x558c54df9180 blob([0x5db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10949 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df9180 blob([0x5db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df9180 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57566e00 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) get_ref 0x0~2000 Blob(0x558c57566e00 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c57566e00) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c57566e00 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c54df9260 blob([0x5dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10950 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df9260 blob([0x5dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df9260 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c581d6d20 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6d20) get_ref 0x0~2000 Blob(0x558c581d6d20 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6d20) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c581d6d20 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c577f3b20 blob([0x5df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10951 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c577f3b20 blob([0x5df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c577f3b20 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c573144d0 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c573144d0) get_ref 0x0~2000 Blob(0x558c573144d0 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c573144d0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c573144d0 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c5809bea0 blob([0x5e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10952 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5809bea0 blob([0x5e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5809bea0 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c54df8070 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8070) get_ref 0x0~2000 Blob(0x558c54df8070 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8070) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c54df8070 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x50000~2000: 0x0~2000 Blob(0x558c5809bce0 blob([0x5e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10953 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5809bce0 blob([0x5e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5809bce0 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c574d2770 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) get_ref 0x0~2000 Blob(0x558c574d2770 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d2770) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c574d2770 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x52000~2000: 0x0~2000 Blob(0x558c5809be30 blob([0x5e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10954 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5809be30 blob([0x5e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5809be30 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c5829cf50 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c5829cf50) get_ref 0x0~2000 Blob(0x558c5829cf50 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c5829cf50) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c5829cf50 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x54000~2000: 0x0~2000 Blob(0x558c5809bc00 blob([0x5e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10955 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c5809bc00 blob([0x5e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c5809bc00 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c574d27e0 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) get_ref 0x0~2000 Blob(0x558c574d27e0 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d27e0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x56000~2000: 0x0~2000 Blob(0x558c54df80e0 blob([0x5e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10956 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df80e0 blob([0x5e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df80e0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575f7ce0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7ce0) get_ref 0x0~2000 Blob(0x558c575f7ce0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c575f7ce0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x58000~2000: 0x0~2000 Blob(0x558c54df9a40 blob([0x5eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10957 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df9a40 blob([0x5eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df9a40 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c574d39d0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d39d0) get_ref 0x0~2000 Blob(0x558c574d39d0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c574d39d0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c574d39d0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c54df9b90 blob([0x5ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10958 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df9b90 blob([0x5ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df9b90 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c575f7c00 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7c00) get_ref 0x0~2000 Blob(0x558c575f7c00 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7c00) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c575f7c00 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c54df9c70 blob([0x5ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10959 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df9c70 blob([0x5ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df9c70 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c57314310 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c57314310) get_ref 0x0~2000 Blob(0x558c57314310 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c57314310) get_ref init 0x2000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c57314310 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup src 0x5e000~1000: 0x0~1000 Blob(0x558c54df9d50 blob([0x5f1000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10960 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared Blob(0x558c54df9d50 blob([0x5f1000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 sbid 0x0)) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) make_blob_shared now Blob(0x558c54df9d50 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=1)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup new Blob(0x558c54df8230 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8230) get_ref 0x0~1000 Blob(0x558c54df8230 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8230) get_ref init 0x1000, 1000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dup dst 0x5e000~1000: 0x0~1000 Blob(0x558c54df8230 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2)))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark inline shard dirty 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dirty_range mark inline shard dirty 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:26# = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# snapset 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# snapset = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# 1 keys 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# 1 keys = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# hinfo_key (42 bytes) 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# hinfo_key (42 bytes) = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000001B_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- SNA_1_000000000000001B_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- SNA_1_0000000000000020_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- SNA_1_0000000000000021_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000025_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- SNA_1_0000000000000025_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000026_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- SNA_1_0000000000000026_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000061.00000000000000000072' <- 0000000061.00000000000000000072 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000061.00000000000000000073' <- 0000000061.00000000000000000073 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a2000 cost 8849 (1 ios * 4000 + 4849 bytes) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a2000 onodes 0x558c575ba280,0x558c57c05b80 shared_blobs 0x558c54df8000,0x558c54df8af0,0x558c54df8bd0,0x558c54df8cb0,0x558c54df8d90,0x558c54df8e70,0x558c54df8f50,0x558c54df9030,0x558c54df9110,0x558c54df91f0,0x558c54df9ab0,0x558c54df9c00,0x558c54df9ce0,0x558c54df9dc0,0x558c54e43ce0,0x558c54e56690,0x558c54e56bd0,0x558c54e56cb0,0x558c54e56d90,0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c573970a0,0x558c575bd3b0,0x558c577f3a40,0x558c577f3c70,0x558c57e63490,0x558c57efd5e0,0x558c57efd6c0,0x558c57efd7a0,0x558c57efd880,0x558c57efd960,0x558c57efda40,0x558c57efdb20,0x558c57efdc00,0x558c5809b490,0x558c5809b500,0x558c5809b570,0x558c5809bc70,0x558c5809bd50,0x558c5809bdc0,0x558c5809bf80,0x558c580a4a10 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update inline shard 1335 bytes from 49 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore.extentmap(0x558c575ba388) reshard 0x[0,ffffffff) of 0 shards on 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard new shard 0x0 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard new shard 0x23000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard new shard 0x46000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard old [] 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head# force 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update shard 0x23000 is 505 bytes (was 0) from 18 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update shard 0x46000 is 353 bytes (was 0) from 13 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 602 (600 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) update 2#1:6c205177:::smithi13140796-8:26# 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) update inline shard 1335 bytes from 49 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore.extentmap(0x558c57c05c88) reshard 0x[0,ffffffff) of 0 shards on 2#1:6c205177:::smithi13140796-8:26# 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard new shard 0x0 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard new shard 0x23000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard new shard 0x46000 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard old [] 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) update 2#1:6c205177:::smithi13140796-8:26# force 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) update shard 0x23000 is 505 bytes (was 0) from 18 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) update shard 0x46000 is 353 bytes (was 0) from 13 extents 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:26# is 425 (423 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aba is 11 SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abd is 11 SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abe is 11 SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abf is 11 SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac0 is 11 SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac1 is 11 SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac2 is 11 SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac3 is 11 SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac4 is 11 SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac5 is 11 SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acd is 11 SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ace is 11 SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acf is 11 SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad0 is 11 SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab8 is 11 SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab4 is 11 SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab7 is 11 SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab6 is 11 SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab5 is 11 SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac7 is 11 SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac6 is 11 SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab9 is 11 SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab2 is 11 SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaa is 11 SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aab is 11 SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aac is 11 SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aad is 11 SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aae is 11 SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaf is 11 SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab0 is 11 SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab1 is 11 SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abc is 11 SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abb is 11 SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acc is 11 SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aca is 11 SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac8 is 11 SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac9 is 11 SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acb is 11 SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab3 is 11 SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2))) 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a2000 allocated 0x[] released 0x[] 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2000 prepare 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a2000 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2000 io_done 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'73 (0'0,61'73] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=57'71 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] op_applied version 61'73 2019-12-07T19:34:38.613+0000 7f8f0197c700 10 osd.7 61 dequeue_op 0x558c57334f20 finish 2019-12-07T19:34:38.613+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8485 < min_flush_size 524288 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57c05b80 had 1 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8508 < min_flush_size 524288 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8508 < min_flush_size 524288 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d43b1~213c to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d43b1 0x4d43b1~213c to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a43b1 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3b1 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x4ed and padding block with 0xb13 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d64ed 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.614+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000277975s (0.000000141s flush + 0.000277834s kv commit) 2019-12-07T19:34:38.614+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.614+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 305 ==== MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=158, reqid=client.4540.0:454, at_version=61'74, trim_to=0'0, roll_forward_to=57'71)) v2 ==== 2142+0+0 (crc 0 0 0) 0x558c574a4800 con 0x558c54c5bc00 2019-12-07T19:34:38.614+0000 7f8f23c56700 15 osd.7 61 enqueue_op 0x558c57335080 prio 127 cost 0 latency 0.000032 epoch 61 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=158, reqid=client.4540.0:454, at_version=61'74, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:38.614+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=158, reqid=client.4540.0:454, at_version=61'74, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) 2019-12-07T19:34:38.614+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 306 ==== MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=159, reqid=client.4540.0:455, at_version=61'75, trim_to=0'0, roll_forward_to=57'71)) v2 ==== 2368+0+0 (crc 0 0 0) 0x558c5822f800 con 0x558c54c5bc00 2019-12-07T19:34:38.614+0000 7f8f23c56700 15 osd.7 61 enqueue_op 0x558c57335ce0 prio 127 cost 0 latency 0.000018 epoch 61 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=159, reqid=client.4540.0:455, at_version=61'75, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:38.614+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=159, reqid=client.4540.0:455, at_version=61'75, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=158, reqid=client.4540.0:454, at_version=61'74, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) queued 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=158, reqid=client.4540.0:454, at_version=61'74, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) pg 0x558c54d5c000 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 osd.7 61 dequeue_op 0x558c57335080 prio 127 cost 0 latency 0.001018 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=158, reqid=client.4540.0:454, at_version=61'74, trim_to=0'0, roll_forward_to=57'71)) v2 pg pg[1.6s2( v 61'73 (0'0,61'73] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] 2019-12-07T19:34:38.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 323 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c5822b800 con 0x558c54c5b400 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'73 (0'0,61'73] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=158, reqid=client.4540.0:454, at_version=61'74, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:38.615+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.bs1 epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s) 2019-12-07T19:34:38.615+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'73 (0'0,61'73] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log log((0'0,61'73], crt=57'71) [61'74 (61'73) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:454 2019-12-07T19:34:38.611663+0000 0 ObjectCleanRegions clean_offsets: [1556480~18446744073707995135], clean_omap: 0, new_object: 0] 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e61) pg 0x558c575ef000 2019-12-07T19:34:38.615+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 324 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558c5822b800 con 0x558c54c5b400 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'74 (0'0,61'74] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'73 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] add_log_entry 61'74 (61'73) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:454 2019-12-07T19:34:38.611663+0000 0 ObjectCleanRegions clean_offsets: [1556480~18446744073707995135], clean_omap: 0, new_object: 0 2019-12-07T19:34:38.615+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.3s4 epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s) 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'74 (0'0,61'74] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'73 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log approx pg log length = 74 2019-12-07T19:34:38.615+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'74 (0'0,61'74] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'73 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.bs0( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s) 2019-12-07T19:34:38.615+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 61'74, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:38.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:38.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e61) pg 0x558c5513e000 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c586a3800 seq 60 2019-12-07T19:34:38.615+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:38.615+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s) 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:38.615+0000 7f8f07187700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a3800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a3800 onodes shared_blobs 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a3800 allocated 0x[] released 0x[] 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c550a2300 seq 113 2019-12-07T19:34:38.615+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3800 prepare 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a3800 2019-12-07T19:34:38.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:38.615+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3800 io_done 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:head#4a key 0x8280000000000000016c2051'w!smithi13140796-8!='0xfffffffffffffffe000000000000004a'o' 2019-12-07T19:34:38.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c586e1b00 seq 70 2019-12-07T19:34:38.615+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:38.615+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:38.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:38.615+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:head#4a 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:38.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:38.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e1b00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e1b00 onodes shared_blobs 2019-12-07T19:34:38.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:38.615+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:head#4a = 0 2019-12-07T19:34:38.615+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# -> 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1287 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.onode(0x558c57c05b80).flush flush done 2019-12-07T19:34:38.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1b00 prepare 2019-12-07T19:34:38.615+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0 2019-12-07T19:34:38.615+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e1b00 2019-12-07T19:34:38.615+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1b00 io_done 2019-12-07T19:34:38.615+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# -> 2#1:6c205177:::smithi13140796-8:head# 0x0~5f000 -> 0x0~5f000 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.615+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x0~2000: 0x0~2000 Blob(0x558c57896380 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54e56f50 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c54e56f50) get_ref 0x0~2000 Blob(0x558c54e56f50 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d64ed~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c54e56f50) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c54e56f50 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d64ed 0x4d64ed~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c578962a0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54d3c310 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3c310) get_ref 0x0~2000 Blob(0x558c54d3c310 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a64ed 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3c310) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x4ed 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c54d3c310 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x962 and padding block with 0x69e 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57f51570 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5792dce0 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c5792dce0) get_ref 0x0~2000 Blob(0x558c5792dce0 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c5792dce0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c5792dce0 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c55278a10 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b3730 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c550b3730) get_ref 0x0~2000 Blob(0x558c550b3730 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c550b3730) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c550b3730 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c550607e0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54d750a0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) get_ref 0x0~2000 Blob(0x558c54d750a0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c54d750a0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c55060770 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c580d7e30 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c580d7e30) get_ref 0x0~2000 Blob(0x558c580d7e30 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c580d7e30) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c580d7e30 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c574d28c0 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) get_ref 0x0~2000 Blob(0x558c574d28c0 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c574d28c0 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c550b2700 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c575f7c70 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7c70) get_ref 0x0~2000 Blob(0x558c575f7c70 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7c70) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c575f7c70 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d6962 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c579d6e70 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5829c000 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c5829c000) get_ref 0x0~2000 Blob(0x558c5829c000 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c5829c000) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5829c000 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x12000~1000: 0x0~1000 Blob(0x558c55060690 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c547402a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c547402a0) get_ref 0x0~1000 Blob(0x558c547402a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c547402a0) get_ref init 0x1000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x12000~1000: 0x0~1000 Blob(0x558c547402a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000090698s (0.000000119s flush + 0.000090579s kv commit) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x13000~2000: 0x0~2000 Blob(0x558c5898b180 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57efd420 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd420) get_ref 0x0~2000 Blob(0x558c57efd420 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd420) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x13000~2000: 0x0~2000 Blob(0x558c57efd420 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x15000~2000: 0x0~2000 Blob(0x558c57654f50 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57c42930 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42930) get_ref 0x0~2000 Blob(0x558c57c42930 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42930) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x15000~2000: 0x0~2000 Blob(0x558c57c42930 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x17000~2000: 0x0~2000 Blob(0x558c579d6fc0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c581d6bd0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6bd0) get_ref 0x0~2000 Blob(0x558c581d6bd0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x17000~2000: 0x0~2000 Blob(0x558c581d6bd0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x19000~2000: 0x0~2000 Blob(0x558c575bc620 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57e63e30 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57e63e30) get_ref 0x0~2000 Blob(0x558c57e63e30 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57e63e30) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x19000~2000: 0x0~2000 Blob(0x558c57e63e30 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d6962~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x1b000~2000: 0x0~2000 Blob(0x558c54677880 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57efd490 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd490) get_ref 0x0~2000 Blob(0x558c57efd490 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d6962 0x4d6962~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd490) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x1b000~2000: 0x0~2000 Blob(0x558c57efd490 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x1d000~2000: 0x0~2000 Blob(0x558c575bc700 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a6962 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x962 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57a07dc0 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.615+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xdd7 and padding block with 0x229 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57a07dc0) get_ref 0x0~2000 Blob(0x558c57a07dc0 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c57a07dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x1d000~2000: 0x0~2000 Blob(0x558c57a07dc0 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x1f000~2000: 0x0~2000 Blob(0x558c57efd340 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c581d6cb0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6cb0) get_ref 0x0~2000 Blob(0x558c581d6cb0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c550a2000,0x558c586a3800> 2019-12-07T19:34:38.615+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x1f000~2000: 0x0~2000 Blob(0x558c581d6cb0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:38.615+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2000 kv_submitted 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a2000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x21000~2000: 0x0~2000 Blob(0x558c57396cb0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2000 finishing 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a2000 onodes 0x558c575ba280,0x558c57c05b80 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54d3df10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a2000 done 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a2300 prepare 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a2000 [] 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df10) get_ref 0x0~2000 Blob(0x558c54d3df10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df10) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x21000~2000: 0x0~2000 Blob(0x558c54d3df10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x23000~2000: 0x0~2000 Blob(0x558c5523a1c0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57efd500 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3800 kv_submitted 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd500) get_ref 0x0~2000 Blob(0x558c57efd500 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a3800 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd500) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3800 finishing 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c57efd500 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a3800 onodes 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a3800 done 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a3800 [] 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x25000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57efd1f0 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd1f0) get_ref 0x0~2000 Blob(0x558c57efd1f0 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=159, reqid=client.4540.0:455, at_version=61'75, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) queued 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c57efd1f0 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x27000~2000: 0x0~2000 Blob(0x558c581d6c40 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c580d7f80 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c580d7f80) get_ref 0x0~2000 Blob(0x558c580d7f80 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d6dd7 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c580d7f80) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c580d7f80 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x29000~2000: 0x0~2000 Blob(0x558c574d3a40 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57efd260 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd260) get_ref 0x0~2000 Blob(0x558c57efd260 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd260) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c57efd260 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c55264070 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5829c7e0 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c5829c7e0) get_ref 0x0~2000 Blob(0x558c5829c7e0 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000112521s (0.000000073s flush + 0.000112448s kv commit) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c5829c7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c5829c7e0 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c581d6d90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6d90) get_ref 0x0~2000 Blob(0x558c581d6d90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6d90) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c581d6d90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2f000~1000: 0x1000~1000 Blob(0x558c57efd2d0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e16c0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e16c0) get_ref 0x1000~1000 Blob(0x558c585e16c0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e16c0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2f000~1000: 0x1000~1000 Blob(0x558c585e16c0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x30000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1730 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1730) get_ref 0x0~2000 Blob(0x558c585e1730 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1730) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c585e1730 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x32000~2000: 0x0~2000 Blob(0x558c579d6e00 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e17a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e17a0) get_ref 0x0~2000 Blob(0x558c585e17a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e17a0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c585e17a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x34000~2000: 0x0~2000 Blob(0x558c55278f50 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1810 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1810) get_ref 0x0~2000 Blob(0x558c585e1810 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1810) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c585e1810 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x36000~2000: 0x0~2000 Blob(0x558c575f6070 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1880 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1880) get_ref 0x0~2000 Blob(0x558c585e1880 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1880) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c585e1880 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x38000~2000: 0x0~2000 Blob(0x558c574d3030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e18f0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e18f0) get_ref 0x0~2000 Blob(0x558c585e18f0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e18f0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c585e18f0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c581d6b60 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1960 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586e1b00> 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1960) get_ref 0x0~2000 Blob(0x558c585e1960 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1960) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1b00 kv_submitted 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c585e1960 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e1b00 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c54cf82a0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e19d0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e1b00 finishing 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e19d0) get_ref 0x0~2000 Blob(0x558c585e19d0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e1b00 onodes 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e19d0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e1b00 done 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c585e19d0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e1b00 [] 2019-12-07T19:34:38.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c575f6000 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.616+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1a40 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1a40) get_ref 0x0~2000 Blob(0x558c585e1a40 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1a40) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c585e1a40 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x40000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1ab0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ab0) get_ref 0x0~2000 Blob(0x558c585e1ab0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c585e1ab0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x42000~2000: 0x0~2000 Blob(0x558c577f3ab0 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1b20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1b20) get_ref 0x0~2000 Blob(0x558c585e1b20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1b20) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c585e1b20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x44000~2000: 0x0~2000 Blob(0x558c574e0690 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1b90 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1b90) get_ref 0x0~2000 Blob(0x558c585e1b90 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1b90) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c585e1b90 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x46000~2000: 0x0~2000 Blob(0x558c575f6380 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1c00 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1c00) get_ref 0x0~2000 Blob(0x558c585e1c00 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1c00) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c585e1c00 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x48000~2000: 0x0~2000 Blob(0x558c57566e00 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1c70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1c70) get_ref 0x0~2000 Blob(0x558c585e1c70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1c70) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c585e1c70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c581d6d20 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1ce0 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ce0) get_ref 0x0~2000 Blob(0x558c585e1ce0 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c585e1ce0 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c573144d0 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1d50 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1d50) get_ref 0x0~2000 Blob(0x558c585e1d50 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1d50) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c585e1d50 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c54df8070 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1dc0 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1dc0) get_ref 0x0~2000 Blob(0x558c585e1dc0 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c585e1dc0 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x50000~2000: 0x0~2000 Blob(0x558c574d2770 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1e30 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1e30) get_ref 0x0~2000 Blob(0x558c585e1e30 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1e30) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c585e1e30 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x52000~2000: 0x0~2000 Blob(0x558c5829cf50 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1ea0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ea0) get_ref 0x0~2000 Blob(0x558c585e1ea0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c585e1ea0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x54000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1f10 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1f10) get_ref 0x0~2000 Blob(0x558c585e1f10 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1f10) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c585e1f10 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x56000~2000: 0x0~2000 Blob(0x558c575f7ce0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e1f80 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1f80) get_ref 0x0~2000 Blob(0x558c585e1f80 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1f80) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c585e1f80 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x58000~2000: 0x0~2000 Blob(0x558c574d39d0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0000 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0000) get_ref 0x0~2000 Blob(0x558c550b0000 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0000) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c550b0000 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c575f7c00 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0070 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0070) get_ref 0x0~2000 Blob(0x558c550b0070 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0070) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c550b0070 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c57314310 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b00e0 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b00e0) get_ref 0x0~2000 Blob(0x558c550b00e0 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b00e0) get_ref init 0x2000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c550b00e0 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x5e000~1000: 0x0~1000 Blob(0x558c54df8230 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0150 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0150) get_ref 0x0~1000 Blob(0x558c550b0150 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0150) get_ref init 0x1000, 1000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x5e000~1000: 0x0~1000 Blob(0x558c550b0150 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) dirty_range mark inline shard dirty 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# -> 2#1:6c205177:::smithi13140796-8:head# = 0 2019-12-07T19:34:38.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:38.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:38.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000061.00000000000000000074' <- 0000000061.00000000000000000074 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a2300 cost 5793 (1 ios * 4000 + 1793 bytes) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a2300 onodes 0x558c575ba280,0x558c579df680 shared_blobs 0x558c54df8000,0x558c54df8af0,0x558c54df8bd0,0x558c54df8cb0,0x558c54df8d90,0x558c54df8e70,0x558c54df8f50,0x558c54df9030,0x558c54df9110,0x558c54df91f0,0x558c54df9ab0,0x558c54df9c00,0x558c54df9ce0,0x558c54df9dc0,0x558c54e43ce0,0x558c54e56690,0x558c54e56bd0,0x558c54e56cb0,0x558c54e56d90,0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c573970a0,0x558c575bd3b0,0x558c577f3a40,0x558c577f3c70,0x558c57e63490,0x558c57efd5e0,0x558c57efd6c0,0x558c57efd7a0,0x558c57efd880,0x558c57efd960,0x558c57efda40,0x558c57efdb20,0x558c57efdc00,0x558c5809b490,0x558c5809b500,0x558c5809b570,0x558c5809bc70,0x558c5809bd50,0x558c5809bdc0,0x558c5809bf80,0x558c580a4a10 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update 2#1:6c205177:::smithi13140796-8:head#4a 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update shard 0x0 is 487 bytes (was 487) from 18 extents 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update shard 0x23000 is 505 bytes (was 505) from 18 extents 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) update shard 0x46000 is 353 bytes (was 353) from 13 extents 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head#4a is 602 (600 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update inline shard 1335 bytes from 49 extents 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore.extentmap(0x558c579df788) reshard 0x[0,ffffffff) of 0 shards on 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard new shard 0x0 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard new shard 0x23000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard new shard 0x46000 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard old [] 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update 2#1:6c205177:::smithi13140796-8:head# force 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update shard 0x23000 is 505 bytes (was 0) from 18 extents 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update shard 0x46000 is 353 bytes (was 0) from 13 extents 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 602 (600 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aba is 11 SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abd is 11 SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abe is 11 SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abf is 11 SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac0 is 11 SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac1 is 11 SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac2 is 11 SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac3 is 11 SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac4 is 11 SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac5 is 11 SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acd is 11 SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ace is 11 SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acf is 11 SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad0 is 11 SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab8 is 11 SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab4 is 11 SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab7 is 11 SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab6 is 11 SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab5 is 11 SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac7 is 11 SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac6 is 11 SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab9 is 11 SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab2 is 11 SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaa is 11 SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aab is 11 SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aac is 11 SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aad is 11 SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aae is 11 SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaf is 11 SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab0 is 11 SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab1 is 11 SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abc is 11 SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abb is 11 SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acc is 11 SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aca is 11 SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac8 is 11 SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac9 is 11 SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acb is 11 SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab3 is 11 SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3))) 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a2300 allocated 0x[] released 0x[] 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2300 prepare 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a2300 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2300 io_done 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'74 (0'0,61'74] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'73 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] op_applied version 61'74 2019-12-07T19:34:38.616+0000 7f8f0197c700 10 osd.7 61 dequeue_op 0x558c57335080 finish 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=159, reqid=client.4540.0:455, at_version=61'75, trim_to=0'0, roll_forward_to=57'71)) v2) prio 127 cost 0 e61) pg 0x558c54d5c000 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 61 dequeue_op 0x558c57335ce0 prio 127 cost 0 latency 0.001755 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=159, reqid=client.4540.0:455, at_version=61'75, trim_to=0'0, roll_forward_to=57'71)) v2 pg pg[1.6s2( v 61'74 (0'0,61'74] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'74 (0'0,61'74] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=159, reqid=client.4540.0:455, at_version=61'75, trim_to=0'0, roll_forward_to=57'71)) v2 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'74 (0'0,61'74] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log log((0'0,61'74], crt=57'71) [61'75 (61'74) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:455 2019-12-07T19:34:38.611669+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0] 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'74 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] add_log_entry 61'75 (61'74) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:455 2019-12-07T19:34:38.611669+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'74 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log approx pg log length = 75 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'74 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:38.616+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:38.616+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 61'75, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4649 < min_flush_size 524288 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6412 < min_flush_size 524288 2019-12-07T19:34:38.616+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c586e0600 seq 114 2019-12-07T19:34:38.616+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:38.616+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:38.616+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:38.616+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:38.616+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000061.00000000000000000075' <- 0000000061.00000000000000000075 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:38.616+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:38.616+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586e0600 cost 6019 (1 ios * 4000 + 2019 bytes) 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586e0600 onodes 0x558c579df680 shared_blobs 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579df680 had 1 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 602 (600 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6435 < min_flush_size 524288 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6435 < min_flush_size 524288 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586e0600 allocated 0x[] released 0x[] 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0600 prepare 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d6dd7~1923 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586e0600 2019-12-07T19:34:38.616+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0600 io_done 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d6dd7 0x4d6dd7~1923 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a6dd7 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xdd7 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6fa and padding block with 0x906 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'74 crt=57'71 lcod 57'71 active mbc={} ps=[23~1]] op_applied version 61'75 2019-12-07T19:34:38.616+0000 7f8f05984700 10 osd.7 61 dequeue_op 0x558c57335ce0 finish 2019-12-07T19:34:38.616+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 61/28 ECSubWriteReply(tid=157, last_complete=61'73, committed=1, applied=1)) v2 -- 0x558c57519400 con 0x558c54c5bc00 2019-12-07T19:34:38.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d86fa 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000187443s (0.000000099s flush + 0.000187344s kv commit) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1746 < min_flush_size 524288 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579df680 had 1 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1769 < min_flush_size 524288 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1769 < min_flush_size 524288 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d86fa~6e9 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d86fa 0x4d86fa~6e9 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a86fa 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6fa 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xde3 and padding block with 0x21d 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c550a2300> 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2300 kv_submitted 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a2300 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2300 finishing 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a2300 onodes 0x558c575ba280,0x558c579df680 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a2300 done 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0600 kv_submitted 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a2300 [] 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.616+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 61/28 ECSubWriteReply(tid=158, last_complete=61'74, committed=1, applied=1)) v2 -- 0x558c57519600 con 0x558c54c5bc00 2019-12-07T19:34:38.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d8de3 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119485s (0.000000098s flush + 0.000119387s kv commit) 2019-12-07T19:34:38.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c586e0600> 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0600 kv_submitted 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586e0600 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586e0600 finishing 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586e0600 onodes 0x558c579df680 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586e0600 done 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586e0600 [] 2019-12-07T19:34:38.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.616+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.616+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 61/28 ECSubWriteReply(tid=159, last_complete=61'75, committed=1, applied=1)) v2 -- 0x558c57951800 con 0x558c54c5bc00 2019-12-07T19:34:38.616+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 307 ==== MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=160, reqid=client.4540.0:455, at_version=0'0, trim_to=0'0, roll_forward_to=61'75)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c58230000 con 0x558c54c5bc00 2019-12-07T19:34:38.618+0000 7f8f23c56700 15 osd.7 61 enqueue_op 0x558c581ccf20 prio 127 cost 0 latency 0.000019 epoch 61 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=160, reqid=client.4540.0:455, at_version=0'0, trim_to=0'0, roll_forward_to=61'75)) v2 2019-12-07T19:34:38.618+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=160, reqid=client.4540.0:455, at_version=0'0, trim_to=0'0, roll_forward_to=61'75)) v2) prio 127 cost 0 e61) 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=160, reqid=client.4540.0:455, at_version=0'0, trim_to=0'0, roll_forward_to=61'75)) v2) prio 127 cost 0 e61) queued 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=160, reqid=client.4540.0:455, at_version=0'0, trim_to=0'0, roll_forward_to=61'75)) v2) prio 127 cost 0 e61) pg 0x558c54d5c000 2019-12-07T19:34:38.618+0000 7f8f0197c700 10 osd.7 61 dequeue_op 0x558c581ccf20 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=160, reqid=client.4540.0:455, at_version=0'0, trim_to=0'0, roll_forward_to=61'75)) v2 pg pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[23~1]] 2019-12-07T19:34:38.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 61/28 ECSubWrite(tid=160, reqid=client.4540.0:455, at_version=0'0, trim_to=0'0, roll_forward_to=61'75)) v2 2019-12-07T19:34:38.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=57'71 active mbc={} ps=[23~1]] append_log log((0'0,61'75], crt=57'71) [] 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] rollforward: entry=61'72 (53'63) clone 1:6c205177:::smithi13140796-8:26 by unknown.0.0:0 2019-12-07T19:34:16.641390+0000 0 snaps [26,25,21,20,1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] rollforward: entry=61'73 (53'63) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:453 2019-12-07T19:34:38.611624+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] rollforward: entry=61'74 (61'73) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:454 2019-12-07T19:34:38.611663+0000 0 ObjectCleanRegions clean_offsets: [1556480~18446744073707995135], clean_omap: 0, new_object: 0 2019-12-07T19:34:38.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] rollforward: entry=61'75 (61'74) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:455 2019-12-07T19:34:38.611669+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:38.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] append_log approx pg log length = 75 2019-12-07T19:34:38.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:38.618+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:38.618+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c550a2600 seq 115 2019-12-07T19:34:38.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#4a onode 0x558c575ba280 txc 0x558c550a2600 2019-12-07T19:34:38.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#4a 0x0 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd260) put_ref 0x0~2000 Blob(0x558c575bd260 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c575bd420) put_ref 0x0~2000 Blob(0x558c575bd420 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57314230) put_ref 0x0~2000 Blob(0x558c57314230 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57314150) put_ref 0x0~2000 Blob(0x558c57314150 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57314070) put_ref 0x0~2000 Blob(0x558c57314070 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573145b0) put_ref 0x0~2000 Blob(0x558c573145b0 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57314700) put_ref 0x0~2000 Blob(0x558c57314700 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573147e0) put_ref 0x0~2000 Blob(0x558c573147e0 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573148c0) put_ref 0x0~2000 Blob(0x558c573148c0 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c573149a0) put_ref 0x0~1000 Blob(0x558c573149a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd570) put_ref 0x0~2000 Blob(0x558c57efd570 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd650) put_ref 0x0~2000 Blob(0x558c57efd650 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd730) put_ref 0x0~2000 Blob(0x558c57efd730 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd810) put_ref 0x0~2000 Blob(0x558c57efd810 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd8f0) put_ref 0x0~2000 Blob(0x558c57efd8f0 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd9d0) put_ref 0x0~2000 Blob(0x558c57efd9d0 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdab0) put_ref 0x0~2000 Blob(0x558c57efdab0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdb90) put_ref 0x0~2000 Blob(0x558c57efdb90 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efdc70) put_ref 0x0~2000 Blob(0x558c57efdc70 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c580a4930) put_ref 0x0~2000 Blob(0x558c580a4930 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54e57420) put_ref 0x0~2000 Blob(0x558c54e57420 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57a070a0) put_ref 0x0~2000 Blob(0x558c57a070a0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d810) put_ref 0x0~2000 Blob(0x558c5792d810 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5809b5e0) put_ref 0x0~2000 Blob(0x558c5809b5e0 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d8f0) put_ref 0x1000~1000 Blob(0x558c5792d8f0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd3b0) put_ref 0x0~2000 Blob(0x558c57efd3b0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3e30) put_ref 0x0~2000 Blob(0x558c577f3e30 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d730) put_ref 0x0~2000 Blob(0x558c5792d730 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d650) put_ref 0x0~2000 Blob(0x558c5792d650 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8a80) put_ref 0x0~2000 Blob(0x558c54df8a80 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8b60) put_ref 0x0~2000 Blob(0x558c54df8b60 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8c40) put_ref 0x0~2000 Blob(0x558c54df8c40 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8d20) put_ref 0x0~2000 Blob(0x558c54df8d20 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8e00) put_ref 0x0~2000 Blob(0x558c54df8e00 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8ee0) put_ref 0x0~2000 Blob(0x558c54df8ee0 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8fc0) put_ref 0x0~2000 Blob(0x558c54df8fc0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df90a0) put_ref 0x0~2000 Blob(0x558c54df90a0 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9180) put_ref 0x0~2000 Blob(0x558c54df9180 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9260) put_ref 0x0~2000 Blob(0x558c54df9260 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3b20) put_ref 0x0~2000 Blob(0x558c577f3b20 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bea0) put_ref 0x0~2000 Blob(0x558c5809bea0 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bce0) put_ref 0x0~2000 Blob(0x558c5809bce0 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5809be30) put_ref 0x0~2000 Blob(0x558c5809be30 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bc00) put_ref 0x0~2000 Blob(0x558c5809bc00 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df80e0) put_ref 0x0~2000 Blob(0x558c54df80e0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9a40) put_ref 0x0~2000 Blob(0x558c54df9a40 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9b90) put_ref 0x0~2000 Blob(0x558c54df9b90 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9c70) put_ref 0x0~2000 Blob(0x558c54df9c70 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9d50) put_ref 0x0~1000 Blob(0x558c54df9d50 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark shard 0x23000 dirty 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore.extentmap(0x558c575ba388) dirty_range mark shard 0x46000 dirty 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c575bd260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x26f000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c575bd420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x271000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c57314230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x273000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c57314150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x275000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c57314070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x277000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c573145b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x279000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c57314700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27b000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c573147e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27d000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c573148c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27f000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~1000: 0x0~1000 Blob(0x558c573149a0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x281000~1000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x13000~2000: 0x0~2000 Blob(0x558c57efd570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5a6000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x15000~2000: 0x0~2000 Blob(0x558c57efd650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5a8000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x17000~2000: 0x0~2000 Blob(0x558c57efd730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5aa000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x19000~2000: 0x0~2000 Blob(0x558c57efd810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ac000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1b000~2000: 0x0~2000 Blob(0x558c57efd8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ae000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1d000~2000: 0x0~2000 Blob(0x558c57efd9d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b0000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1f000~2000: 0x0~2000 Blob(0x558c57efdab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b2000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x21000~2000: 0x0~2000 Blob(0x558c57efdb90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b4000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x23000~2000: 0x0~2000 Blob(0x558c57efdc70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b6000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x25000~2000: 0x0~2000 Blob(0x558c580a4930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b8000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x27000~2000: 0x0~2000 Blob(0x558c54e57420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ba000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x29000~2000: 0x0~2000 Blob(0x558c57a070a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5bc000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2b000~2000: 0x0~2000 Blob(0x558c5792d810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5be000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2d000~2000: 0x0~2000 Blob(0x558c5809b5e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c0000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2f000~1000: 0x1000~1000 Blob(0x558c5792d8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c2000~1000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c57efd3b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c3000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c577f3e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c5000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c5792d730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c7000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~2000: 0x0~2000 Blob(0x558c5792d650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c9000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x38000~2000: 0x0~2000 Blob(0x558c54df8a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5cb000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3a000~2000: 0x0~2000 Blob(0x558c54df8b60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5cd000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3c000~2000: 0x0~2000 Blob(0x558c54df8c40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5cf000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3e000~2000: 0x0~2000 Blob(0x558c54df8d20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d1000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x40000~2000: 0x0~2000 Blob(0x558c54df8e00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d3000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x42000~2000: 0x0~2000 Blob(0x558c54df8ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d5000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x44000~2000: 0x0~2000 Blob(0x558c54df8fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d7000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x46000~2000: 0x0~2000 Blob(0x558c54df90a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d9000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x48000~2000: 0x0~2000 Blob(0x558c54df9180 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5db000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4a000~2000: 0x0~2000 Blob(0x558c54df9260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5dd000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~2000: 0x0~2000 Blob(0x558c577f3b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5df000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4e000~2000: 0x0~2000 Blob(0x558c5809bea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e1000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x50000~2000: 0x0~2000 Blob(0x558c5809bce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e3000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x52000~2000: 0x0~2000 Blob(0x558c5809be30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e5000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x54000~2000: 0x0~2000 Blob(0x558c5809bc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e7000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x56000~2000: 0x0~2000 Blob(0x558c54df80e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e9000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x58000~2000: 0x0~2000 Blob(0x558c54df9a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5eb000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5a000~2000: 0x0~2000 Blob(0x558c54df9b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ed000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5c000~2000: 0x0~2000 Blob(0x558c54df9c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ef000~2000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5e000~1000: 0x0~1000 Blob(0x558c54df9d50 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5f1000~1000] 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x23000 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x46000 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#4a = 0 2019-12-07T19:34:38.619+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a2600 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a2600 onodes shared_blobs 0x558c54df8000,0x558c54df8af0,0x558c54df8bd0,0x558c54df8cb0,0x558c54df8d90,0x558c54df8e70,0x558c54df8f50,0x558c54df9030,0x558c54df9110,0x558c54df91f0,0x558c54df9ab0,0x558c54df9c00,0x558c54df9ce0,0x558c54df9dc0,0x558c54e43ce0,0x558c54e56690,0x558c54e56bd0,0x558c54e56cb0,0x558c54e56d90,0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c573970a0,0x558c575bd3b0,0x558c577f3a40,0x558c577f3c70,0x558c57e63490,0x558c57efd5e0,0x558c57efd6c0,0x558c57efd7a0,0x558c57efd880,0x558c57efd960,0x558c57efda40,0x558c57efdb20,0x558c57efdc00,0x558c5809b490,0x558c5809b500,0x558c5809b570,0x558c5809bc70,0x558c5809bd50,0x558c5809bdc0,0x558c5809bf80,0x558c580a4a10 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aba is 11 SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abd is 11 SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abe is 11 SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abf is 11 SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac0 is 11 SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac1 is 11 SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac2 is 11 SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac3 is 11 SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac4 is 11 SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac5 is 11 SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acd is 11 SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ace is 11 SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acf is 11 SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad0 is 11 SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab8 is 11 SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab4 is 11 SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab7 is 11 SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab6 is 11 SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab5 is 11 SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac7 is 11 SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac6 is 11 SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab9 is 11 SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab2 is 11 SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaa is 11 SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aab is 11 SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aac is 11 SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aad is 11 SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aae is 11 SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaf is 11 SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab0 is 11 SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab1 is 11 SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abc is 11 SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abb is 11 SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acc is 11 SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aca is 11 SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac8 is 11 SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac9 is 11 SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acb is 11 SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab3 is 11 SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2))) 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a2600 allocated 0x[] released 0x[] 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2600 prepare 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a2600 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2600 io_done 2019-12-07T19:34:38.619+0000 7f8f0197c700 10 osd.7 61 dequeue_op 0x558c581ccf20 finish 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2570 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575ba280 had 1 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2593 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2593 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d8de3~a21 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d8de3 0x4d8de3~a21 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a8de3 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xde3 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x804 and padding block with 0x7fc 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d9804 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000155623s (0.000000119s flush + 0.000155504s kv commit) 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c550a2600> 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2600 kv_submitted 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a2600 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2600 finishing 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a2600 onodes 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a2600 done 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a2600 [] 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 61/28 ECSubWriteReply(tid=160, last_complete=61'75, committed=1, applied=1)) v2 -- 0x558c584fec00 con 0x558c54c5bc00 2019-12-07T19:34:38.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.619+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 213 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c5822b100 con 0x558c54c19800 2019-12-07T19:34:38.619+0000 7f8f24c58700 15 osd.7 61 enqueue_peering_evt 1.ds2 epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s) 2019-12-07T19:34:38.619+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e61) pg 0x558c55141000 2019-12-07T19:34:38.619+0000 7f8f06185700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.ds0( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s) 2019-12-07T19:34:38.619+0000 7f8f06185700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.619+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:38.619+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:38.619+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5853b200 seq 68 2019-12-07T19:34:38.619+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:38.619+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:38.619+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:38.619+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:38.619+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:38.619+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853b200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.619+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853b200 onodes shared_blobs 2019-12-07T19:34:38.619+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853b200 allocated 0x[] released 0x[] 2019-12-07T19:34:38.619+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b200 prepare 2019-12-07T19:34:38.619+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853b200 2019-12-07T19:34:38.619+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b200 io_done 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:38.619+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 214 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e61/61) v1 ==== 1060+0+0 (crc 0 0 0) 0x558c5822b100 con 0x558c54c19800 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d9804~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d9804 0x4d9804~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f24c58700 15 osd.7 61 enqueue_peering_evt 1.5s4 epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s) 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a9804 2019-12-07T19:34:38.619+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x804 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xc79 and padding block with 0x387 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e61) pg 0x558c57522000 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s) 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[e~1,15~1,1a~1,23~1]] proc_primary_info updating purged_snaps to [23~1] 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c58756600 seq 25 2019-12-07T19:34:38.619+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4d9c79 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58756600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58756600 onodes shared_blobs 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58756600 allocated 0x[] released 0x[] 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756600 prepare 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58756600 2019-12-07T19:34:38.619+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756600 io_done 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000121965s (0.000000088s flush + 0.000121877s kv commit) 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853b200> 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b200 kv_submitted 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853b200 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b200 finishing 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853b200 onodes 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853b200 done 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853b200 [] 2019-12-07T19:34:38.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.619+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:38.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:38.619+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4d9c79~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4d9c79 0x4d9c79~475 to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4a9c79 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xc79 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xee and padding block with 0xf12 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4da0ee 2019-12-07T19:34:38.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:38.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:38.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000124683s (0.000000064s flush + 0.000124619s kv commit) 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:38.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58756600> 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:38.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756600 kv_submitted 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58756600 2019-12-07T19:34:38.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756600 finishing 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58756600 onodes 2019-12-07T19:34:38.620+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58756600 done 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:38.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58756600 [] 2019-12-07T19:34:38.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:38.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:38.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:38.629+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4547 172.21.15.131:0/2305695222 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558c585f9e00 con 0x558c54c9f400 2019-12-07T19:34:38.629+0000 7f8f164bb700 20 osd.7 61 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:38.629+0000 7f8f164bb700 10 osd.7 61 do_waiters -- start 2019-12-07T19:34:38.629+0000 7f8f164bb700 10 osd.7 61 do_waiters -- finish 2019-12-07T19:34:38.629+0000 7f8f164bb700 20 osd.7 61 _dispatch 0x558c585f9e00 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:38.631+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/2305695222 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558c54ce0fc0 con 0x558c54c9f400 2019-12-07T19:34:38.633+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2305695222 conn(0x558c54c9f400 msgr2=0x558c54e34580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 80 2019-12-07T19:34:38.633+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2305695222 conn(0x558c54c9f400 msgr2=0x558c54e34580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:38.633+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2305695222 conn(0x558c54c9f400 0x558c54e34580 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:38.633+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/2305695222 conn(0x558c54c9f400 0x558c54e34580 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:38.633+0000 7f8f164bb700 2 osd.7 61 ms_handle_reset con 0x558c54c9f400 session 0x558c57bfa9c0 2019-12-07T19:34:38.649+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.656+0000 7f8f0998c700 15 osd.7 61 enqueue_peering_evt 1.cs0 epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:38.656+0000 7f8f0998c700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) queued 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) pg 0x558c57676000 2019-12-07T19:34:38.656+0000 7f8f06986700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.4 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.1 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.2 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.3 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until readable_until[_ub] 70.092849942s (sent 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.cs3 pg_lease(ru 70.092849942s ub 78.093078553s int 16s) e61/61) v1 -- 0x558c57613600 con 0x558c54c64000 2019-12-07T19:34:38.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease(1.cs4 pg_lease(ru 70.092849942s ub 78.093078553s int 16s) e61/61) v1 -- 0x558c5763c600 con 0x558c54c19800 2019-12-07T19:34:38.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.cs5 pg_lease(ru 70.092849942s ub 78.093078553s int 16s) e61/61) v1 -- 0x558c57519a00 con 0x558c54c5bc00 2019-12-07T19:34:38.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.cs2 pg_lease(ru 70.092849942s ub 78.093078553s int 16s) e61/61) v1 -- 0x558c57518000 con 0x558c54c5b400 2019-12-07T19:34:38.656+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.cs1 pg_lease(ru 70.092849942s ub 78.093078553s int 16s) e61/61) v1 -- 0x558c5841f600 con 0x558c54c19c00 2019-12-07T19:34:38.656+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:38.656+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=STANDBY pgs=11 cs=3 l=0 rx=0 tx=0).connect 2019-12-07T19:34:38.656+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.656+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 unknown :-1 s=BANNER_CONNECTING pgs=11 cs=3 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:38.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 325 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 78.093078553s) e61/61) v1 ==== 42+0+0 (unknown 0 0 0) 0x558c57edfc20 con 0x558c54c5b400 2019-12-07T19:34:38.656+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.cs0 epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.4 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 526 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 78.093078553s) e61/61) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54df7c20 con 0x558c54c64000 2019-12-07T19:34:38.656+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 215 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 78.093078553s) e61/61) v1 ==== 42+0+0 (crc 0 0 0) 0x558c57edef00 con 0x558c54c19800 2019-12-07T19:34:38.656+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.4 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.656+0000 7f8f24457700 15 osd.7 61 enqueue_peering_evt 1.cs0 epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.1 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f24c58700 15 osd.7 61 enqueue_peering_evt 1.cs0 epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.2 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.2 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.656+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.1 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.656+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 308 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 78.093078553s) e61/61) v1 ==== 42+0+0 (crc 0 0 0) 0x558c57edfc20 con 0x558c54c5bc00 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.4 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.656+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.cs0 epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.3 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.3 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.4 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) pg 0x558c57676000 2019-12-07T19:34:38.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.4 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.4 ruub 78.093078553s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.1 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.2 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.3 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until readable_until[_ub] 70.092849942s (sent 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:38.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.2 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.2 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) pg 0x558c57676000 2019-12-07T19:34:38.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.2 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.4 ruub 78.093078553s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.1 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.2 ruub 78.093078553s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.3 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until readable_until[_ub] 70.092849942s (sent 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f1199c700 10 osd.7 61 ms_handle_connect con 0x558c54c19c00 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:38.656+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.656+0000 7f8f24457700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558c54c19c00 0x558c54c38680 crc :-1 s=READY pgs=11 cs=3 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=e020c6588efc571e server_cookie=4bffdf0ee71c7be4 in_seq=120 out_seq=188 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.1 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.1 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) pg 0x558c57676000 2019-12-07T19:34:38.656+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.1 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.4 ruub 78.093078553s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.1 ruub 78.093078553s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.2 ruub 78.093078553s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.3 ruub 70.092849942s 2019-12-07T19:34:38.656+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until readable_until[_ub] 70.092849942s (sent 78.093078553s) 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:38.657+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.3 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.3 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) pg 0x558c57676000 2019-12-07T19:34:38.657+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.3 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.5 ruub 70.092849942s 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.4 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.1 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.2 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.3 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until readable_until[_ub] 70.092849942s (sent 78.093078553s) 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:38.657+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:38.657+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 121 ==== pg_lease_ack(1.cs0 pg_lease_ack(ruub 78.093078553s) e61/61) v1 ==== 42+0+0 (crc 0 0 0) 0x558c54df7c20 con 0x558c54c19c00 2019-12-07T19:34:38.657+0000 7f8f24457700 15 osd.7 61 enqueue_peering_evt 1.cs0 epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.5 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.657+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.5 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.5 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.5 pg_lease_ack(ruub 78.093078553s)) prio 255 cost 10 e61) pg 0x558c57676000 2019-12-07T19:34:38.657+0000 7f8f06986700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MLeaseAck epoch 61 from osd.5 pg_lease_ack(ruub 78.093078553s) 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.5 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.4 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.1 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.2 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until peer osd.3 ruub 78.093078553s 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recalc_readable_until readable_until[_ub] 78.093078553s (sent 78.093078553s) 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:38.657+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:38.657+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:38.658+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 309 ==== pg_lease(1.6s2 pg_lease(ru 70.116514081s ub 78.116716330s int 16s) e61/61) v1 ==== 58+0+0 (crc 0 0 0) 0x558c584fec00 con 0x558c54c5bc00 2019-12-07T19:34:38.658+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.6s2 epoch_sent: 61 epoch_requested: 61 MLease epoch 61 from osd.3 pg_lease(ru 70.116514081s ub 78.116716330s int 16s) 2019-12-07T19:34:38.658+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLease epoch 61 from osd.3 pg_lease(ru 70.116514081s ub 78.116716330s int 16s)) prio 255 cost 10 e61) 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLease epoch 61 from osd.3 pg_lease(ru 70.116514081s ub 78.116716330s int 16s)) prio 255 cost 10 e61) queued 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 61 epoch_requested: 61 MLease epoch 61 from osd.3 pg_lease(ru 70.116514081s ub 78.116716330s int 16s)) prio 255 cost 10 e61) pg 0x558c54d5c000 2019-12-07T19:34:38.658+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 61 epoch_requested: 61 MLease epoch 61 from osd.3 pg_lease(ru 70.116514081s ub 78.116716330s int 16s) 2019-12-07T19:34:38.658+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] proc_lease pg_lease(ru 70.116514081s ub 78.116716330s int 16s) 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] peer_clock_delta_ub 0.022363612s -> ru 70.094150469s 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] proc_lease readable_until now 70.094150469s 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] peer_clock_delta_lb 0.021996890s -> ruub 78.094719440s 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] proc_lease readable_until_ub now 78.094719440s 2019-12-07T19:34:38.658+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.6s0 pg_lease_ack(ruub 78.116716330s) e61/61) v1 -- 0x558c57f10b40 con 0x558c54c5bc00 2019-12-07T19:34:38.658+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:38.658+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:38.699+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.749+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.799+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 10 osd.7 61 tick_without_osd_lock 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f432000/0x40000000/0x1680000000, data 0xe01a0d/0xbcc000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 25 osd.7 61 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476453+0000 last_rx_front 2019-12-07T19:34:38.476418+0000 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 25 osd.7 61 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476352+0000 last_rx_front 2019-12-07T19:34:38.476394+0000 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 25 osd.7 61 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476378+0000 last_rx_front 2019-12-07T19:34:38.476470+0000 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 25 osd.7 61 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476367+0000 last_rx_front 2019-12-07T19:34:38.476505+0000 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 25 osd.7 61 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476409+0000 last_rx_front 2019-12-07T19:34:38.476432+0000 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 20 osd.7 61 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 10 osd.7 61 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 20 osd.7 61 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:38.805+0000 7f8f1fbbe700 10 osd.7 61 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:38.850+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.855+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 19 ==== osd_ping(ping e61 up_from 13 ping_stamp 2019-12-07T19:34:38.855965+0000/62.313477600s send_stamp 62.313477600s delta_ub 0.022363612s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c580b8c00 con 0x558c552d1400 2019-12-07T19:34:38.855+0000 7f8f24457700 20 osd.7 61 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021846770s,0.022363612s]) 2019-12-07T19:34:38.855+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 19 ==== osd_ping(ping e61 up_from 13 ping_stamp 2019-12-07T19:34:38.855965+0000/62.313477600s send_stamp 62.313477600s delta_ub 0.022363612s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c584fec00 con 0x558c54c18000 2019-12-07T19:34:38.855+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e61 up_from 13 ping_stamp 2019-12-07T19:34:38.855965+0000/62.313477600s send_stamp 62.291630830s delta_ub 18446744073.687704846s) v5 -- 0x558c57613600 con 0x558c552d1400 2019-12-07T19:34:38.855+0000 7f8f23c56700 20 osd.7 61 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021777771s,0.022363612s]) 2019-12-07T19:34:38.855+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e61 up_from 13 ping_stamp 2019-12-07T19:34:38.855965+0000/62.313477600s send_stamp 62.291699829s delta_ub 18446744073.687773845s) v5 -- 0x558c57518000 con 0x558c54c18000 2019-12-07T19:34:38.900+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:38.950+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.000+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.051+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.101+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.151+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.186+0000 7f8f21451700 10 osd.7 61 tick 2019-12-07T19:34:39.186+0000 7f8f21451700 10 osd.7 61 do_waiters -- start 2019-12-07T19:34:39.186+0000 7f8f21451700 10 osd.7 61 do_waiters -- finish 2019-12-07T19:34:39.186+0000 7f8f21451700 20 osd.7 61 tick last_purged_snaps_scrub 2019-12-07T19:34:38.205126+0000 next 2019-12-07T19:35:38.205126+0000 2019-12-07T19:34:39.200+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.215+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:39.251+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.301+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.350+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.355+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.3 172.21.15.131:0/37076 20 ==== osd_ping(ping e61 up_from 13 ping_stamp 2019-12-07T19:34:39.356253+0000/62.813765651s send_stamp 62.813765651s delta_ub 0.022300776s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5841f600 con 0x558c552d1400 2019-12-07T19:34:39.355+0000 7f8f24457700 20 osd.7 61 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021867428s,0.022300776s]) 2019-12-07T19:34:39.355+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.3 172.21.15.131:0/37076 20 ==== osd_ping(ping e61 up_from 13 ping_stamp 2019-12-07T19:34:39.356253+0000/62.813765651s send_stamp 62.813765651s delta_ub 0.022300776s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57519a00 con 0x558c54c18000 2019-12-07T19:34:39.355+0000 7f8f24457700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e61 up_from 13 ping_stamp 2019-12-07T19:34:39.356253+0000/62.813765651s send_stamp 62.791898223s delta_ub 18446744073.687684188s) v5 -- 0x558c57613600 con 0x558c552d1400 2019-12-07T19:34:39.355+0000 7f8f23c56700 20 osd.7 61 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.021821695s,0.022300776s]) 2019-12-07T19:34:39.355+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e61 up_from 13 ping_stamp 2019-12-07T19:34:39.356253+0000/62.813765651s send_stamp 62.791943956s delta_ub 18446744073.687729921s) v5 -- 0x558c57518000 con 0x558c54c18000 2019-12-07T19:34:39.363+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e61 up_from 13 ping_stamp 2019-12-07T19:34:39.364387+0000/62.813613899s send_stamp 62.813613899s delta_ub 0.014133497s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c584fa000 con 0x558c58147400 2019-12-07T19:34:39.363+0000 7f8f24c58700 20 osd.7 61 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013528911s,0.014133497s]) 2019-12-07T19:34:39.363+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e61 up_from 13 ping_stamp 2019-12-07T19:34:39.364387+0000/62.813613899s send_stamp 62.800084988s delta_ub 18446744073.696022705s) v5 -- 0x558c584fe800 con 0x558c58147400 2019-12-07T19:34:39.363+0000 7f8f24c58700 10 osd.7 61 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 60 -> 61 (as per caller) 2019-12-07T19:34:39.363+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e61 up_from 13 ping_stamp 2019-12-07T19:34:39.364387+0000/62.813613899s send_stamp 62.813613899s delta_ub 0.014133497s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5763c600 con 0x558c548d3000 2019-12-07T19:34:39.363+0000 7f8f24c58700 20 osd.7 61 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [0.013459428s,0.014133497s]) 2019-12-07T19:34:39.363+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e61 up_from 13 ping_stamp 2019-12-07T19:34:39.364387+0000/62.813613899s send_stamp 62.800154471s delta_ub 18446744073.696092188s) v5 -- 0x558c584fe800 con 0x558c548d3000 2019-12-07T19:34:39.401+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.451+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.502+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.552+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.602+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1523259 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.611+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 310 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c5822b800 con 0x558c54c5bc00 2019-12-07T19:34:39.611+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.fs5 epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s) 2019-12-07T19:34:39.611+0000 7f8f23c56700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.611+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 311 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c5822b800 con 0x558c54c5bc00 2019-12-07T19:34:39.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e62) pg 0x558c57383000 2019-12-07T19:34:39.611+0000 7f8f05183700 20 osd.7 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 62, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.611+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.611+0000 7f8f23c56700 15 osd.7 61 enqueue_peering_evt 1.1s1 epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s) 2019-12-07T19:34:39.611+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e62) pg 0x558c5736a000 2019-12-07T19:34:39.611+0000 7f8f04181700 20 osd.7 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 62, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.611+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.611+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 527 ==== osd_map(61..62 src has 1..62) v4 ==== 1568+0+0 (crc 0 0 0) 0x558c5852c240 con 0x558c54c64000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 osd.7 61 OSD::ms_dispatch: osd_map(61..62 src has 1..62) v4 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 osd.7 61 do_waiters -- start 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 osd.7 61 do_waiters -- finish 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 osd.7 61 _dispatch 0x558c5852c240 osd_map(61..62 src has 1..62) v4 2019-12-07T19:34:39.611+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 528 ==== MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=303, reqid=osd.1.0:303, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 ==== 3529+0+0 (crc 0 0 0) 0x558c58231000 con 0x558c54c64000 2019-12-07T19:34:39.611+0000 7f8f24457700 15 osd.7 61 enqueue_op 0x558c57eff8c0 prio 127 cost 0 latency 0.000025 epoch 62 MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=303, reqid=osd.1.0:303, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:39.611+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=303, reqid=osd.1.0:303, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.611+0000 7f8f1199c700 3 osd.7 61 handle_osd_map epochs [61,62], i have 61, src has [1,62] 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 osd.7 61 handle_osd_map got inc map for epoch 62 2019-12-07T19:34:39.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=303, reqid=osd.1.0:303, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.611+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=303, reqid=osd.1.0:303, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e62) pg 0x558c5500a000 2019-12-07T19:34:39.611+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3904e826:::osdmap.61:0# 0x0~0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580f71f0 blob([0xbcc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580f7260 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 61 dequeue_op 0x558c57eff8c0 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=303, reqid=osd.1.0:303, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 pg pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580f72d0 blob([!~1000,0xbcd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558c580f7340 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 61 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 60 -> 62 (as per caller) 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3904e826:::osdmap.61:0# 0x0~1464 = 5220 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 osd.7 61 add_map_bl 61 5220 bytes 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=303, reqid=osd.1.0:303, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:39.611+0000 7f8f08189700 20 snap_mapper.update_snaps 1:9b8adabe:::smithi13140796-6:20 1b was 2019-12-07T19:34:39.611+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.611+0000 7f8f08189700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.611+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.611+0000 7f8f08189700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:20 1b,20 2019-12-07T19:34:39.611+0000 7f8f08189700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:20 1b 2019-12-07T19:34:39.611+0000 7f8f08189700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:39.611+0000 7f8f08189700 20 snap_mapper.update_snaps rm SNA_1_0000000000000020_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] append_log log((0'0,59'48], crt=59'48) [62'49 (57'43) modify 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:39.612114+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'50 (59'48) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:39.612114+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 62'49 (0'0,62'49] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] add_log_entry 62'49 (57'43) modify 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:39.612114+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 osd.7 61 got_full_map 62, nothing requested 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] add_log_entry 62'50 (59'48) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:39.612114+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] append_log approx pg log length = 50 2019-12-07T19:34:39.611+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.611+0000 7f8f08189700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.611+0000 7f8f08189700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'49, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 osd.7 61 handle_osd_map pg_num_history pg_num_history(e62 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {62={}} 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 osd.7 61 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e62 [1,62] lci=[13,62]) 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c578aa600 seq 54 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cadf972f:::inc_osdmap.62:0# key 0x7f7fffffffffffffffcadf97'/!inc_osdmap.62!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:39.611+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:39.611+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c584fcc00 seq 92 2019-12-07T19:34:39.611+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# 1 keys 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:39.611+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# 1 keys = 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cadf972f:::inc_osdmap.62:0# 0x0~269 2019-12-07T19:34:39.611+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# hinfo_key (42 bytes) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1288 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cadf972f:::inc_osdmap.62:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:39.611+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:20# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~269 2019-12-07T19:34:39.611+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aa600 1 blobs 2019-12-07T19:34:39.611+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:39.611+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbce000~1000/1000,1000,0 2019-12-07T19:34:39.611+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.9s4_head 4#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbce000~1000] 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579d75e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829cb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579d75e0 blob([0xbce000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829cb60 sbid 0x0)) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore.blob(0x558c579d75e0) get_ref 0x0~269 Blob(0x558c579d75e0 blob([0xbce000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829cb60 sbid 0x0)) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore.blob(0x558c579d75e0) get_ref init 0x1000, 1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558c579d75e0 blob([0xbce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5829cb60 sbid 0x0)) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5829cb78 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x269 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f788) dirty_range mark inline shard dirty 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cadf972f:::inc_osdmap.62:0# 0x0~269 = 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3404e826:::osdmap.62:0# key 0x7f7fffffffffffffff3404e8'&!osdmap.62!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:39.611+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3404e826:::osdmap.62:0# 0x0~1454 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1289 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3404e826:::osdmap.62:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~454 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aa600 2 blobs 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:39.611+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbcf000~2000/1000,2000,0 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbcf000~2000] 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58226000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a07260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58226000 blob([0xbcf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07260 sbid 0x0)) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore.blob(0x558c58226000) get_ref 0x0~1000 Blob(0x558c58226000 blob([0xbcf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07260 sbid 0x0)) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore.blob(0x558c58226000) get_ref init 0x1000, 1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c58226000 blob([0xbcf000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57a07260 sbid 0x0)) 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57a07278 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a07490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a076c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.611+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a07490 blob([!~1000,0xbd0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a076c0 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.blob(0x558c57a07490) get_ref 0x1000~454 Blob(0x558c57a07490 blob([!~1000,0xbd0000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57a076c0 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.blob(0x558c57a07490) get_ref init 0x2000, 1000 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558c57a07490 blob([!~1000,0xbd0000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c57a076c0 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57a076d8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1454 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851fa08) dirty_range mark inline shard dirty 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3404e826:::osdmap.62:0# 0x0~1454 = 0 2019-12-07T19:34:39.612+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aa600 0 blobs 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:39.612+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:39.612+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:39.612+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c578aa600 0 blobs 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c578aa600 cost 23302 (4 ios * 4000 + 7302 bytes) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c578aa600 onodes 0x558c53a91900,0x558c55276c80,0x558c5851f680,0x558c5851f900 shared_blobs 2019-12-07T19:34:39.612+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- OBJ_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:39.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:39.612+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.4_0000000000000001.9D15B5D7.20.smithi13140796-6..' <- SNA_1_0000000000000020_.4_0000000000000001.9D15B5D7.20.smithi13140796-6.. 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f788) update #-1:cadf972f:::inc_osdmap.62:0# 2019-12-07T19:34:39.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851f788) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cadf972f:::inc_osdmap.62:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851fa08) update #-1:3404e826:::osdmap.62:0# 2019-12-07T19:34:39.612+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore.extentmap(0x558c5851fa08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3404e826:::osdmap.62:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000062.00000000000000000049' <- 0000000062.00000000000000000049 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.0000000062.00000000000000000050' <- 0000000062.00000000000000000050 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c578aa600 allocated 0x[bce000~3000] released 0x[] 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 freelist allocate 0xbce000~3000 2019-12-07T19:34:39.612+0000 7f8f1199c700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:39.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584fcc00 cost 8482 (1 ios * 4000 + 4482 bytes) 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584fcc00 onodes 0x558c57573900,0x558c57a35400 shared_blobs 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore.extentmap(0x558c57573a08) update 4#1:9b8adabe:::smithi13140796-6:20# 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa600 prepare 2019-12-07T19:34:39.612+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c578aa600 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:20# is 124356 (123297 bytes onode + 2 bytes spanning blobs + 1057 bytes inline extents) 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore.extentmap(0x558c57a35508) update 4#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:39.612+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa600 aio_wait 2019-12-07T19:34:39.612+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c578aa600 2019-12-07T19:34:39.612+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa600 io_done 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:9b8adabe:::smithi13140796-6:head# is 210619 (210617 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851f680 had 1 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5851f900 had 1 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3855745024 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997474) bluestore 86 GiB free (0.959871), bluefs_ratio 0.0399085 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4da0ee~226c to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4da0ee 0x4da0ee~226c to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~4b0000 x_off 0x4aa0ee 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xee 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x35a and padding block with 0xca6 2019-12-07T19:34:39.612+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 313 ==== MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=161, reqid=osd.3.0:161, at_version=62'77, trim_to=0'0, roll_forward_to=61'75)) v2 ==== 3481+0+0 (crc 0 0 0) 0x558c575c0800 con 0x558c54c5bc00 2019-12-07T19:34:39.612+0000 7f8f23c56700 15 osd.7 61 enqueue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.000031 epoch 62 MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=161, reqid=osd.3.0:161, at_version=62'77, trim_to=0'0, roll_forward_to=61'75)) v2 2019-12-07T19:34:39.612+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=161, reqid=osd.3.0:161, at_version=62'77, trim_to=0'0, roll_forward_to=61'75)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x4dc35a 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.612+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=161, reqid=osd.3.0:161, at_version=62'77, trim_to=0'0, roll_forward_to=61'75)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=161, reqid=osd.3.0:161, at_version=62'77, trim_to=0'0, roll_forward_to=61'75)) v2) prio 127 cost 0 e62) pg 0x558c54d5c000 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000206407s (0.000046690s flush + 0.000159717s kv commit) 2019-12-07T19:34:39.612+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.612+0000 7f8f0197c700 10 osd.7 61 dequeue_op 0x558c581ccdc0 prio 127 cost 0 latency 0.000088 MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=161, reqid=osd.3.0:161, at_version=62'77, trim_to=0'0, roll_forward_to=61'75)) v2 pg pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.612+0000 7f8f0197c700 10 osd.7 61 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 61 -> 62 (as per caller) 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584fcc00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcc00 prepare 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.612+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584fcc00 2019-12-07T19:34:39.612+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=161, reqid=osd.3.0:161, at_version=62'77, trim_to=0'0, roll_forward_to=61'75)) v2 2019-12-07T19:34:39.612+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcc00 io_done 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c578aa600> 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.612+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa600 kv_submitted 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c578aa600 2019-12-07T19:34:39.612+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:6c205177:::smithi13140796-8:26 1b,21,25,26 was 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c578aa600 osr 0x558c54ea4b40 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 51 0x1b000~1000 2019-12-07T19:34:39.612+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.612+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 52 0x1b000~1000 crc ada932ca 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) _discard drop 51 0x2000~1000 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c58141400) prepare_write seq 52 0x2000~1000 crc 72d8b7b6 2019-12-07T19:34:39.612+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.612+0000 7f8f08189700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] op_applied version 62'50 2019-12-07T19:34:39.612+0000 7f8f08189700 10 osd.7 61 dequeue_op 0x558c57eff8c0 finish 2019-12-07T19:34:39.612+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7 61 _committed_osd_maps 62..62 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7 61 advance to epoch 62 (<= last 62 <= newest_map 62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 7 osd.7 62 consume_map version 62 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.8s5 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.0s3 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.1s1 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.9s4 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.as5 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.2s0 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.bs1 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.3s4 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.cs0 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.4s0 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.5s4 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.ds2 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.es4 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.6s2 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.fs5 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 identify_splits_and_merges 1.7s4 e61 to e62 pg_nums {18=16} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 62 prune_pg_created 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:0.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:1.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 pending_peering first epoch 62 <= 62, requeueing 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {62=} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:2.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:3.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:4.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:5.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:6.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7:7.consume_map 62 (was 61) 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 pending_peering first epoch 62 <= 62, requeueing 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {62=} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:39.612+0000 7f8f141a1700 10 osd.7 62 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.8s5 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.0s3 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s)) prio 255 cost 10 e62) pg 0x558c5736a000 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.1s1 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.9s4 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.as5 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.2s0 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.bs1 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.3s4 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.cs0 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 61 -> 62 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.4s0 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.5s4 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.ds2 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.es4 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.6s2 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.fs5 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f141a1700 15 osd.7 62 enqueue_peering_evt 1.7s4 epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.612+0000 7f8f141a1700 7 osd.7 62 activate_map version 62 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c57676000 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c57e8b000 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 61 -> 62 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.612+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064655423s ub 75.064655423s int 16s) 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean ps=[23~1]] state: Active advmap 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 61 -> 62 2019-12-07T19:34:39.612+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] on_active_advmap new removed_snaps [20~1], snap_trimq now [20~1] 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 61 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:39.612+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c576dcf00 1.1s1_head 2019-12-07T19:34:39.612+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10140 = 0x558c58c65b00 seq 27 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.612+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.612+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._biginfo' <- _biginfo 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.612+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:39.612+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._epoch' <- _epoch 2019-12-07T19:34:39.612+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:39.612+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049b'._info' <- _info 2019-12-07T19:34:39.612+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.612+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58c65b00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.612+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58c65b00 onodes shared_blobs 2019-12-07T19:34:39.612+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58c65b00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.612+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 prepare 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] agent_stop 2019-12-07T19:34:39.612+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58c65b00 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.612+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 io_done 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] activate_map 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c5736a000 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] state: Active: handling ActMap 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.612+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] Active: kicking snap trim 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c5500a000 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c57e88000 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] exit NotTrimming 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] enter Trimming 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 61 -> 62 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] enter Trimming/WaitReservation 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 61 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 61 -> 62 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 61 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.612+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:39.612+0000 7f8f0297e700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] publish_stats_to_osd 61:209 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[23~1]] state: Active advmap 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.612+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.612+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] on_active_advmap new removed_snaps [20~1], snap_trimq now [20~1] 2019-12-07T19:34:39.612+0000 7f8f0297e700 7 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] state: handle ActMap primary 2019-12-07T19:34:39.612+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c57380000 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.612+0000 7f8f0297e700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] publish_stats_to_osd 61:210 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.612+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f04982700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.612+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 61 -> 62 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.612+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.612+0000 7f8f04982700 10 osd.7 pg_epoch: 61 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:39.612+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.612+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] agent_stop 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.612+0000 7f8f03980700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.612+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c548ab000 2019-12-07T19:34:39.612+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.612+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.612+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:39.612+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] state: Active: handling ActMap 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] Active: kicking snap trim 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 61 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c54d98000 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[23~1]] state: Active advmap 2019-12-07T19:34:39.613+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] exit NotTrimming 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] on_active_advmap new removed_snaps [20~1], snap_trimq now [20~1] 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] enter Trimming 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] enter Trimming/WaitReservation 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c5500d000 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s)) prio 255 cost 10 e62) pg 0x558c57383000 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f03980700 15 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] publish_stats_to_osd 61:175 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:39.613+0000 7f8f03980700 7 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] state: handle ActMap primary 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 61 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f03980700 15 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] publish_stats_to_osd 61:176 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] state: Active: handling ActMap 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f03980700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 61 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] Active: kicking snap trim 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] exit NotTrimming 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c575ef000 2019-12-07T19:34:39.613+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] enter Trimming 2019-12-07T19:34:39.613+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] enter Trimming/WaitReservation 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f0297e700 15 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] publish_stats_to_osd 61:163 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 61 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:39.613+0000 7f8f0297e700 7 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] state: handle ActMap primary 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f0297e700 15 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] publish_stats_to_osd 61:164 2019-12-07T19:34:39.613+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.fs0( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.064626080s ub 75.064626080s int 16s) 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f0b990700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0b990700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] SnapTrimmer state: NotTrimming: trimming 20 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f105a0 1.fs5_head 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] exit Trimming/WaitReservation 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c576aa640 = 0x558c5875d500 seq 34 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c5513e000 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._biginfo' <- _biginfo 2019-12-07T19:34:39.613+0000 7f8f0b990700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:211 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._epoch' <- _epoch 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049c'._info' <- _info 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875d500 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875d500 onodes shared_blobs 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875d500 allocated 0x[] released 0x[] 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f0b990700 10 osd.7 62 queueing pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean trimq=[20~1] ps=[23~1]] for snaptrim 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=62) prio 5 cost 1048576 e62) 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 61 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d500 prepare 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875d500 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d500 io_done 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c57383000 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:39.613+0000 7f8f0b990700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:212 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f0b990700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c54d5f000 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f0b990700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] SnapTrimmer state: NotTrimming: trimming 20 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] exit Trimming/WaitReservation 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0b990700 15 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:177 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 61 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f0b990700 10 osd.7 62 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean trimq=[20~1] ps=[23~1]] for snaptrim 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=62) prio 5 cost 1048576 e62) 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f0b990700 15 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:178 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=62) prio 5 cost 1048576 e62) queued 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=62) prio 5 cost 1048576 e62) pg 0x558c57676000 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] snap_trimmer posting 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 20 2019-12-07T19:34:39.613+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.613+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:39.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=62) prio 5 cost 1048576 e62) queued 2019-12-07T19:34:39.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=62) prio 5 cost 1048576 e62) pg 0x558c57e88000 2019-12-07T19:34:39.613+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[20~1] ps=[23~1]] snap_trimmer posting 2019-12-07T19:34:39.613+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 20 2019-12-07T19:34:39.613+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.613+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:39.613+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 312 ==== osd_map(62..62 src has 1..62) v4 ==== 661+0+0 (crc 0 0 0) 0x558c585e2240 con 0x558c54c5bc00 2019-12-07T19:34:39.613+0000 7f8f1199c700 20 osd.7 62 OSD::ms_dispatch: osd_map(62..62 src has 1..62) v4 2019-12-07T19:34:39.613+0000 7f8f1199c700 10 osd.7 62 do_waiters -- start 2019-12-07T19:34:39.613+0000 7f8f1199c700 10 osd.7 62 do_waiters -- finish 2019-12-07T19:34:39.613+0000 7f8f1199c700 20 osd.7 62 _dispatch 0x558c585e2240 osd_map(62..62 src has 1..62) v4 2019-12-07T19:34:39.613+0000 7f8f1199c700 3 osd.7 62 handle_osd_map epochs [62,62], i have 62, src has [1,62] 2019-12-07T19:34:39.613+0000 7f8f1199c700 10 osd.7 62 no new maps here, dropping 2019-12-07T19:34:39.613+0000 7f8f0918b700 20 osd.7 62 agent_entry empty queue 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c57522000 2019-12-07T19:34:39.613+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 15526 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 61 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 48294 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 81062 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 113830 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 146598 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c55141000 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 61 -> 62 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 61 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.613+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 179366 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.613+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 212134 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.613+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 244902 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 277670 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 310438 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 337961 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 529 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57392a00 con 0x558c54c64000 2019-12-07T19:34:39.613+0000 7f8f24457700 15 osd.7 62 enqueue_peering_evt 1.8s5 epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s) 2019-12-07T19:34:39.613+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s)) prio 255 cost 10 e62) pg 0x558c57380000 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.8s0( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068731979s ub 75.068731979s int 16s) 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58500300 seq 56 2019-12-07T19:34:39.613+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._biginfo' <- _biginfo 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58500300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58500300 onodes shared_blobs 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58500300 allocated 0x[] released 0x[] 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500300 prepare 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58500300 2019-12-07T19:34:39.613+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500300 io_done 2019-12-07T19:34:39.613+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57573900 had 1 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57a35400 had 1 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 339095 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57976a00 had 1 2019-12-07T19:34:39.613+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 340229 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:26 1b,20,21,25,26 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f180 had 1 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 340252 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 340252 < min_flush_size 524288 2019-12-07T19:34:39.613+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:6c205177:::smithi13140796-8:26 1b,21,25,26 2019-12-07T19:34:39.613+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x4dc35a~5311c to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x4dc35a 0x4dc35a~5311c to file(ino 16 size 0x4e0000 mtime 2019-12-07T19:34:38.611137+0000 allocated 4e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~4b0000]) 2019-12-07T19:34:39.613+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_0000000000000020_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _allocate len 0x4f476 from 0 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _allocate len 0x4f476 from 1 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x50000/10000,50000,acd610000 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd610000~50000/10000,50000,acd610000 2019-12-07T19:34:39.613+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x530000 to include allocated 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 77 (was clean) 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x530000 mtime 2019-12-07T19:34:39.615186+0000 allocated 530000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~500000]) 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~500000 x_off 0x4ac35a 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x35a 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.613+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x476 and padding block with 0xb8a 2019-12-07T19:34:39.613+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 61'75 (0'0,61'75] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] append_log log((0'0,61'75], crt=61'75) [62'76 (61'72) modify 1:6c205177:::smithi13140796-8:26 by unknown.0.0:0 2019-12-07T19:34:39.612394+0000 0 snaps [1b,21,25,26] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'77 (61'75) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:39.612394+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 62'76 (0'0,62'76] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'75 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] add_log_entry 62'76 (61'72) modify 1:6c205177:::smithi13140796-8:26 by unknown.0.0:0 2019-12-07T19:34:39.612394+0000 0 snaps [1b,21,25,26] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'75 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] add_log_entry 62'77 (61'75) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:39.612394+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'75 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] append_log approx pg log length = 77 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.614+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'75 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.614+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'76, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5730e7c0) upper_bound after SNA_1_0000000000000020_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C' 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5730e7c0) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57630940) upper_bound after SNA_1_0000000000000020_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.2' 2019-12-07T19:34:39.614+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000020_.0_0000000000000001.C) returns 0 SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239810 in raw 0x558c55239810 len 70 nref 1) ) 2019-12-07T19:34:39.614+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore.OmapIteratorImpl(0x558c57630940) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:39.614+0000 7f8f07988700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000020_.0_0000000000000001.2) returns 0 SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c552382c0 in raw 0x558c552382c0 len 70 nref 1) ) 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5730e7c0) upper_bound after SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. key 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore.OmapIteratorImpl(0x558c5730e7c0) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' 2019-12-07T19:34:39.614+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..) returns 0 SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2..,buffer::list(len=69, buffer::ptr(0~69 0x558c55239810 in raw 0x558c55239810 len 69 nref 1) ) 2019-12-07T19:34:39.614+0000 7f8f06986700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:39.614+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:39.614+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[23~1]] SnapTrimmer state: adding snap 20 to purged_snaps 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:39.614+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: found obc in cache: 0x558c585278c0 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: 0x558c585278c0 1:3a5a3f5b:::smithi13140796-23:23 rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:23(59'47 client.4540.0:286 dirty|data_digest s 507904 uv 8 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 23=[]:{23=[21,20,1b]} 2019-12-07T19:34:39.614+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[20~1,23~1]] SnapTrimmer state: purged_snaps now [20~1,23~1], snap_trimq now [] 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: found obc in cache: 0x558c57557b00 2019-12-07T19:34:39.614+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c550a2900 seq 116 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: 0x558c57557b00 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(59'48 client.4540.0:443 dirty|data_digest s 507904 uv 46 dd c13735ee alloc_hint [0 0 0]) exists: 1 ssc: 0x558c55159320 snapset: 23=[]:{23=[21,20,1b]} 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c587ce000 seq 33 2019-12-07T19:34:39.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# 1 keys 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:3a5a3f5b:::smithi13140796-23:23 old_snaps 1b,20,21 old snapset 23=[]:{23=[21,20,1b]} 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# 1 keys = 0 2019-12-07T19:34:39.614+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] simple_opc_create 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:39.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# hinfo_key (42 bytes) 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._biginfo' <- _biginfo 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:3a5a3f5b:::smithi13140796-23:23 snaps 1b,20,21 -> 1b,21 2019-12-07T19:34:39.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:39.614+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:39.614+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c587ce000 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c587ce000 onodes shared_blobs 2019-12-07T19:34:39.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c587ce000 allocated 0x[] released 0x[] 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:3a5a3f5b:::smithi13140796-23:23 new snapset 23=[]:{23=[21,1b]} on 1:3a5a3f5b:::smithi13140796-23:head(59'48 client.4540.0:443 dirty|data_digest s 507904 uv 46 dd c13735ee alloc_hint [0 0 0]) 2019-12-07T19:34:39.614+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce000 prepare 2019-12-07T19:34:39.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.614+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c587ce000 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:3a5a3f5b:::smithi13140796-23:23 writing updated snapset on 1:3a5a3f5b:::smithi13140796-23:head, snapset is 23=[]:{23=[21,1b]} 2019-12-07T19:34:39.614+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce000 io_done 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] new_repop rep_tid 59 (no op) 2019-12-07T19:34:39.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.614+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[20~1,23~1]] share_pg_info 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.2_0000000000000001.6340A8EE.26.smithi13140796-8..' <- SNA_1_0000000000000020_.2_0000000000000001.6340A8EE.26.smithi13140796-8.. 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] new_repop: repgather(0x558c576b6ea0 0'0 rep_tid=59 committed?=0 r=0) 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] simple_opc_submit 0x558c576b6ea0 2019-12-07T19:34:39.614+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:39.614+0000 7f8f06986700 7 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] issue_repop rep_tid 59 o 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000062.00000000000000000076' <- 0000000062.00000000000000000076 2019-12-07T19:34:39.614+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c57453100 con 0x558c54c64000 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000062.00000000000000000077' <- 0000000062.00000000000000000077 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:23 v=62'54 tt=0'0 tid=59 reqid=osd.7.0:59 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a2900 cost 8458 (1 ios * 4000 + 4458 bytes) 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:23 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a2900 onodes 0x558c579df680,0x558c57c05b80 shared_blobs 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 594 (592 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:39.614+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c58124300 con 0x558c54c19800 2019-12-07T19:34:39.614+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) update 2#1:6c205177:::smithi13140796-8:26# 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key = 0 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:26# is 425 (423 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a2900 allocated 0x[] released 0x[] 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:23 projected size 507904 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2900 prepare 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a2900 2019-12-07T19:34:39.614+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c5822aa00 con 0x558c54c5bc00 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2900 io_done 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: found on disk, size 126976 2019-12-07T19:34:39.614+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=61'75 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] op_applied version 62'77 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 507904 2019-12-07T19:34:39.614+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c58228000 con 0x558c54c5b400 2019-12-07T19:34:39.614+0000 7f8f0197c700 10 osd.7 62 dequeue_op 0x558c581ccdc0 finish 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:23 v=62'54 tt=0'0 tid=59 reqid=osd.7.0:59 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:39.614+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:23 v=62'54 tt=0'0 tid=59 reqid=osd.7.0:59 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:39.614+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c58229c00 con 0x558c54c19c00 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 NullEvt) prio 255 cost 10 e62) pg 0x558c54d5c000 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:23 v=62'54 tt=0'0 tid=59 reqid=osd.7.0:59 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:39.614+0000 7f8f07988700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[20~1,23~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:23, current size is 507904 buffers are {} 2019-12-07T19:34:39.614+0000 7f8f07988700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean+snaptrim ps=[20~1,23~1]] exit Trimming 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] handle_advance_map: 62 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 61 -> 62 2019-12-07T19:34:39.614+0000 7f8f07988700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): new_size start 507904 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 61 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:39.614+0000 7f8f07988700 15 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:179 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_append: {} 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:23 resetting hinfo to logical size 507904 2019-12-07T19:34:39.614+0000 7f8f07988700 20 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] enter NotTrimming 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] state: Started advmap 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 507904 buffers are {} 2019-12-07T19:34:39.614+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:39.614+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:39.614+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] plpg_on_pool_change 2019-12-07T19:34:39.614+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] snap_trimmer complete 2019-12-07T19:34:39.614+0000 7f8f0b990700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): new_size start 507904 2019-12-07T19:34:39.614+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] hit_set_clear 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:39.614+0000 7f8f0b990700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] SnapTrimmer state: NotTrimming: trimming 20 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_append: {} 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] agent_stop 2019-12-07T19:34:39.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim_wait trimq=[20~1] ps=[23~1]] exit Trimming/WaitReservation 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7 62 advance_pg new pool opts old pool opts 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 507904 2019-12-07T19:34:39.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:39.614+0000 7f8f0b990700 15 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:165 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] handle_activate_map: 62 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:23={},1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:34:39.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:39.614+0000 7f8f0b990700 10 osd.7 62 queueing pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean trimq=[20~1] ps=[23~1]] for snaptrim 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:23 v=62'54 tt=0'0 tid=59 reqid=osd.7.0:59 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:39.614+0000 7f8f0b990700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=62) prio 5 cost 1048576 e62) 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] activate_map 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] activate_map: Not dirtying info: last_persisted is 61 while current is 62 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:23,{} 2019-12-07T19:34:39.614+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:39.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.614+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:34:39.614+0000 7f8f0b990700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=62) prio 5 cost 1048576 e62) queued 2019-12-07T19:34:39.614+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 lcod 61'75 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 NullEvt 2019-12-07T19:34:39.614+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:39.614+0000 7f8f0b990700 15 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:166 2019-12-07T19:34:39.614+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.614+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGSnapTrim(pgid=1.4s0epoch_queued=62) prio 5 cost 1048576 e62) pg 0x558c548ab000 2019-12-07T19:34:39.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[20~1] ps=[23~1]] snap_trimmer posting 2019-12-07T19:34:39.614+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 20 2019-12-07T19:34:39.614+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.614+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 62/28 ECSubWrite(tid=59, reqid=osd.7.0:59, at_version=62'54, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c57a44800 con 0x558c54c64000 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 send_incremental_map 60 -> 62 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:39.614+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c1df972f:::inc_osdmap.61:0# 0x0~0 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~383 size 0x383 (899) 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580d7c00 blob([0xbcb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x383) SharedBlob(0x558c580f7c70 sbid 0x0)) need 0x0~383 cache has 0x[0~383] 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c1df972f:::inc_osdmap.61:0# 0x0~383 = 899 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 add_map_inc_bl 61 899 bytes 2019-12-07T19:34:39.614+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cadf972f:::inc_osdmap.62:0# 0x0~0 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579d75e0 blob([0xbce000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558c5829cb60 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:39.614+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cadf972f:::inc_osdmap.62:0# 0x0~269 = 617 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 add_map_inc_bl 62 617 bytes 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(61..62 src has 1..62) v4 -- 0x558c58234900 con 0x558c54c19800 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 60 -> 62 (shared) 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 62/28 ECSubWrite(tid=59, reqid=osd.7.0:59, at_version=62'54, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c57a42800 con 0x558c54c19800 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 62/28 ECSubWrite(tid=59, reqid=osd.7.0:59, at_version=62'54, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c57a44000 con 0x558c54c5bc00 2019-12-07T19:34:39.614+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 530 ==== pg_info2(1.as5 1.as5 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57392a00 con 0x558c54c64000 2019-12-07T19:34:39.614+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 326 ==== osd_map(62..62 src has 1..62) v4 ==== 661+0+0 (unknown 0 0 0) 0x558c5738e900 con 0x558c54c5b400 2019-12-07T19:34:39.614+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 327 ==== MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=120, reqid=osd.4.0:120, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 ==== 3426+0+0 (unknown 0 0 0) 0x558c5748d000 con 0x558c54c5b400 2019-12-07T19:34:39.614+0000 7f8f1199c700 20 osd.7 62 OSD::ms_dispatch: osd_map(62..62 src has 1..62) v4 2019-12-07T19:34:39.614+0000 7f8f1199c700 10 osd.7 62 do_waiters -- start 2019-12-07T19:34:39.614+0000 7f8f1199c700 10 osd.7 62 do_waiters -- finish 2019-12-07T19:34:39.614+0000 7f8f1199c700 20 osd.7 62 _dispatch 0x558c5738e900 osd_map(62..62 src has 1..62) v4 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 send_incremental_map 61 -> 62 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:39.614+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c581cdce0 prio 127 cost 0 latency 0.000029 epoch 62 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=120, reqid=osd.4.0:120, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:39.614+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=120, reqid=osd.4.0:120, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.614+0000 7f8f1199c700 3 osd.7 62 handle_osd_map epochs [62,62], i have 62, src has [1,62] 2019-12-07T19:34:39.614+0000 7f8f24457700 15 osd.7 62 enqueue_peering_evt 1.as5 epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s) 2019-12-07T19:34:39.614+0000 7f8f1199c700 10 osd.7 62 no new maps here, dropping 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(62..62 src has 1..62) v4 -- 0x558c582346c0 con 0x558c54c5b400 2019-12-07T19:34:39.614+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 61 -> 62 (shared) 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 62/28 ECSubWrite(tid=59, reqid=osd.7.0:59, at_version=62'54, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c57a43000 con 0x558c54c5b400 2019-12-07T19:34:39.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 send_incremental_map 60 -> 62 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:39.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s)) prio 255 cost 10 e62) pg 0x558c57e8b000 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(61..62 src has 1..62) v4 -- 0x558c58234480 con 0x558c54c19c00 2019-12-07T19:34:39.614+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.as0( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.069826990s ub 75.069826990s int 16s) 2019-12-07T19:34:39.614+0000 7f8f06986700 10 osd.7 62 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 60 -> 62 (shared) 2019-12-07T19:34:39.614+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.614+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 62/28 ECSubWrite(tid=59, reqid=osd.7.0:59, at_version=62'54, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c58154000 con 0x558c54c19c00 2019-12-07T19:34:39.614+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:39.614+0000 7f8f06986700 20 snap_mapper.update_snaps 1:3a5a3f5b:::smithi13140796-23:23 1b,21 was 2019-12-07T19:34:39.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9f2c0 1.as5_head 2019-12-07T19:34:39.614+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10f00 = 0x558c58260f00 seq 31 2019-12-07T19:34:39.614+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:39.614+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# 2019-12-07T19:34:39.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._biginfo' <- _biginfo 2019-12-07T19:34:39.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._epoch' <- _epoch 2019-12-07T19:34:39.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049e'._info' <- _info 2019-12-07T19:34:39.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.as5_head 5#1:50000000::::head# = 0 2019-12-07T19:34:39.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58260f00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58260f00 onodes shared_blobs 2019-12-07T19:34:39.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58260f00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260f00 prepare 2019-12-07T19:34:39.614+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58260f00 2019-12-07T19:34:39.614+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260f00 io_done 2019-12-07T19:34:39.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.614+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:39.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=120, reqid=osd.4.0:120, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.614+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=120, reqid=osd.4.0:120, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.614+0000 7f8f07187700 10 osd.7 62 dequeue_op 0x558c581cdce0 prio 127 cost 0 latency 0.000149 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=120, reqid=osd.4.0:120, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 pg pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.614+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=120, reqid=osd.4.0:120, at_version=62'50, trim_to=0'0, roll_forward_to=59'48)) v2 2019-12-07T19:34:39.614+0000 7f8f07187700 20 snap_mapper.update_snaps 1:d07269f7:::smithi13140796-28:23 1b,21 was 2019-12-07T19:34:39.614+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.614+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:39.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 531 ==== pg_info2(1.0s3 1.0s3 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c57453100 con 0x558c54c64000 2019-12-07T19:34:39.615+0000 7f8f24457700 15 osd.7 62 enqueue_peering_evt 1.0s3 epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s) 2019-12-07T19:34:39.615+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s)) prio 255 cost 10 e62) pg 0x558c5500d000 2019-12-07T19:34:39.615+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.0s0( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 76.078109650s ub 76.078109650s int 16s) 2019-12-07T19:34:39.615+0000 7f8f04982700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.615+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:39.615+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a71c20 1.0s3_head 2019-12-07T19:34:39.615+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6c80 = 0x558c5875ac00 seq 52 2019-12-07T19:34:39.615+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# 2019-12-07T19:34:39.615+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:39.615+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:39.615+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:39.615+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.0s3_head 3#1:00000000::::head# = 0 2019-12-07T19:34:39.615+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875ac00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.615+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875ac00 onodes shared_blobs 2019-12-07T19:34:39.615+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.615+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875ac00 prepare 2019-12-07T19:34:39.615+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875ac00 2019-12-07T19:34:39.615+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875ac00 io_done 2019-12-07T19:34:39.615+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:39.615+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 532 ==== MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=304, reqid=osd.1.0:304, at_version=62'12, trim_to=0'0, roll_forward_to=59'10)) v2 ==== 3417+0+0 (crc 0 0 0) 0x558c58154000 con 0x558c54c64000 2019-12-07T19:34:39.615+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c57efec60 prio 127 cost 0 latency 0.000030 epoch 62 MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=304, reqid=osd.1.0:304, at_version=62'12, trim_to=0'0, roll_forward_to=59'10)) v2 2019-12-07T19:34:39.615+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=304, reqid=osd.1.0:304, at_version=62'12, trim_to=0'0, roll_forward_to=59'10)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=304, reqid=osd.1.0:304, at_version=62'12, trim_to=0'0, roll_forward_to=59'10)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.615+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=304, reqid=osd.1.0:304, at_version=62'12, trim_to=0'0, roll_forward_to=59'10)) v2) prio 127 cost 0 e62) pg 0x558c54d98000 2019-12-07T19:34:39.615+0000 7f8f0197c700 10 osd.7 62 dequeue_op 0x558c57efec60 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=304, reqid=osd.1.0:304, at_version=62'12, trim_to=0'0, roll_forward_to=59'10)) v2 pg pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.615+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=304, reqid=osd.1.0:304, at_version=62'12, trim_to=0'0, roll_forward_to=59'10)) v2 2019-12-07T19:34:39.616+0000 7f8f0197c700 20 snap_mapper.update_snaps 1:733d7474:::smithi13140796-5:23 1b,21 was 2019-12-07T19:34:39.616+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.616+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush cnt:1 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x52f476 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (77) on file(ino 16 size 0x530000 mtime 2019-12-07T19:34:39.615186+0000 allocated 530000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~500000]), flushing log 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x530000 mtime 2019-12-07T19:34:39.615186+0000 allocated 530000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~500000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 77 len 0x4a crc 0x46a285a5) 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x4c000~1000 to file(ino 1 size 0x4c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x4c000 0x4c000~1000 to file(ino 1 size 0x4c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x4d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4c000 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x4d000 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:39.616+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 328 ==== pg_info2(1.3s4 1.3s4 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c5822aa00 con 0x558c54c5b400 2019-12-07T19:34:39.616+0000 7f8f23c56700 15 osd.7 62 enqueue_peering_evt 1.3s4 epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s) 2019-12-07T19:34:39.616+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s)) prio 255 cost 10 e62) pg 0x558c5513e000 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.3s0( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.057037678s ub 75.057037678s int 16s) 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0e780 1.3s4_head 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e140 = 0x558c586a3b00 seq 71 2019-12-07T19:34:39.616+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.3s4_head 4#1:c0000000::::head# = 0 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c586a3b00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000545 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c586a3b00 onodes shared_blobs 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c586a3b00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3b00 prepare 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c586a3b00 2019-12-07T19:34:39.616+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3b00 io_done 2019-12-07T19:34:39.616+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 77 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x530000 mtime 2019-12-07T19:34:39.615186+0000 allocated 530000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~500000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 3 cleaned 0 in 0.003221866s (0.000000138s flush + 0.003221728s kv commit) 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584fcc00,0x558c58c65b00,0x558c5875d500> 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcc00 kv_submitted 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584fcc00 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcc00 finishing 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584fcc00 onodes 0x558c57573900,0x558c57a35400 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584fcc00 done 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584fcc00 [] 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 kv_submitted 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58c65b00 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58c65b00 finishing 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58c65b00 onodes 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58c65b00 done 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10140 q now empty 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58c65b00 [] 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d500 kv_submitted 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875d500 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d500 finishing 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2954 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875d500 onodes 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875d500 done 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c576aa640 q now empty 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875d500 [] 2019-12-07T19:34:39.616+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6196 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.616+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.616+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.616+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 62/41 ECSubWriteReply(tid=303, last_complete=62'50, committed=1, applied=1)) v2 -- 0x558c58333e00 con 0x558c54c64000 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579df680 had 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57c05b80 had 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 7330 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:39.616+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.616+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e500 had 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8464 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148780 had 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9598 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148000 had 1 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630bc0) upper_bound after SNA_1_0000000000000020_.0_0000000000000001.4 key 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.4' 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9621 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9621 < min_flush_size 524288 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c57630bc0) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x530000 mtime 2019-12-07T19:34:39.615186+0000 allocated 530000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~500000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x52f476~2595 to file(ino 16 size 0x530000 mtime 2019-12-07T19:34:39.615186+0000 allocated 530000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~500000]) 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000020_.0_0000000000000001.4) returns 0 SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x52f476 0x52f476~2595 to file(ino 16 size 0x530000 mtime 2019-12-07T19:34:39.615186+0000 allocated 530000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~500000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _allocate len 0x1a0b from 0 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _allocate len 0x1a0b from 1 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd660000 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd660000~10000/10000,10000,acd660000 2019-12-07T19:34:39.616+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x540000 to include allocated 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 78 (was clean) 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x4ff476 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x476 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xa0b and padding block with 0x5f5 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x540000 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[23~1]] SnapTrimmer state: adding snap 20 to purged_snaps 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[20~1,23~1]] SnapTrimmer state: purged_snaps now [20~1,23~1], snap_trimq now [] 2019-12-07T19:34:39.616+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.616+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f0e0 1.4s0_head 2019-12-07T19:34:39.616+0000 7f8f0197c700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525f540 = 0x558c585e4c00 seq 50 2019-12-07T19:34:39.616+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# 2019-12-07T19:34:39.616+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.616+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.616+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:39.616+0000 7f8f07187700 20 snap_mapper.get_snaps 1:d07269f7:::smithi13140796-28:23 1b,20,21 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:39.616+0000 7f8f0197c700 20 snap_mapper.get_snaps 1:733d7474:::smithi13140796-5:23 1b,20,21 2019-12-07T19:34:39.616+0000 7f8f06986700 20 snap_mapper.get_snaps 1:3a5a3f5b:::smithi13140796-23:23 1b,20,21 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:39.616+0000 7f8f07187700 20 snap_mapper.set_snaps 1:d07269f7:::smithi13140796-28:23 1b,21 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.4s0_head 0#1:20000000::::head# = 0 2019-12-07T19:34:39.616+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585e4c00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.616+0000 7f8f06986700 20 snap_mapper.set_snaps 1:3a5a3f5b:::smithi13140796-23:23 1b,21 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585e4c00 onodes shared_blobs 2019-12-07T19:34:39.616+0000 7f8f0197c700 20 snap_mapper.set_snaps 1:733d7474:::smithi13140796-5:23 1b,21 2019-12-07T19:34:39.616+0000 7f8f06986700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:39.616+0000 7f8f0197c700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585e4c00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.616+0000 7f8f07187700 20 snap_mapper.update_snaps rm SNA_1_0000000000000020_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:39.616+0000 7f8f0197c700 20 snap_mapper.update_snaps rm SNA_1_0000000000000020_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:39.616+0000 7f8f06986700 20 snap_mapper.update_snaps rm SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4c00 prepare 2019-12-07T19:34:39.616+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585e4c00 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4c00 io_done 2019-12-07T19:34:39.616+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:39.616+0000 7f8f06986700 20 trim 62'53 (59'47) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.616+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[20~1,23~1]] share_pg_info 2019-12-07T19:34:39.616+0000 7f8f06986700 20 trim 62'54 (59'48) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.616+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 59'48 (0'0,59'48] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] append_log log((0'0,59'48], crt=59'48) [62'49 (59'47) modify 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:39.615048+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'50 (59'48) modify 1:d07269f7:::smithi13140796-28:head by unknown.0.0:0 2019-12-07T19:34:39.615048+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.4s2 1.4s2 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c5509ce00 con 0x558c54c64000 2019-12-07T19:34:39.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 59'10 (0'0,59'10] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] append_log log((0'0,59'10], crt=59'10) [62'11 (59'9) modify 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:34:39.616529+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'12 (59'10) modify 1:733d7474:::smithi13140796-5:head by unknown.0.0:0 2019-12-07T19:34:39.616529+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.616+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'49 (0'0,62'49] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] add_log_entry 62'49 (59'47) modify 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:39.615048+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'11 (0'0,62'11] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=59'10 crt=59'10 lcod 59'10 active mbc={} ps=[23~1]] add_log_entry 62'11 (59'9) modify 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:34:39.616529+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.616+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 60'52 (0'0,60'52] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log log((0'0,60'52], crt=60'52) [62'53 (59'47) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'54 (59'48) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.4s4 1.4s4 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c5509d500 con 0x558c54c19800 2019-12-07T19:34:39.616+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] add_log_entry 62'50 (59'48) modify 1:d07269f7:::smithi13140796-28:head by unknown.0.0:0 2019-12-07T19:34:39.615048+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.616+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'53 (0'0,62'53] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] add_log_entry 62'53 (59'47) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.616+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] append_log approx pg log length = 50 2019-12-07T19:34:39.616+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.4s1 1.4s1 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c5509dc00 con 0x558c54c5bc00 2019-12-07T19:34:39.616+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x531a0b 2019-12-07T19:34:39.616+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] add_log_entry 62'54 (59'48) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.617+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.617+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.616+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=59'10 crt=59'10 lcod 59'10 active mbc={} ps=[23~1]] add_log_entry 62'12 (59'10) modify 1:733d7474:::smithi13140796-5:head by unknown.0.0:0 2019-12-07T19:34:39.616529+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log approx pg log length = 54 2019-12-07T19:34:39.617+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'49, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.4s5 1.4s5 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c5509e300 con 0x558c54c5b400 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=59'10 crt=59'10 lcod 59'10 active mbc={} ps=[23~1]] append_log approx pg log length = 12 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.617+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=59'10 crt=59'10 lcod 59'10 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.617+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.4s3 1.4s3 1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 -- 0x558c5509ea00 con 0x558c54c19c00 2019-12-07T19:34:39.617+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'53, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[20~1,23~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (78) on file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]), flushing log 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:39.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean+snaptrim ps=[20~1,23~1]] exit Trimming 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 78 len 0x4a crc 0x569b0e89) 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c550a2c00 seq 31 2019-12-07T19:34:39.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x4d000~1000 to file(ino 1 size 0x4d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x4d000 0x4d000~1000 to file(ino 1 size 0x4d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58bb3b00 seq 61 2019-12-07T19:34:39.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:23# 1 keys 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x4e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.617+0000 7f8f0297e700 15 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:167 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4d000 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:23# 1 keys = 0 2019-12-07T19:34:39.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# 1 keys 2019-12-07T19:34:39.617+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] enter NotTrimming 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58749500 seq 73 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# 1 keys = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 1 keys 2019-12-07T19:34:39.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# 1 keys = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:39.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:23# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys = 0 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] snap_trimmer complete 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:34:39.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- OBJ_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x4e000 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:39.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5..' <- SNA_1_0000000000000020_.4_0000000000000001.ECCBE2E2.23.smithi13140796-5.. 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000062.00000000000000000011' <- 0000000062.00000000000000000011 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.0000000062.00000000000000000012' <- 0000000062.00000000000000000012 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- OBJ_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._epoch' <- _epoch 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' <- SNA_1_0000000000000020_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23.. 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a2c00 cost 8396 (1 ios * 4000 + 4396 bytes) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a2c00 onodes 0x558c579de500,0x558c579dea00 shared_blobs 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579de608) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000062.00000000000000000053' <- 0000000062.00000000000000000053 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 650 (563 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000062.00000000000000000054' <- 0000000062.00000000000000000054 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579deb08) update 4#1:733d7474:::smithi13140796-5:23# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._epoch' <- _epoch 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- OBJ_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:733d7474:::smithi13140796-5:23# is 497 (410 bytes onode + 2 bytes spanning blobs + 85 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a2c00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58749500 cost 8407 (1 ios * 4000 + 4407 bytes) 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.1_0000000000000001.B0E469FE.23.smithi13140796-28..' <- SNA_1_0000000000000020_.1_0000000000000001.B0E469FE.23.smithi13140796-28.. 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58749500 onodes 0x558c575a3b80,0x558c57bd4500 shared_blobs 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2c00 prepare 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a2c00 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore.extentmap(0x558c575a3c88) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2c00 io_done 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 1003 (565 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4608) update 0#1:3a5a3f5b:::smithi13140796-23:23# 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000062.00000000000000000049' <- 0000000062.00000000000000000049 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:23# is 850 (412 bytes onode + 2 bytes spanning blobs + 436 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58749500 allocated 0x[] released 0x[] 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=59'10 crt=59'10 lcod 59'10 active mbc={} ps=[23~1]] op_applied version 62'12 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749500 prepare 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000062.00000000000000000050' <- 0000000062.00000000000000000050 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58749500 2019-12-07T19:34:39.617+0000 7f8f0197c700 10 osd.7 62 dequeue_op 0x558c57efec60 finish 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749500 io_done 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._epoch' <- _epoch 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bb3b00 cost 8407 (1 ios * 4000 + 4407 bytes) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] op_applied version 62'54 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bb3b00 onodes 0x558c54e44780,0x558c57f35400 shared_blobs 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 586 (584 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore.extentmap(0x558c57f35508) update 1#1:d07269f7:::smithi13140796-28:23# 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:23# is 1567 (413 bytes onode + 2 bytes spanning blobs + 1152 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bb3b00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] eval_repop repgather(0x558c576b6ea0 62'54 rep_tid=59 committed?=0 r=0) (no op) 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000112 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] calc_trim_to_aggressive limit = 60'52 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb3b00 prepare 2019-12-07T19:34:39.617+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bb3b00 2019-12-07T19:34:39.617+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb3b00 io_done 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:39.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: found obc in cache: 0x558c579e66c0 2019-12-07T19:34:39.617+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=59'48 crt=59'48 lcod 59'48 active mbc={} ps=[23~1]] op_applied version 62'50 2019-12-07T19:34:39.617+0000 7f8f07187700 10 osd.7 62 dequeue_op 0x558c581cdce0 finish 2019-12-07T19:34:39.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: 0x558c579e66c0 1:33f5bf01:::smithi13140796-2:25 rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:25(60'49 client.4540.0:380 dirty|data_digest s 1622016 uv 21 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: found obc in cache: 0x558c57571680 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 78 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_object_context: 0x558c57571680 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(60'52 client.4540.0:451 dirty|data_digest s 1622016 uv 52 dd 15460b02 alloc_hint [0 0 0]) exists: 1 ssc: 0x558c576b6120 snapset: 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 6 cleaned 0 in 0.000515400s (0.000000115s flush + 0.000515285s kv commit) 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:33f5bf01:::smithi13140796-2:25 old_snaps 1b,20,21,25 old snapset 25=[]:{25=[25,21,20,1b]} 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58500300,0x558c587ce000,0x558c550a2900,0x558c58260f00,0x558c5875ac00,0x558c586a3b00> 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500300 kv_submitted 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58500300 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500300 finishing 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] simple_opc_create 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54d2cc80 had 1 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58500300 onodes 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58500300 done 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58500300 [] 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5139 < min_flush_size 524288 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce000 kv_submitted 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:33f5bf01:::smithi13140796-2:25 snaps 1b,20,21,25 -> 1b,21,25 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c587ce000 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce000 finishing 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c587ce000 onodes 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c587ce000 done 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c587ce000 [] 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2900 kv_submitted 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a2900 2019-12-07T19:34:39.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579de500 had 1 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579dea00 had 1 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 3 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2900 finishing 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a2900 onodes 0x558c579df680,0x558c57c05b80 2019-12-07T19:34:39.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a2900 done 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a2900 [] 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 9859 < min_flush_size 524288 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260f00 kv_submitted 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:33f5bf01:::smithi13140796-2:25 new snapset 25=[]:{25=[25,21,1b]} on 1:33f5bf01:::smithi13140796-2:head(60'52 client.4540.0:451 dirty|data_digest s 1622016 uv 52 dd 15460b02 alloc_hint [0 0 0]) 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58260f00 2019-12-07T19:34:39.617+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58260f00 finishing 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58260f00 onodes 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58260f00 done 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10f00 q now empty 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58260f00 [] 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.617+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875ac00 kv_submitted 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875ac00 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 1:33f5bf01:::smithi13140796-2:25 writing updated snapset on 1:33f5bf01:::smithi13140796-2:head, snapset is 25=[]:{25=[25,21,1b]} 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875ac00 finishing 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875ac00 onodes 2019-12-07T19:34:39.617+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875ac00 done 2019-12-07T19:34:39.617+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 62/28 ECSubWriteReply(tid=161, last_complete=62'77, committed=1, applied=1)) v2 -- 0x558c57519000 con 0x558c54c5bc00 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6c80 q now empty 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875ac00 [] 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a3b80 had 1 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3b00 kv_submitted 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c586a3b00 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd4500 had 1 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] new_repop rep_tid 60 (no op) 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 2 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c586a3b00 finishing 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c586a3b00 onodes 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c586a3b00 done 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e140 q now empty 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] new_repop: repgather(0x558c576b6360 0'0 rep_tid=60 committed?=0 r=0) 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c586a3b00 [] 2019-12-07T19:34:39.617+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.617+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14879 < min_flush_size 524288 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] simple_opc_submit 0x558c576b6360 2019-12-07T19:34:39.617+0000 7f8f06986700 7 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] issue_repop rep_tid 60 o 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f35400 had 1 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:25 v=62'56 tt=0'0 tid=60 reqid=osd.7.0:60 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14902 < min_flush_size 524288 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 14902 < min_flush_size 524288 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:25 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x531a0b~3a36 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# 2019-12-07T19:34:39.617+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x531a0b 0x531a0b~3a36 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x501a0b 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xa0b 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.617+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x441 and padding block with 0xbbf 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# hinfo_key 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# hinfo_key = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:33f5bf01:::smithi13140796-2:25 projected size 1622016 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] get_hash_info: found on disk, size 405504 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:34:39.617+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 1622016 2019-12-07T19:34:39.617+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] start_rmw: Op(1:33f5bf01:::smithi13140796-2:25 v=62'56 tt=0'0 tid=60 reqid=osd.7.0:60 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:25 v=62'56 tt=0'0 tid=60 reqid=osd.7.0:60 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:25 v=62'56 tt=0'0 tid=60 reqid=osd.7.0:60 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:25, current size is 1622016 buffers are {} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): new_size start 1622016 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_append: {} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:33f5bf01:::smithi13140796-2:25 resetting hinfo to logical size 1622016 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 1622016 buffers are {} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): new_size start 1622016 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): to_append: {} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 1622016 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:25={},1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:25 v=62'56 tt=0'0 tid=60 reqid=osd.7.0:60 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:25,{} 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:34:39.617+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 62/28 ECSubWrite(tid=60, reqid=osd.7.0:60, at_version=62'56, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c58155800 con 0x558c54c64000 2019-12-07T19:34:39.617+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 62/28 ECSubWrite(tid=60, reqid=osd.7.0:60, at_version=62'56, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c54e40800 con 0x558c54c19800 2019-12-07T19:34:39.617+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 62/28 ECSubWrite(tid=60, reqid=osd.7.0:60, at_version=62'56, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c54e41000 con 0x558c54c5bc00 2019-12-07T19:34:39.617+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 62/28 ECSubWrite(tid=60, reqid=osd.7.0:60, at_version=62'56, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c54e3f000 con 0x558c54c5b400 2019-12-07T19:34:39.617+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 62/28 ECSubWrite(tid=60, reqid=osd.7.0:60, at_version=62'56, trim_to=0'0, roll_forward_to=60'52)) v2 -- 0x558c57e00000 con 0x558c54c19c00 2019-12-07T19:34:39.617+0000 7f8f06986700 20 snap_mapper.update_snaps 1:33f5bf01:::smithi13140796-2:25 1b,21,25 was 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 20 snap_mapper.get_snaps 1:33f5bf01:::smithi13140796-2:25 1b,20,21,25 2019-12-07T19:34:39.617+0000 7f8f06986700 20 snap_mapper.set_snaps 1:33f5bf01:::smithi13140796-2:25 1b,21,25 2019-12-07T19:34:39.617+0000 7f8f06986700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:39.617+0000 7f8f06986700 20 snap_mapper.update_snaps rm SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:39.617+0000 7f8f06986700 20 earliest_dup_version = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 20 trim 62'55 (60'49) modify 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.617+0000 7f8f06986700 20 trim 62'56 (60'52) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'54 (0'0,62'54] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log log((0'0,62'54], crt=60'52) [62'55 (60'49) modify 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 snaps [1b,21,25] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'56 (60'52) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'55 (0'0,62'55] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=62'54 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] add_log_entry 62'55 (60'49) modify 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 snaps [1b,21,25] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=62'54 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] add_log_entry 62'56 (60'52) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=62'54 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log approx pg log length = 56 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=62'54 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.617+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.617+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'55, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58749800 seq 74 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# 1 keys 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# 1 keys = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:25# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' <- OBJ_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2..' <- SNA_1_0000000000000020_.0_0000000000000001.CCFADF08.25.smithi13140796-2.. 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000062.00000000000000000055' <- 0000000062.00000000000000000055 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.0000000062.00000000000000000056' <- 0000000062.00000000000000000056 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._fastinfo' <- _fastinfo 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58749800 cost 7628 (1 ios * 4000 + 3628 bytes) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58749800 onodes 0x558c57bd4c80,0x558c57bd5400 shared_blobs 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd4d88) update 0#1:33f5bf01:::smithi13140796-2:25# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:25# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore.extentmap(0x558c57bd5508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 588 (586 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58749800 allocated 0x[] released 0x[] 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749800 prepare 2019-12-07T19:34:39.617+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58749800 2019-12-07T19:34:39.617+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749800 io_done 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 lua=62'54 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] op_applied version 62'56 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] eval_repop repgather(0x558c576b6360 62'56 rep_tid=60 committed?=0 r=0) (no op) 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] calc_trim_to_aggressive limit = 60'52 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] enter Trimming/WaitRepops 2019-12-07T19:34:39.617+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] snap_trimmer complete 2019-12-07T19:34:39.617+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:39.617+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:39.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 533 ==== MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=305, reqid=osd.1.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57e00000 con 0x558c54c64000 2019-12-07T19:34:39.618+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c578d2420 prio 127 cost 0 latency 0.000022 epoch 62 MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=305, reqid=osd.1.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.618+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=305, reqid=osd.1.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=305, reqid=osd.1.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGOpItem(op=MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=305, reqid=osd.1.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) pg 0x558c5500a000 2019-12-07T19:34:39.618+0000 7f8f04181700 10 osd.7 62 dequeue_op 0x558c578d2420 prio 127 cost 0 latency 0.000198 MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=305, reqid=osd.1.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 pg pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.618+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.9s4 62/41 ECSubWrite(tid=305, reqid=osd.1.0:303, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.618+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] append_log log((0'0,62'50], crt=59'48) [] 2019-12-07T19:34:39.618+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[23~1]] rollforward: entry=62'49 (57'43) modify 1:9b8adabe:::smithi13140796-6:20 by unknown.0.0:0 2019-12-07T19:34:39.612114+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.618+0000 7f8f04181700 20 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[23~1]] rollforward: entry=62'50 (59'48) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:39.612114+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.618+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[23~1]] append_log approx pg log length = 50 2019-12-07T19:34:39.618+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.618+0000 7f8f04181700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.618+0000 7f8f04181700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c580d2f00 seq 93 2019-12-07T19:34:39.618+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:39.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c580d2f00 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c580d2f00 onodes shared_blobs 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c580d2f00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2f00 prepare 2019-12-07T19:34:39.618+0000 7f8f04181700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c580d2f00 2019-12-07T19:34:39.618+0000 7f8f04181700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2f00 io_done 2019-12-07T19:34:39.618+0000 7f8f04181700 10 osd.7 62 dequeue_op 0x558c578d2420 finish 2019-12-07T19:34:39.618+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.618+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 534 ==== pg_info2(1.9s4 1.9s4 1.9s0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c5509ea00 con 0x558c54c64000 2019-12-07T19:34:39.618+0000 7f8f24457700 15 osd.7 62 enqueue_peering_evt 1.9s4 epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.9s0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s) 2019-12-07T19:34:39.618+0000 7f8f24457700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.9s0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.9s0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.9s0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s)) prio 255 cost 10 e62) pg 0x558c5500a000 2019-12-07T19:34:39.618+0000 7f8f08189700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.9s0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068844211s ub 75.068844211s int 16s) 2019-12-07T19:34:39.618+0000 7f8f08189700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.618+0000 7f8f08189700 10 log is not dirty 2019-12-07T19:34:39.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0ef00 1.9s4_head 2019-12-07T19:34:39.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf6000 = 0x558c584fcf00 seq 94 2019-12-07T19:34:39.618+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# 2019-12-07T19:34:39.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:39.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:39.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.9s4_head 4#1:90000000::::head# = 0 2019-12-07T19:34:39.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584fcf00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:39.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584fcf00 onodes shared_blobs 2019-12-07T19:34:39.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584fcf00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcf00 prepare 2019-12-07T19:34:39.618+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584fcf00 2019-12-07T19:34:39.618+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcf00 io_done 2019-12-07T19:34:39.618+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.618+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 314 ==== MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=162, reqid=osd.3.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=62'77)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c54e41000 con 0x558c54c5bc00 2019-12-07T19:34:39.618+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c581cda20 prio 127 cost 0 latency 0.000021 epoch 62 MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=162, reqid=osd.3.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:39.618+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=162, reqid=osd.3.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=162, reqid=osd.3.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=162, reqid=osd.3.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e62) pg 0x558c54d5c000 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 osd.7 62 dequeue_op 0x558c581cda20 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=162, reqid=osd.3.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=62'77)) v2 pg pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 62/28 ECSubWrite(tid=162, reqid=osd.3.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=61'75 active mbc={} ps=[23~1]] append_log log((0'0,62'77], crt=61'75) [] 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[23~1]] rollforward: entry=62'76 (61'72) modify 1:6c205177:::smithi13140796-8:26 by unknown.0.0:0 2019-12-07T19:34:39.612394+0000 0 snaps [1b,21,25,26] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[23~1]] rollforward: entry=62'77 (61'75) modify 1:6c205177:::smithi13140796-8:head by unknown.0.0:0 2019-12-07T19:34:39.612394+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[23~1]] append_log approx pg log length = 77 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.618+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c550a2f00 seq 117 2019-12-07T19:34:39.618+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a2f00 cost 5321 (1 ios * 4000 + 1321 bytes) 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a2f00 onodes shared_blobs 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a2f00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2f00 prepare 2019-12-07T19:34:39.618+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a2f00 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2f00 io_done 2019-12-07T19:34:39.618+0000 7f8f0197c700 10 osd.7 62 dequeue_op 0x558c581cda20 finish 2019-12-07T19:34:39.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x535441 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.001992262s (0.000000075s flush + 0.001992187s kv commit) 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 3091 < min_flush_size 524288 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd4c80 had 1 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57bd5400 had 1 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4229 < min_flush_size 524288 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 2 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5339 < min_flush_size 524288 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148500 had 1 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6477 < min_flush_size 524288 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6500 < min_flush_size 524288 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6500 < min_flush_size 524288 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x535441~1964 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x535441 0x535441~1964 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x505441 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x441 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xda5 and padding block with 0x25b 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585e4c00,0x558c550a2c00,0x558c58749500,0x558c58bb3b00> 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4c00 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585e4c00 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4c00 finishing 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585e4c00 onodes 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4c00 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525f540 q now empty 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585e4c00 [] 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2c00 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a2c00 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2c00 finishing 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x536da5 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a2c00 onodes 0x558c579de500,0x558c579dea00 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a2c00 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a2c00 [] 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749500 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 62/41 ECSubWriteReply(tid=304, last_complete=62'12, committed=1, applied=1)) v2 -- 0x558c58512800 con 0x558c54c64000 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58749500 2019-12-07T19:34:39.619+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749500 finishing 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58749500 onodes 0x558c575a3b80,0x558c57bd4500 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58749500 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58749800 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58749500 [] 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb3b00 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bb3b00 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 4 cleaned 0 in 0.000191152s (0.000000075s flush + 0.000191077s kv commit) 2019-12-07T19:34:39.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb3b00 finishing 2019-12-07T19:34:39.619+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58bb3b00 onodes 0x558c54e44780,0x558c57f35400 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bb3b00 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58bb3b00 [] 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58749800,0x558c580d2f00,0x558c584fcf00,0x558c550a2f00> 2019-12-07T19:34:39.619+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749800 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58749800 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749800 finishing 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58749800 onodes 0x558c57bd4c80,0x558c57bd5400 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58749800 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58749800 [] 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2f00 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c580d2f00 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c580d2f00 finishing 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c580d2f00 onodes 2019-12-07T19:34:39.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c580d2f00 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584fcf00 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c580d2f00 [] 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcf00 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584fcf00 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fcf00 finishing 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584fcf00 onodes 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584fcf00 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf6000 q now empty 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584fcf00 [] 2019-12-07T19:34:39.619+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 62/41 ECSubWriteReply(tid=120, last_complete=62'50, committed=1, applied=1)) v2 -- 0x558c54c59200 con 0x558c54c5b400 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2f00 kv_submitted 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a2f00 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a2f00 finishing 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a2f00 onodes 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a2f00 done 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a2f00 [] 2019-12-07T19:34:39.619+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.619+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.619+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 62/28 ECSubWriteReply(tid=162, last_complete=62'77, committed=1, applied=1)) v2 -- 0x558c58513e00 con 0x558c54c5bc00 2019-12-07T19:34:39.619+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 62/41 ECSubWriteReply(tid=305, last_complete=62'50, committed=1, applied=1)) v2 -- 0x558c58333800 con 0x558c54c64000 2019-12-07T19:34:39.619+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 329 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c584fec00 con 0x558c54c5b400 2019-12-07T19:34:39.619+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c581cc6e0 prio 196 cost 0 latency 0.000016 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.619+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.619+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 330 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c57519a00 con 0x558c54c5b400 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.619+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c581cd4a0 prio 196 cost 0 latency 0.000008 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.619+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c581cc6e0 prio 196 cost 0 latency 0.000043 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c581cc6e0 finish 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.619+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 122 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58333800 con 0x558c54c19c00 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c581cd4a0 prio 196 cost 0 latency 0.000058 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.619+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c578d26e0 prio 196 cost 0 latency 0.000014 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.619+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c581cd4a0 finish 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.619+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c578d26e0 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.619+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c578d26e0 finish 2019-12-07T19:34:39.619+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.620+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 535 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c580b8c00 con 0x558c54c64000 2019-12-07T19:34:39.620+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c587546e0 prio 196 cost 0 latency 0.000016 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.620+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.620+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c587546e0 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.620+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.620+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c587546e0 finish 2019-12-07T19:34:39.620+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.620+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 315 ==== pg_info2(1.6s2 1.6s2 1.6s0( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c5509dc00 con 0x558c54c5bc00 2019-12-07T19:34:39.620+0000 7f8f23c56700 15 osd.7 62 enqueue_peering_evt 1.6s2 epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.6s0( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 78.116716330s ub 78.116716330s int 16s) 2019-12-07T19:34:39.620+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.6s0( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 78.116716330s ub 78.116716330s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.6s0( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 78.116716330s ub 78.116716330s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.6s0( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 78.116716330s ub 78.116716330s int 16s)) prio 255 cost 10 e62) pg 0x558c54d5c000 2019-12-07T19:34:39.620+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 3(0) info: 1.6s0( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 78.116716330s ub 78.116716330s int 16s) 2019-12-07T19:34:39.620+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.620+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:39.620+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:39.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5876c000 seq 118 2019-12-07T19:34:39.620+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:39.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:39.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:39.620+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:39.620+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5876c000 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:39.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5876c000 onodes shared_blobs 2019-12-07T19:34:39.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5876c000 allocated 0x[] released 0x[] 2019-12-07T19:34:39.620+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c000 prepare 2019-12-07T19:34:39.620+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5876c000 2019-12-07T19:34:39.620+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c000 io_done 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x536da5~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x536da5 0x536da5~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x506da5 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xda5 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x212 and padding block with 0xdee 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x537212 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116672s (0.000000090s flush + 0.000116582s kv commit) 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5876c000> 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.620+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 216 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c5509d500 con 0x558c54c19800 2019-12-07T19:34:39.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c000 kv_submitted 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5876c000 2019-12-07T19:34:39.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c000 finishing 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5876c000 onodes 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5876c000 done 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:39.620+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5876c000 [] 2019-12-07T19:34:39.620+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.620+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.620+0000 7f8f24c58700 15 osd.7 62 enqueue_peering_evt 1.5s4 epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s) 2019-12-07T19:34:39.620+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s)) prio 255 cost 10 e62) pg 0x558c57522000 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082067284s ub 75.082067284s int 16s) 2019-12-07T19:34:39.620+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.620+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df65a0 1.5s4_head 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6e280 = 0x558c58756900 seq 26 2019-12-07T19:34:39.620+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._biginfo' <- _biginfo 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._epoch' <- _epoch 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049a'._info' <- _info 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58756900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58756900 onodes shared_blobs 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58756900 allocated 0x[] released 0x[] 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756900 prepare 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58756900 2019-12-07T19:34:39.620+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756900 io_done 2019-12-07T19:34:39.620+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58092a00 had 1 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x537212~485 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x537212 0x537212~485 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x507212 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x212 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x697 and padding block with 0x969 2019-12-07T19:34:39.620+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x537697 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000114167s (0.000000086s flush + 0.000114081s kv commit) 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58756900> 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756900 kv_submitted 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58756900 2019-12-07T19:34:39.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756900 finishing 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58756900 onodes 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58756900 done 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6e280 q now empty 2019-12-07T19:34:39.621+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58756900 [] 2019-12-07T19:34:39.621+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.621+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.621+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 217 ==== MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=79, reqid=osd.2.0:79, at_version=62'32, trim_to=0'0, roll_forward_to=57'30)) v2 ==== 2768+0+0 (crc 0 0 0) 0x558c54e40800 con 0x558c54c19800 2019-12-07T19:34:39.621+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c5511fb80 prio 127 cost 0 latency 0.000029 epoch 62 MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=79, reqid=osd.2.0:79, at_version=62'32, trim_to=0'0, roll_forward_to=57'30)) v2 2019-12-07T19:34:39.621+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=79, reqid=osd.2.0:79, at_version=62'32, trim_to=0'0, roll_forward_to=57'30)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 316 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57518000 con 0x558c54c5bc00 2019-12-07T19:34:39.621+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c581cc9a0 prio 196 cost 0 latency 0.000016 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.621+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=79, reqid=osd.2.0:79, at_version=62'32, trim_to=0'0, roll_forward_to=57'30)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.621+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c581cc9a0 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=79, reqid=osd.2.0:79, at_version=62'32, trim_to=0'0, roll_forward_to=57'30)) v2) prio 127 cost 0 e62) pg 0x558c55141000 2019-12-07T19:34:39.621+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.621+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c581cc9a0 finish 2019-12-07T19:34:39.621+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 62 dequeue_op 0x558c5511fb80 prio 127 cost 0 latency 0.000138 MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=79, reqid=osd.2.0:79, at_version=62'32, trim_to=0'0, roll_forward_to=57'30)) v2 pg pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=79, reqid=osd.2.0:79, at_version=62'32, trim_to=0'0, roll_forward_to=57'30)) v2 2019-12-07T19:34:39.621+0000 7f8f06185700 20 snap_mapper.remove_oid 1:bca18876:::smithi13140796-30:20 2019-12-07T19:34:39.621+0000 7f8f06185700 20 snap_mapper._remove_oid 1:bca18876:::smithi13140796-30:20 2019-12-07T19:34:39.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.621+0000 7f8f06185700 20 snap_mapper.get_snaps 1:bca18876:::smithi13140796-30:20 20 2019-12-07T19:34:39.621+0000 7f8f06185700 20 snap_mapper.clear_snaps 1:bca18876:::smithi13140796-30:20 2019-12-07T19:34:39.621+0000 7f8f06185700 20 snap_mapper.clear_snaps rm OBJ_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:39.621+0000 7f8f06185700 20 snap_mapper._remove_oid rm SNA_1_0000000000000020_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 57'30 (0'0,57'30] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] append_log log((0'0,57'30], crt=57'30) [62'31 (55'25) delete 1:bca18876:::smithi13140796-30:20 by unknown.0.0:0 2019-12-07T19:34:39.621745+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'32 (57'30) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:39.621745+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'31 (0'0,62'31] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=57'30 crt=57'30 lcod 57'30 active mbc={} ps=[23~1]] add_log_entry 62'31 (55'25) delete 1:bca18876:::smithi13140796-30:20 by unknown.0.0:0 2019-12-07T19:34:39.621745+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=57'30 crt=57'30 lcod 57'30 active mbc={} ps=[23~1]] add_log_entry 62'32 (57'30) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:39.621745+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=57'30 crt=57'30 lcod 57'30 active mbc={} ps=[23~1]] append_log approx pg log length = 32 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=57'30 crt=57'30 lcod 57'30 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.621+0000 7f8f06185700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.621+0000 7f8f06185700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'31, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5853b500 seq 69 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) get_onode oid 2#1:bca18876:::smithi13140796-30:20#1f key 0x828000000000000001bca188'v!smithi13140796-30!='0x0000000000000020000000000000001f'o' 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.ds2_head 0x558c54d0f4a0) r -2 v.len 0 2019-12-07T19:34:39.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# -> 2#1:bca18876:::smithi13140796-30:20#1f 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.ds2_head 2#1:bca18876:::smithi13140796-30:20# -> 2#1:bca18876:::smithi13140796-30:20#1f = 0 2019-12-07T19:34:39.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:34:39.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.ds2_head 2#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.D35811E6.20.smithi13140796-30..' <- OBJ_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.2_0000000000000001.D35811E6.20.smithi13140796-30..' <- SNA_1_0000000000000020_.2_0000000000000001.D35811E6.20.smithi13140796-30.. 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.621+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000062.00000000000000000031' <- 0000000062.00000000000000000031 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.0000000062.00000000000000000032' <- 0000000062.00000000000000000032 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853b500 cost 7663 (1 ios * 4000 + 3663 bytes) 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853b500 onodes 0x558c54e12c80,0x558c585cf180 shared_blobs 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore.extentmap(0x558c54e12d88) update 2#1:bca18876:::smithi13140796-30:20#1f 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore.extentmap(0x558c54e12d88) update shard 0x0 is 489 bytes (was 489) from 18 extents 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore.extentmap(0x558c54e12d88) update shard 0x24000 is 491 bytes (was 491) from 18 extents 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore.extentmap(0x558c54e12d88) update shard 0x48000 is 302 bytes (was 302) from 11 extents 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:20#1f is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore.extentmap(0x558c585cf288) update 2#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:bca18876:::smithi13140796-30:head# is 556 (554 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853b500 allocated 0x[] released 0x[] 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b500 prepare 2019-12-07T19:34:39.621+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853b500 2019-12-07T19:34:39.621+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b500 io_done 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 lua=57'30 crt=57'30 lcod 57'30 active mbc={} ps=[23~1]] op_applied version 62'32 2019-12-07T19:34:39.621+0000 7f8f06185700 10 osd.7 62 dequeue_op 0x558c5511fb80 finish 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2409 < min_flush_size 524288 2019-12-07T19:34:39.621+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5163 < min_flush_size 524288 2019-12-07T19:34:39.621+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 536 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58512800 con 0x558c54c64000 2019-12-07T19:34:39.621+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c58754840 prio 196 cost 0 latency 0.000013 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.621+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12c80 had 1 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585cf180 had 1 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57c04000 had 1 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5186 < min_flush_size 524288 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5186 < min_flush_size 524288 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x537697~1442 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.621+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x537697 0x537697~1442 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 331 ==== MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=121, reqid=osd.4.0:120, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c54e3f000 con 0x558c54c5b400 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x507697 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x697 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.621+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xad9 and padding block with 0x527 2019-12-07T19:34:39.621+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c581cc580 prio 127 cost 0 latency 0.000019 epoch 62 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=121, reqid=osd.4.0:120, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.621+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=121, reqid=osd.4.0:120, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.621+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 537 ==== MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=306, reqid=osd.1.0:306, at_version=62'43, trim_to=0'0, roll_forward_to=57'41)) v2 ==== 3362+0+0 (crc 0 0 0) 0x558c58155800 con 0x558c54c64000 2019-12-07T19:34:39.621+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c587554a0 prio 127 cost 0 latency 0.000029 epoch 62 MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=306, reqid=osd.1.0:306, at_version=62'43, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:39.621+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=306, reqid=osd.1.0:306, at_version=62'43, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.621+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 332 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558c5509dc00 con 0x558c54c5b400 2019-12-07T19:34:39.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.621+0000 7f8f23c56700 15 osd.7 62 enqueue_peering_evt 1.bs1 epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.bs0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s) 2019-12-07T19:34:39.621+0000 7f8f23c56700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.bs0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.621+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c58754840 prio 196 cost 0 latency 0.000092 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.621+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.621+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c58754840 finish 2019-12-07T19:34:39.621+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=306, reqid=osd.1.0:306, at_version=62'43, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.621+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=306, reqid=osd.1.0:306, at_version=62'43, trim_to=0'0, roll_forward_to=57'41)) v2) prio 127 cost 0 e62) pg 0x558c54d5f000 2019-12-07T19:34:39.621+0000 7f8f05183700 10 osd.7 62 dequeue_op 0x558c587554a0 prio 127 cost 0 latency 0.000107 MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=306, reqid=osd.1.0:306, at_version=62'43, trim_to=0'0, roll_forward_to=57'41)) v2 pg pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=306, reqid=osd.1.0:306, at_version=62'43, trim_to=0'0, roll_forward_to=57'41)) v2 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.bs0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f05183700 20 snap_mapper.update_snaps 1:ec1285ab:::smithi13140796-19:20 1b was 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.bs0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s)) prio 255 cost 10 e62) pg 0x558c575ef000 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 4(0) info: 1.bs0( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.058137057s ub 75.058137057s int 16s) 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.622+0000 7f8f05183700 20 snap_mapper.get_snaps 1:ec1285ab:::smithi13140796-19:20 1b,20 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x538ad9 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.622+0000 7f8f05183700 20 snap_mapper.set_snaps 1:ec1285ab:::smithi13140796-19:20 1b 2019-12-07T19:34:39.622+0000 7f8f05183700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:39.622+0000 7f8f05183700 20 snap_mapper.update_snaps rm SNA_1_0000000000000020_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000234259s (0.000000108s flush + 0.000234151s kv commit) 2019-12-07T19:34:39.622+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 218 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c584fa000 con 0x558c54c19800 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58bc2000 seq 62 2019-12-07T19:34:39.622+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c5511edc0 prio 196 cost 0 latency 0.000014 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.622+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.622+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._biginfo' <- _biginfo 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 57'41 (0'0,57'41] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] append_log log((0'0,57'41], crt=57'41) [62'42 (57'40) modify 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:39.621425+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'43 (57'41) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:39.621425+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bc2000 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'42 (0'0,62'42] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[23~1]] add_log_entry 62'42 (57'40) modify 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:39.621425+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bc2000 onodes shared_blobs 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bc2000 allocated 0x[] released 0x[] 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2000 prepare 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bc2000 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[23~1]] add_log_entry 62'43 (57'41) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:39.621425+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2000 io_done 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[23~1]] append_log approx pg log length = 43 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5511edc0 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.622+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=121, reqid=osd.4.0:120, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=59, last_complete=62'54, committed=1, applied=1)) v2 2019-12-07T19:34:39.622+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'42, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=121, reqid=osd.4.0:120, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c581cc580 prio 127 cost 0 latency 0.000243 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=121, reqid=osd.4.0:120, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 pg pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:23 v=62'54 tt=0'0 tid=59 reqid=osd.7.0:59 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=60'52 crt=60'52 lcod 60'51 mlcod 60'51 active+clean+snaptrim trimq=[20~1] ps=[23~1]] repop_all_committed: repop tid 59 all committed 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=121, reqid=osd.4.0:120, at_version=0'0, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] eval_repop repgather(0x558c576b6ea0 62'54 rep_tid=59 committed?=1 r=0) (no op) 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=59'48 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'50], crt=59'48) [] 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] commit: repgather(0x558c576b6ea0 62'54 rep_tid=59 committed?=1 r=0) 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c583b7200 seq 91 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'49 (59'47) modify 1:d07269f7:::smithi13140796-28:23 by unknown.0.0:0 2019-12-07T19:34:39.615048+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.622+0000 7f8f0297e700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:213 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# 1 keys 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# 1 keys = 0 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] removing repgather(0x558c576b6ea0 62'54 rep_tid=59 committed?=1 r=0) 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# hinfo_key (42 bytes) 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'50 (59'48) modify 1:d07269f7:::smithi13140796-28:head by unknown.0.0:0 2019-12-07T19:34:39.615048+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:20# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] q front is repgather(0x558c576b6ea0 62'54 rep_tid=59 committed?=1 r=0) 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 50 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] remove_repop repgather(0x558c576b6ea0 62'54 rep_tid=59 committed?=1 r=0) 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.622+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- OBJ_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:23 v=62'54 tt=0'0 tid=59 reqid=osd.7.0:59 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:23=[],1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.4_0000000000000001.73841A5D.20.smithi13140796-19..' <- SNA_1_0000000000000020_.4_0000000000000001.73841A5D.20.smithi13140796-19.. 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.622+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000062.00000000000000000042' <- 0000000062.00000000000000000042 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000062.00000000000000000043' <- 0000000062.00000000000000000043 2019-12-07T19:34:39.622+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5511edc0 finish 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:39.622+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58bc2300 seq 63 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:39.622+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b7200 cost 8335 (1 ios * 4000 + 4335 bytes) 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b7200 onodes 0x558c585ce000,0x558c585ce280 shared_blobs 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce108) update 4#1:ec1285ab:::smithi13140796-19:20# 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:20# is 427 (425 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce388) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bc2300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bc2300 onodes shared_blobs 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bc2300 allocated 0x[] released 0x[] 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b7200 allocated 0x[] released 0x[] 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2300 prepare 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x538ad9~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bc2300 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2300 io_done 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7200 prepare 2019-12-07T19:34:39.622+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b7200 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x538ad9 0x538ad9~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c581cc580 finish 2019-12-07T19:34:39.622+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7200 io_done 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x508ad9 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xad9 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf46 and padding block with 0xba 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=57'41 crt=57'41 lcod 57'41 active mbc={} ps=[23~1]] op_applied version 62'43 2019-12-07T19:34:39.622+0000 7f8f05183700 10 osd.7 62 dequeue_op 0x558c587554a0 finish 2019-12-07T19:34:39.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853b500> 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b500 kv_submitted 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853b500 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b500 finishing 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853b500 onodes 0x558c54e12c80,0x558c585cf180 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853b500 done 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853b500 [] 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x538f46 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122040s (0.000000095s flush + 0.000121945s kv commit) 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58bc2000> 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2000 kv_submitted 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bc2000 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2000 finishing 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58bc2000 onodes 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bc2000 done 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bc2300 kv_queued 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58bc2000 [] 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.622+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4835 < min_flush_size 524288 2019-12-07T19:34:39.622+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 62/41 ECSubWriteReply(tid=79, last_complete=62'32, committed=1, applied=1)) v2 -- 0x558c5795da00 con 0x558c54c19800 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce000 had 1 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce280 had 1 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4858 < min_flush_size 524288 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 4858 < min_flush_size 524288 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x538f46~12fa to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x538f46 0x538f46~12fa to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x508f46 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf46 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x240 and padding block with 0xdc0 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53a240 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.622+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 2 cleaned 0 in 0.000146278s (0.000000079s flush + 0.000146199s kv commit) 2019-12-07T19:34:39.622+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58bc2300,0x558c583b7200> 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2300 kv_submitted 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bc2300 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2300 finishing 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58bc2300 onodes 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bc2300 done 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58bc2300 [] 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7200 kv_submitted 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b7200 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7200 finishing 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b7200 onodes 0x558c585ce000,0x558c585ce280 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b7200 done 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b7200 [] 2019-12-07T19:34:39.622+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.622+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.622+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 62/41 ECSubWriteReply(tid=121, last_complete=62'50, committed=1, applied=1)) v2 -- 0x558c58115a00 con 0x558c54c5b400 2019-12-07T19:34:39.622+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 123 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58333e00 con 0x558c54c19c00 2019-12-07T19:34:39.622+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c58755600 prio 196 cost 0 latency 0.000015 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.622+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.622+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 62/41 ECSubWriteReply(tid=306, last_complete=62'43, committed=1, applied=1)) v2 -- 0x558c54d76000 con 0x558c54c64000 2019-12-07T19:34:39.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.622+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.622+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c58755600 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.622+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.622+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c58755600 finish 2019-12-07T19:34:39.622+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 317 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58115a00 con 0x558c54c5bc00 2019-12-07T19:34:39.624+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c5511f340 prio 196 cost 0 latency 0.000011 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5511f340 prio 196 cost 0 latency 0.000042 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5511f340 finish 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 219 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795da00 con 0x558c54c19800 2019-12-07T19:34:39.624+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c5511e2c0 prio 196 cost 0 latency 0.000011 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c5511e2c0 prio 196 cost 0 latency 0.000030 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=60, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:25 v=62'56 tt=0'0 tid=60 reqid=osd.7.0:60 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 luod=62'54 crt=60'52 lcod 60'52 mlcod 60'52 active+clean+snaptrim trimq=[20~1] ps=[23~1]] repop_all_committed: repop tid 60 all committed 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] eval_repop repgather(0x558c576b6360 62'56 rep_tid=60 committed?=1 r=0) (no op) 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] commit: repgather(0x558c576b6360 62'56 rep_tid=60 committed?=1 r=0) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.624+0000 7f8f06986700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:214 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] removing repgather(0x558c576b6360 62'56 rep_tid=60 committed?=1 r=0) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] q front is repgather(0x558c576b6360 62'56 rep_tid=60 committed?=1 r=0) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] exit Trimming/WaitRepops 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 62 queueing pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] for snaptrim 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=62) prio 5 cost 1048576 e62) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] prepare_stats_for_publish reporting purged_snaps [23~1] 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=62) prio 5 cost 1048576 e62) queued 2019-12-07T19:34:39.624+0000 7f8f06986700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] publish_stats_to_osd 62:215 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] remove_repop repgather(0x558c576b6360 62'56 rep_tid=60 committed?=1 r=0) 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:25 v=62'56 tt=0'0 tid=60 reqid=osd.7.0:60 roll_forward_to=60'52 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:25=[],1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:25 v=0'0 tt=0'0 tid=61 reqid=osd.7.0:60 roll_forward_to=62'56 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:25 v=0'0 tt=0'0 tid=61 reqid=osd.7.0:60 roll_forward_to=62'56 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:39.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 62/28 ECSubWrite(tid=61, reqid=osd.7.0:60, at_version=0'0, trim_to=0'0, roll_forward_to=62'56)) v2 -- 0x558c5822e000 con 0x558c54c64000 2019-12-07T19:34:39.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 62/28 ECSubWrite(tid=61, reqid=osd.7.0:60, at_version=0'0, trim_to=0'0, roll_forward_to=62'56)) v2 -- 0x558c5822f000 con 0x558c54c19800 2019-12-07T19:34:39.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 62/28 ECSubWrite(tid=61, reqid=osd.7.0:60, at_version=0'0, trim_to=0'0, roll_forward_to=62'56)) v2 -- 0x558c58231800 con 0x558c54c5bc00 2019-12-07T19:34:39.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 62/28 ECSubWrite(tid=61, reqid=osd.7.0:60, at_version=0'0, trim_to=0'0, roll_forward_to=62'56)) v2 -- 0x558c575c1800 con 0x558c54c5b400 2019-12-07T19:34:39.624+0000 7f8f06986700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 62/28 ECSubWrite(tid=61, reqid=osd.7.0:60, at_version=0'0, trim_to=0'0, roll_forward_to=62'56)) v2 -- 0x558c57dff800 con 0x558c54c19c00 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=60'52 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log log((0'0,62'56], crt=60'52) [] 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] rollforward: entry=62'53 (59'47) modify 1:3a5a3f5b:::smithi13140796-23:23 by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] rollforward: entry=62'54 (59'48) modify 1:3a5a3f5b:::smithi13140796-23:head by unknown.0.0:0 2019-12-07T19:34:39.615350+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] rollforward: entry=62'55 (60'49) modify 1:33f5bf01:::smithi13140796-2:25 by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 snaps [1b,21,25] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] rollforward: entry=62'56 (60'52) modify 1:33f5bf01:::smithi13140796-2:head by unknown.0.0:0 2019-12-07T19:34:39.618495+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log approx pg log length = 56 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.624+0000 7f8f06986700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.624+0000 7f8f06986700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:39.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c58749b00 seq 75 2019-12-07T19:34:39.624+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:39.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:39.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:39.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58749b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:39.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58749b00 onodes shared_blobs 2019-12-07T19:34:39.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58749b00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749b00 prepare 2019-12-07T19:34:39.624+0000 7f8f06986700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58749b00 2019-12-07T19:34:39.624+0000 7f8f06986700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749b00 io_done 2019-12-07T19:34:39.624+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c5511e2c0 finish 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=62) prio 5 cost 1048576 e62) pg 0x558c57676000 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] snap_trimmer posting 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 20 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53a240~471 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53a240 0x53a240~471 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) get_omap_iterator has_omap = 1 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50a240 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x240 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6b1 and padding block with 0x94f 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c5856dd80) upper_bound after SNA_1_0000000000000020_.0_0000000000000001.C key 0x00000000000000000000000000000402'.SNA_1_0000000000000020_.0_0000000000000001.C' 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore.OmapIteratorImpl(0x558c5856dd80) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..' 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000020_.0_0000000000000001.C) returns 0 SNA_1_0000000000000021_.0_0000000000000001.C5A5CFAD.23.smithi13140796-23..,buffer::list(len=70, buffer::ptr(0~70 0x558c55239600 in raw 0x558c55239600 len 70 nref 1) ) 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim trimq=[20~1] ps=[23~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim ps=[23~1]] SnapTrimmer state: adding snap 20 to purged_snaps 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim ps=[20~1,23~1]] SnapTrimmer state: purged_snaps now [20~1,23~1], snap_trimq now [] 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38000 1.cs0_head 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e000 = 0x558c585e4f00 seq 76 2019-12-07T19:34:39.624+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._biginfo' <- _biginfo 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000456'._info' <- _info 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53a6b1 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c585e4f00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c585e4f00 onodes shared_blobs 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c585e4f00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4f00 prepare 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c585e4f00 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4f00 io_done 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000134558s (0.000000081s flush + 0.000134477s kv commit) 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58749b00> 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749b00 kv_submitted 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58749b00 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim ps=[20~1,23~1]] share_pg_info 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58749b00 finishing 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58749b00 onodes 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58749b00 done 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4f00 kv_queued 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58749b00 [] 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5756f680 had 1 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e62/62) v1 -- 0x558c5509f100 con 0x558c54c64000 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53a6b1~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53a6b1 0x53a6b1~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50a6b1 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6b1 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb1e and padding block with 0x4e2 2019-12-07T19:34:39.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e62/62) v1 -- 0x558c5509f800 con 0x558c54c19800 2019-12-07T19:34:39.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e62/62) v1 -- 0x558c58536e00 con 0x558c54c5bc00 2019-12-07T19:34:39.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e62/62) v1 -- 0x558c58537500 con 0x558c54c5b400 2019-12-07T19:34:39.624+0000 7f8f0297e700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e62/62) v1 -- 0x558c57a52000 con 0x558c54c19c00 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim ps=[20~1,23~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean+snaptrim ps=[20~1,23~1]] exit Trimming 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:39.624+0000 7f8f0297e700 15 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:216 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] enter NotTrimming 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] snap_trimmer complete 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f23c56700 0 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 msgr2=0x558c54c6fb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53ab1e 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.624+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 333 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58513e00 con 0x558c54c5b400 2019-12-07T19:34:39.624+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c5511eb00 prio 196 cost 0 latency 0.000013 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000221680s (0.000000091s flush + 0.000221589s kv commit) 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.624+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.624+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c585e4f00> 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4f00 kv_submitted 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c585e4f00 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c585e4f00 finishing 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c585e4f00 onodes 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c585e4f00 done 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e000 q now empty 2019-12-07T19:34:39.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 msgr2=0x558c54c6fb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 62 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c585e4f00 [] 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 msgr2=0x558c54c6fb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:39.624+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=READY pgs=26 cs=2 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:39.624+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 124 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54d76000 con 0x558c54c19c00 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5511eb00 prio 196 cost 0 latency 0.000040 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:39.624+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c58755760 prio 196 cost 0 latency 0.000010 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5511eb00 finish 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.624+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=READY pgs=26 cs=2 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:39.624+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c58755760 prio 196 cost 0 latency 0.000027 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.624+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c58755760 finish 2019-12-07T19:34:39.624+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.624+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c58bb0400 0x558c54c70680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:39.624+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> conn(0x558c58bb0400 0x558c54c70680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:39.624+0000 7f8f24c58700 10 osd.7 62 ms_handle_authentication new session 0x558c54c5c680 con 0x558c58bb0400 entity osd.4 addr 2019-12-07T19:34:39.624+0000 7f8f24c58700 10 osd.7 62 ms_handle_authentication session 0x558c54c5c680 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:39.624+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c58bb0400 0x558c54c70680 crc :-1 s=SESSION_ACCEPTING pgs=32 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558c54c5b400 2019-12-07T19:34:39.624+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c58bb0400 0x558c54c70680 crc :-1 s=SESSION_ACCEPTING pgs=32 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:34:39.624+0000 7f8f1199c700 2 osd.7 62 ms_handle_reset con 0x558c58bb0400 session 0x558c54c5c680 2019-12-07T19:34:39.624+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558c54c5b400 0x558c54c6fb80 unknown :-1 s=READY pgs=32 cs=3 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=f8982639b78e8be6 server_cookie=84aece1fce04073f in_seq=333 out_seq=344 2019-12-07T19:34:39.625+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 538 ==== MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=307, reqid=osd.1.0:304, at_version=0'0, trim_to=0'0, roll_forward_to=62'12)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c5822e000 con 0x558c54c64000 2019-12-07T19:34:39.625+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c587558c0 prio 127 cost 0 latency 0.000020 epoch 62 MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=307, reqid=osd.1.0:304, at_version=0'0, trim_to=0'0, roll_forward_to=62'12)) v2 2019-12-07T19:34:39.625+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=307, reqid=osd.1.0:304, at_version=0'0, trim_to=0'0, roll_forward_to=62'12)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=307, reqid=osd.1.0:304, at_version=0'0, trim_to=0'0, roll_forward_to=62'12)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.625+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=307, reqid=osd.1.0:304, at_version=0'0, trim_to=0'0, roll_forward_to=62'12)) v2) prio 127 cost 0 e62) pg 0x558c54d98000 2019-12-07T19:34:39.625+0000 7f8f0197c700 10 osd.7 62 dequeue_op 0x558c587558c0 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=307, reqid=osd.1.0:304, at_version=0'0, trim_to=0'0, roll_forward_to=62'12)) v2 pg pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.es4 62/41 ECSubWrite(tid=307, reqid=osd.1.0:304, at_version=0'0, trim_to=0'0, roll_forward_to=62'12)) v2 2019-12-07T19:34:39.625+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=59'10 active mbc={} ps=[23~1]] append_log log((0'0,62'12], crt=59'10) [] 2019-12-07T19:34:39.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[23~1]] rollforward: entry=62'11 (59'9) modify 1:733d7474:::smithi13140796-5:23 by unknown.0.0:0 2019-12-07T19:34:39.616529+0000 0 snaps [1b,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.625+0000 7f8f0197c700 20 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[23~1]] rollforward: entry=62'12 (59'10) modify 1:733d7474:::smithi13140796-5:head by unknown.0.0:0 2019-12-07T19:34:39.616529+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[23~1]] append_log approx pg log length = 12 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.626+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c550a3200 seq 32 2019-12-07T19:34:39.626+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a3200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a3200 onodes shared_blobs 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a3200 allocated 0x[] released 0x[] 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3200 prepare 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a3200 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3200 io_done 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 osd.7 62 dequeue_op 0x558c587558c0 finish 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53ab1e~471 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53ab1e 0x53ab1e~471 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50ab1e 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb1e 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf8f and padding block with 0x71 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53af8f 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000127992s (0.000000067s flush + 0.000127925s kv commit) 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c550a3200> 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3200 kv_submitted 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a3200 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3200 finishing 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a3200 onodes 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a3200 done 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a3200 [] 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 62/41 ECSubWriteReply(tid=307, last_complete=62'12, committed=1, applied=1)) v2 -- 0x558c58513c00 con 0x558c54c64000 2019-12-07T19:34:39.626+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 539 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f600 con 0x558c54c64000 2019-12-07T19:34:39.626+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c58755a20 prio 196 cost 0 latency 0.000013 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.626+0000 7f8f24457700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.626+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 540 ==== pg_info2(1.es4 1.es4 1.es0( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c64000 2019-12-07T19:34:39.626+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c58755a20 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:39.626+0000 7f8f24457700 15 osd.7 62 enqueue_peering_evt 1.es4 epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.es0( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s) 2019-12-07T19:34:39.626+0000 7f8f24457700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.es0( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.626+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.626+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c58755a20 finish 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.es0( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.es0( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s)) prio 255 cost 10 e62) pg 0x558c54d98000 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.es0( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068542267s ub 75.068542267s int 16s) 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:39.626+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 220 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59a00 con 0x558c54c19800 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c57f10000 1.es4_head 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57895b80 = 0x558c550a3500 seq 33 2019-12-07T19:34:39.626+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c5740c000 prio 196 cost 0 latency 0.000009 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.626+0000 7f8f24c58700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.626+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._biginfo' <- _biginfo 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049d'._info' <- _info 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a3500 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a3500 onodes shared_blobs 2019-12-07T19:34:39.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a3500 allocated 0x[] released 0x[] 2019-12-07T19:34:39.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3500 prepare 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a3500 2019-12-07T19:34:39.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.626+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3500 io_done 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.626+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5740c000 prio 196 cost 0 latency 0.000032 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:39.626+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.626+0000 7f8f0297e700 10 osd.7 62 dequeue_op 0x558c5740c000 finish 2019-12-07T19:34:39.626+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12a00 had 1 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53af8f~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53af8f 0x53af8f~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50af8f 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf8f 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3fc and padding block with 0xc04 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53b3fc 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.626+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000122009s (0.000000100s flush + 0.000121909s kv commit) 2019-12-07T19:34:39.626+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c550a3500> 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3500 kv_submitted 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a3500 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3500 finishing 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a3500 onodes 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a3500 done 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57895b80 q now empty 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a3500 [] 2019-12-07T19:34:39.626+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.626+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.626+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:39.626+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 318 ==== MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54c59200 con 0x558c54c5bc00 2019-12-07T19:34:39.626+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c5511ec60 prio 196 cost 0 latency 0.000012 epoch 62 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.626+0000 7f8f23c56700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) queued 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2) prio 196 cost 0 e62) pg 0x558c57676000 2019-12-07T19:34:39.626+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c5511ec60 prio 196 cost 0 latency 0.000030 MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 pg pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:39.626+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.cs0 62/28 ECSubWriteReply(tid=61, last_complete=62'56, committed=1, applied=1)) v2 2019-12-07T19:34:39.626+0000 7f8f06986700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:25 v=0'0 tt=0'0 tid=61 reqid=osd.7.0:60 roll_forward_to=62'56 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:39.626+0000 7f8f06986700 10 osd.7 62 dequeue_op 0x558c5511ec60 finish 2019-12-07T19:34:39.626+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:39.627+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 221 ==== MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=80, reqid=osd.2.0:79, at_version=0'0, trim_to=0'0, roll_forward_to=62'32)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5822f000 con 0x558c54c19800 2019-12-07T19:34:39.627+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c5740de40 prio 127 cost 0 latency 0.000020 epoch 62 MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=80, reqid=osd.2.0:79, at_version=0'0, trim_to=0'0, roll_forward_to=62'32)) v2 2019-12-07T19:34:39.627+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=80, reqid=osd.2.0:79, at_version=0'0, trim_to=0'0, roll_forward_to=62'32)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 541 ==== pg_info2(1.7s4 1.7s4 1.7s0( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c576b8700 con 0x558c54c64000 2019-12-07T19:34:39.627+0000 7f8f24457700 15 osd.7 62 enqueue_peering_evt 1.7s4 epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.7s0( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s) 2019-12-07T19:34:39.627+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.7s0( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.627+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 542 ==== MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=308, reqid=osd.1.0:306, at_version=0'0, trim_to=0'0, roll_forward_to=62'43)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57dff800 con 0x558c54c64000 2019-12-07T19:34:39.627+0000 7f8f24457700 15 osd.7 62 enqueue_op 0x558c58755b80 prio 127 cost 0 latency 0.000013 epoch 62 MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=308, reqid=osd.1.0:306, at_version=0'0, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:39.627+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=308, reqid=osd.1.0:306, at_version=0'0, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.7s0( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.7s0( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s)) prio 255 cost 10 e62) pg 0x558c54d5f000 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 1(0) info: 1.7s0( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.068898077s ub 75.068898077s int 16s) 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5875d800 seq 92 2019-12-07T19:34:39.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875d800 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875d800 onodes shared_blobs 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875d800 allocated 0x[] released 0x[] 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d800 prepare 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875d800 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d800 io_done 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=308, reqid=osd.1.0:306, at_version=0'0, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=308, reqid=osd.1.0:306, at_version=0'0, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e62) pg 0x558c54d5f000 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 62 dequeue_op 0x558c58755b80 prio 127 cost 0 latency 0.000140 MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=308, reqid=osd.1.0:306, at_version=0'0, trim_to=0'0, roll_forward_to=62'43)) v2 pg pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 62/41 ECSubWrite(tid=308, reqid=osd.1.0:306, at_version=0'0, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=57'41 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'43], crt=57'41) [] 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'42 (57'40) modify 1:ec1285ab:::smithi13140796-19:20 by unknown.0.0:0 2019-12-07T19:34:39.621425+0000 0 snaps [1b] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'43 (57'41) modify 1:ec1285ab:::smithi13140796-19:head by unknown.0.0:0 2019-12-07T19:34:39.621425+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 43 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.627+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c5875db00 seq 93 2019-12-07T19:34:39.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:39.627+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875db00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875db00 onodes shared_blobs 2019-12-07T19:34:39.627+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875db00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.627+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53b3fc~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875db00 prepare 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875db00 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875db00 io_done 2019-12-07T19:34:39.627+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53b3fc 0x53b3fc~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.627+0000 7f8f0117b700 10 osd.7 62 dequeue_op 0x558c58755b80 finish 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.627+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50b3fc 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3fc 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.627+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x869 and padding block with 0x797 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53b869 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=80, reqid=osd.2.0:79, at_version=0'0, trim_to=0'0, roll_forward_to=62'32)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGOpItem(op=MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=80, reqid=osd.2.0:79, at_version=0'0, trim_to=0'0, roll_forward_to=62'32)) v2) prio 127 cost 0 e62) pg 0x558c55141000 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000136960s (0.000000095s flush + 0.000136865s kv commit) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 osd.7 62 dequeue_op 0x558c5740de40 prio 127 cost 0 latency 0.000554 MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=80, reqid=osd.2.0:79, at_version=0'0, trim_to=0'0, roll_forward_to=62'32)) v2 pg pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] _handle_message: MOSDECSubOpWrite(1.ds2 62/41 ECSubWrite(tid=80, reqid=osd.2.0:79, at_version=0'0, trim_to=0'0, roll_forward_to=62'32)) v2 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=57'30 active mbc={} ps=[23~1]] append_log log((0'0,62'32], crt=57'30) [] 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53b869~471 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53b869 0x53b869~471 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50b869 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x869 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcda and padding block with 0x326 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[23~1]] rollforward: entry=62'31 (55'25) delete 1:bca18876:::smithi13140796-30:20 by unknown.0.0:0 2019-12-07T19:34:39.621745+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[23~1]] rollforward: entry=62'32 (57'30) modify 1:bca18876:::smithi13140796-30:head by unknown.0.0:0 2019-12-07T19:34:39.621745+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[23~1]] append_log approx pg log length = 32 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875d800> 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d800 kv_submitted 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875d800 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875d800 finishing 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875d800 onodes 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875d800 done 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875db00 kv_submitted 2019-12-07T19:34:39.628+0000 7f8f0217d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875d800 [] 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:39.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c58756c00 seq 70 2019-12-07T19:34:39.628+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.628+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bca18876:::smithi13140796-30:20#1f onode 0x558c54e12c80 txc 0x558c58756c00 2019-12-07T19:34:39.628+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.ds2_head 2#1:bca18876:::smithi13140796-30:20#1f 0x0 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3e30) put_ref 0x0~2000 Blob(0x558c574e3e30 blob([0xae6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53bcda 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a930) put_ref 0x0~2000 Blob(0x558c5898a930 blob([0xae8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c585d5d50) put_ref 0x0~2000 Blob(0x558c585d5d50 blob([0xaea000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3d50) put_ref 0x0~2000 Blob(0x558c574e3d50 blob([0xaec000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a850) put_ref 0x0~2000 Blob(0x558c5898a850 blob([0xaee000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000126376s (0.000000109s flush + 0.000126267s kv commit) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5809b810) put_ref 0x0~2000 Blob(0x558c5809b810 blob([0xaf0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a8c0) put_ref 0x0~2000 Blob(0x558c5898a8c0 blob([0xaf2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3c70) put_ref 0x0~2000 Blob(0x558c574e3c70 blob([0xaf4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a7e0) put_ref 0x0~2000 Blob(0x558c5898a7e0 blob([0xaf6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3b90) put_ref 0x0~2000 Blob(0x558c574e3b90 blob([0xaf8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a770) put_ref 0x0~2000 Blob(0x558c5898a770 blob([0xafa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3ab0) put_ref 0x0~2000 Blob(0x558c574e3ab0 blob([0xafc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a700) put_ref 0x0~2000 Blob(0x558c5898a700 blob([0xafe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e39d0) put_ref 0x0~2000 Blob(0x558c574e39d0 blob([0xb00000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a690) put_ref 0x0~2000 Blob(0x558c5898a690 blob([0xb02000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e38f0) put_ref 0x0~2000 Blob(0x558c574e38f0 blob([0xb04000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a620) put_ref 0x0~2000 Blob(0x558c5898a620 blob([0xb06000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875db00> 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875db00 kv_submitted 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875db00 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3810) put_ref 0x0~2000 Blob(0x558c574e3810 blob([0xb08000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875db00 finishing 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875db00 onodes 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a5b0) put_ref 0x0~2000 Blob(0x558c5898a5b0 blob([0xb0a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875db00 done 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875db00 [] 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3730) put_ref 0x0~2000 Blob(0x558c574e3730 blob([0xb0c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a540) put_ref 0x0~2000 Blob(0x558c5898a540 blob([0xb0e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e33b0) put_ref 0x0~2000 Blob(0x558c574e33b0 blob([0xb10000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.628+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 62/41 ECSubWriteReply(tid=308, last_complete=62'43, committed=1, applied=1)) v2 -- 0x558c54d76400 con 0x558c54c64000 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a4d0) put_ref 0x0~2000 Blob(0x558c5898a4d0 blob([0xb12000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3500) put_ref 0x0~2000 Blob(0x558c574e3500 blob([0xb14000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a460) put_ref 0x0~2000 Blob(0x558c5898a460 blob([0xb16000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e35e0) put_ref 0x0~2000 Blob(0x558c574e35e0 blob([0xb18000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c578de000) put_ref 0x0~2000 Blob(0x558c578de000 blob([0xb1a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e36c0) put_ref 0x0~2000 Blob(0x558c574e36c0 blob([0xb1c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5809acb0) put_ref 0x0~2000 Blob(0x558c5809acb0 blob([0xb1e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c574e3340) put_ref 0x0~2000 Blob(0x558c574e3340 blob([0xb20000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c57583b20) put_ref 0x0~2000 Blob(0x558c57583b20 blob([0xb22000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898a9a0) put_ref 0x0~2000 Blob(0x558c5898a9a0 blob([0xb24000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aa10) put_ref 0x0~2000 Blob(0x558c5898aa10 blob([0xb26000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aa80) put_ref 0x0~2000 Blob(0x558c5898aa80 blob([0xb28000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898aaf0) put_ref 0x0~2000 Blob(0x558c5898aaf0 blob([0xb2a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ab60) put_ref 0x0~2000 Blob(0x558c5898ab60 blob([0xb2c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898abd0) put_ref 0x0~2000 Blob(0x558c5898abd0 blob([0xb2e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ac40) put_ref 0x0~2000 Blob(0x558c5898ac40 blob([0xb30000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898acb0) put_ref 0x0~2000 Blob(0x558c5898acb0 blob([0xb32000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ad20) put_ref 0x0~2000 Blob(0x558c5898ad20 blob([0xb34000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c5898ad90) put_ref 0x0~2000 Blob(0x558c5898ad90 blob([0xb36000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6000) put_ref 0x0~2000 Blob(0x558c579d6000 blob([0xb38000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6070) put_ref 0x0~2000 Blob(0x558c579d6070 blob([0xb3a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c579d60e0) put_ref 0x0~2000 Blob(0x558c579d60e0 blob([0xb3c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6150) put_ref 0x0~2000 Blob(0x558c579d6150 blob([0xb3e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c579d61c0) put_ref 0x0~2000 Blob(0x558c579d61c0 blob([0xb40000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.blob(0x558c579d6230) put_ref 0x0~2000 Blob(0x558c579d6230 blob([0xb42000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dirty_range mark shard 0x24000 dirty 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore.extentmap(0x558c54e12d88) dirty_range mark shard 0x48000 dirty 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c574e3e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xae6000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c5898a930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xae8000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c585d5d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaea000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c574e3d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaec000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c5898a850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaee000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c5809b810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf0000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c5898a8c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf2000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c574e3c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf4000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5898a7e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf6000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c574e3b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaf8000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c5898a770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xafa000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c574e3ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xafc000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c5898a700 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xafe000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c574e39d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb00000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c5898a690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb02000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c574e38f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb04000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c5898a620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb06000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c574e3810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb08000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c5898a5b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb0a000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c574e3730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb0c000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c5898a540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb0e000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c574e33b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb10000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c5898a4d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb12000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c574e3500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb14000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c5898a460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb16000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c574e35e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb18000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c578de000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb1a000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~2000: 0x0~2000 Blob(0x558c574e36c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb1c000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x38000~2000: 0x0~2000 Blob(0x558c5809acb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb1e000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3a000~2000: 0x0~2000 Blob(0x558c574e3340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb20000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3c000~2000: 0x0~2000 Blob(0x558c57583b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb22000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3e000~2000: 0x0~2000 Blob(0x558c5898a9a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb24000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x40000~2000: 0x0~2000 Blob(0x558c5898aa10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb26000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x42000~2000: 0x0~2000 Blob(0x558c5898aa80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb28000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x44000~2000: 0x0~2000 Blob(0x558c5898aaf0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb2a000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x46000~2000: 0x0~2000 Blob(0x558c5898ab60 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb2c000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x48000~2000: 0x0~2000 Blob(0x558c5898abd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb2e000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4a000~2000: 0x0~2000 Blob(0x558c5898ac40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb30000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~2000: 0x0~2000 Blob(0x558c5898acb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb32000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4e000~2000: 0x0~2000 Blob(0x558c5898ad20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb34000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x50000~2000: 0x0~2000 Blob(0x558c5898ad90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb36000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x52000~2000: 0x0~2000 Blob(0x558c579d6000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb38000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x54000~2000: 0x0~2000 Blob(0x558c579d6070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb3a000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x56000~2000: 0x0~2000 Blob(0x558c579d60e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb3c000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x58000~2000: 0x0~2000 Blob(0x558c579d6150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb3e000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5a000~2000: 0x0~2000 Blob(0x558c579d61c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb40000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5c000~2000: 0x0~2000 Blob(0x558c579d6230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=2)))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xb42000~2000] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x24000 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x48000 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c54cf8230,0x558c550604d0,0x558c574d2cb0,0x558c574fd9d0,0x558c578f00e0,0x558c578f0150,0x558c578f1110,0x558c578f1180,0x558c578f1340,0x558c5794d9d0,0x558c57a06bd0,0x558c57a07f80,0x558c57bea540,0x558c57beac40,0x558c57db5500,0x558c57db5570,0x558c57db5960,0x558c57e63d50,0x558c57f512d0,0x558c57f51340,0x558c57f51810,0x558c57ff4310,0x558c5809b180,0x558c5809b3b0,0x558c58c67c00,0x558c58c67ce0,0x558c58c67dc0,0x558c58c67ea0,0x558c58c67f80,0x558c58c78070,0x558c58c78150,0x558c58c78230,0x558c58c78310,0x558c58c783f0,0x558c58c784d0,0x558c58c785b0,0x558c58c78690,0x558c58c78770,0x558c58c78850,0x558c58c78930,0x558c58c78a10,0x558c58c78af0,0x558c58c78bd0,0x558c58c78cb0,0x558c58c78d90,0x558c58c78e70,0x558c58c78f50 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.ds2_head 2#1:bca18876:::smithi13140796-30:20#1f = 0 2019-12-07T19:34:39.628+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58756c00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58756c00 onodes shared_blobs 0x558c54cf8230,0x558c550604d0,0x558c574d2cb0,0x558c574fd9d0,0x558c578f00e0,0x558c578f0150,0x558c578f1110,0x558c578f1180,0x558c578f1340,0x558c5794d9d0,0x558c57a06bd0,0x558c57a07f80,0x558c57bea540,0x558c57beac40,0x558c57db5500,0x558c57db5570,0x558c57db5960,0x558c57e63d50,0x558c57f512d0,0x558c57f51340,0x558c57f51810,0x558c57ff4310,0x558c5809b180,0x558c5809b3b0,0x558c58c67c00,0x558c58c67ce0,0x558c58c67dc0,0x558c58c67ea0,0x558c58c67f80,0x558c58c78070,0x558c58c78150,0x558c58c78230,0x558c58c78310,0x558c58c783f0,0x558c58c784d0,0x558c58c785b0,0x558c58c78690,0x558c58c78770,0x558c58c78850,0x558c58c78930,0x558c58c78a10,0x558c58c78af0,0x558c58c78bd0,0x558c58c78cb0,0x558c58c78d90,0x558c58c78e70,0x558c58c78f50 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e8 is 11 SharedBlob(0x558c54cf8230 loaded (sbid 0x29e8 ref_map(0xaf8000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ea is 11 SharedBlob(0x558c550604d0 loaded (sbid 0x29ea ref_map(0xafc000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f2 is 11 SharedBlob(0x558c574d2cb0 loaded (sbid 0x29f2 ref_map(0xb0c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e6 is 11 SharedBlob(0x558c574fd9d0 loaded (sbid 0x29e6 ref_map(0xaf4000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e2 is 11 SharedBlob(0x558c578f00e0 loaded (sbid 0x29e2 ref_map(0xaec000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e1 is 11 SharedBlob(0x558c578f0150 loaded (sbid 0x29e1 ref_map(0xaea000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e0 is 11 SharedBlob(0x558c578f1110 loaded (sbid 0x29e0 ref_map(0xae8000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e4 is 11 SharedBlob(0x558c578f1180 loaded (sbid 0x29e4 ref_map(0xaf0000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e7 is 11 SharedBlob(0x558c578f1340 loaded (sbid 0x29e7 ref_map(0xaf6000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ed is 11 SharedBlob(0x558c5794d9d0 loaded (sbid 0x29ed ref_map(0xb02000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 222 ==== pg_info2(1.ds2 1.ds2 1.ds0( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) e62/62) v1 ==== 1076+0+0 (crc 0 0 0) 0x558c5509f800 con 0x558c54c19800 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ee is 11 SharedBlob(0x558c57a06bd0 loaded (sbid 0x29ee ref_map(0xb04000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f3 is 11 SharedBlob(0x558c57a07f80 loaded (sbid 0x29f3 ref_map(0xb0e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f0 is 11 SharedBlob(0x558c57bea540 loaded (sbid 0x29f0 ref_map(0xb08000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f5 is 11 SharedBlob(0x558c57beac40 loaded (sbid 0x29f5 ref_map(0xb12000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f1 is 11 SharedBlob(0x558c57db5500 loaded (sbid 0x29f1 ref_map(0xb0a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ef is 11 SharedBlob(0x558c57db5570 loaded (sbid 0x29ef ref_map(0xb06000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29eb is 11 SharedBlob(0x558c57db5960 loaded (sbid 0x29eb ref_map(0xafe000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f24c58700 15 osd.7 62 enqueue_peering_evt 1.ds2 epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.ds0( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e5 is 11 SharedBlob(0x558c57e63d50 loaded (sbid 0x29e5 ref_map(0xaf2000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e3 is 11 SharedBlob(0x558c57f512d0 loaded (sbid 0x29e3 ref_map(0xaee000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f24c58700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.ds0( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e62) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29df is 11 SharedBlob(0x558c57f51340 loaded (sbid 0x29df ref_map(0xae6000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29e9 is 11 SharedBlob(0x558c57f51810 loaded (sbid 0x29e9 ref_map(0xafa000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ec is 11 SharedBlob(0x558c57ff4310 loaded (sbid 0x29ec ref_map(0xb00000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f6 is 11 SharedBlob(0x558c5809b180 loaded (sbid 0x29f6 ref_map(0xb14000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f4 is 11 SharedBlob(0x558c5809b3b0 loaded (sbid 0x29f4 ref_map(0xb10000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f7 is 11 SharedBlob(0x558c58c67c00 loaded (sbid 0x29f7 ref_map(0xb16000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f8 is 11 SharedBlob(0x558c58c67ce0 loaded (sbid 0x29f8 ref_map(0xb18000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.ds0( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e62) queued 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29f9 is 11 SharedBlob(0x558c58c67dc0 loaded (sbid 0x29f9 ref_map(0xb1a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fa is 11 SharedBlob(0x558c58c67ea0 loaded (sbid 0x29fa ref_map(0xb1c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fb is 11 SharedBlob(0x558c58c67f80 loaded (sbid 0x29fb ref_map(0xb1e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fc is 11 SharedBlob(0x558c58c78070 loaded (sbid 0x29fc ref_map(0xb20000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fd is 11 SharedBlob(0x558c58c78150 loaded (sbid 0x29fd ref_map(0xb22000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29fe is 11 SharedBlob(0x558c58c78230 loaded (sbid 0x29fe ref_map(0xb24000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x29ff is 11 SharedBlob(0x558c58c78310 loaded (sbid 0x29ff ref_map(0xb26000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a00 is 11 SharedBlob(0x558c58c783f0 loaded (sbid 0x2a00 ref_map(0xb28000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a01 is 11 SharedBlob(0x558c58c784d0 loaded (sbid 0x2a01 ref_map(0xb2a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a02 is 11 SharedBlob(0x558c58c785b0 loaded (sbid 0x2a02 ref_map(0xb2c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a03 is 11 SharedBlob(0x558c58c78690 loaded (sbid 0x2a03 ref_map(0xb2e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a04 is 11 SharedBlob(0x558c58c78770 loaded (sbid 0x2a04 ref_map(0xb30000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a05 is 11 SharedBlob(0x558c58c78850 loaded (sbid 0x2a05 ref_map(0xb32000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a06 is 11 SharedBlob(0x558c58c78930 loaded (sbid 0x2a06 ref_map(0xb34000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a07 is 11 SharedBlob(0x558c58c78a10 loaded (sbid 0x2a07 ref_map(0xb36000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a08 is 11 SharedBlob(0x558c58c78af0 loaded (sbid 0x2a08 ref_map(0xb38000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a09 is 11 SharedBlob(0x558c58c78bd0 loaded (sbid 0x2a09 ref_map(0xb3a000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0a is 11 SharedBlob(0x558c58c78cb0 loaded (sbid 0x2a0a ref_map(0xb3c000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0b is 11 SharedBlob(0x558c58c78d90 loaded (sbid 0x2a0b ref_map(0xb3e000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0c is 11 SharedBlob(0x558c58c78e70 loaded (sbid 0x2a0c ref_map(0xb40000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0d is 11 SharedBlob(0x558c58c78f50 loaded (sbid 0x2a0d ref_map(0xb42000~2000=1))) 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58756c00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756c00 prepare 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58756c00 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756c00 io_done 2019-12-07T19:34:39.628+0000 7f8f0217d700 10 osd.7 62 dequeue_op 0x558c5740de40 finish 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2526 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.ds0( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s)) prio 255 cost 10 e62) pg 0x558c55141000 2019-12-07T19:34:39.628+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[23~1]] do_peering_event: epoch_sent: 62 epoch_requested: 62 MInfoRec from 2(0) info: 1.ds0( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) pg_lease(ru 75.082846331s ub 75.082846331s int 16s) 2019-12-07T19:34:39.628+0000 7f8f06185700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[23~1]] proc_primary_info updating purged_snaps to [20~1,23~1] 2019-12-07T19:34:39.628+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:39.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d0f4a0 1.ds2_head 2019-12-07T19:34:39.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c5525e280 = 0x558c5853b800 seq 71 2019-12-07T19:34:39.628+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# 2019-12-07T19:34:39.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:39.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e12c80 had 1 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:39.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.ds2_head 2#1:b0000000::::head# = 0 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2549 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2549 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5853b800 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:39.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5853b800 onodes shared_blobs 2019-12-07T19:34:39.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5853b800 allocated 0x[] released 0x[] 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53bcda~9f5 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53bcda 0x53bcda~9f5 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b800 prepare 2019-12-07T19:34:39.628+0000 7f8f06185700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5853b800 2019-12-07T19:34:39.628+0000 7f8f06185700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b800 io_done 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50bcda 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcda 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6cf and padding block with 0x931 2019-12-07T19:34:39.628+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53c6cf 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000178173s (0.000000119s flush + 0.000178054s kv commit) 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58756c00> 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756c00 kv_submitted 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58756c00 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58756c00 finishing 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58756c00 onodes 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58756c00 done 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853b800 kv_submitted 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148280 had 1 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58756c00 [] 2019-12-07T19:34:39.628+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53c6cf~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.628+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53c6cf 0x53c6cf~46d to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.628+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50c6cf 2019-12-07T19:34:39.628+0000 7f8f06185700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 62/41 ECSubWriteReply(tid=80, last_complete=62'32, committed=1, applied=1)) v2 -- 0x558c53a5e600 con 0x558c54c19800 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6cf 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb3c and padding block with 0x4c4 2019-12-07T19:34:39.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53cb3c 2019-12-07T19:34:39.629+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.629+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000118109s (0.000000081s flush + 0.000118028s kv commit) 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.629+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5853b800> 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b800 kv_submitted 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5853b800 2019-12-07T19:34:39.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5853b800 finishing 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5853b800 onodes 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5853b800 done 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c5525e280 q now empty 2019-12-07T19:34:39.629+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5853b800 [] 2019-12-07T19:34:39.629+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.629+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.629+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.629+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:39.632+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 334 ==== MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=122, reqid=client.4540.0:457, at_version=62'52, trim_to=0'0, roll_forward_to=62'50)) v2 ==== 183698+0+0 (unknown 0 0 0) 0x558c57a42800 con 0x558c54c5b400 2019-12-07T19:34:39.632+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c54e78840 prio 127 cost 0 latency 0.000036 epoch 62 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=122, reqid=client.4540.0:457, at_version=62'52, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.632+0000 7f8f24c58700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=122, reqid=client.4540.0:457, at_version=62'52, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=122, reqid=client.4540.0:457, at_version=62'52, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.632+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=122, reqid=client.4540.0:457, at_version=62'52, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.632+0000 7f8f07187700 10 osd.7 62 dequeue_op 0x558c54e78840 prio 127 cost 0 latency 0.000089 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=122, reqid=client.4540.0:457, at_version=62'52, trim_to=0'0, roll_forward_to=62'50)) v2 pg pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.632+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=122, reqid=client.4540.0:457, at_version=62'52, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.632+0000 7f8f07187700 20 snap_mapper.add_oid 1:d07269f7:::smithi13140796-28:26 25,26 2019-12-07T19:34:39.632+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.632+0000 7f8f07187700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:39.632+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.632+0000 7f8f07187700 20 snap_mapper.get_snaps 1:d07269f7:::smithi13140796-28:26 got.empty() 2019-12-07T19:34:39.632+0000 7f8f07187700 20 snap_mapper.set_snaps 1:d07269f7:::smithi13140796-28:26 25,26 2019-12-07T19:34:39.632+0000 7f8f07187700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.B0E469FE.26.smithi13140796-28.. 2019-12-07T19:34:39.632+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_0000000000000025_.1_0000000000000001.B0E469FE.26.smithi13140796-28.. 2019-12-07T19:34:39.632+0000 7f8f07187700 20 snap_mapper.add_oid set SNA_1_0000000000000026_.1_0000000000000001.B0E469FE.26.smithi13140796-28.. 2019-12-07T19:34:39.632+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'50 (0'0,62'50] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'50], crt=62'50) [62'51 (62'50) clone 1:d07269f7:::smithi13140796-28:26 by unknown.0.0:0 2019-12-07T19:34:35.633784+0000 0 snaps [26,25] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,62'52 (62'50) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:457 2019-12-07T19:34:39.631098+0000 0 ObjectCleanRegions clean_offsets: [0~2818048,3538944~18446744073706012671], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.632+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'51 (0'0,62'51] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'50 crt=62'50 lcod 62'50 active mbc={} ps=[20~1,23~1]] add_log_entry 62'51 (62'50) clone 1:d07269f7:::smithi13140796-28:26 by unknown.0.0:0 2019-12-07T19:34:35.633784+0000 0 snaps [26,25] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.633+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'52 (0'0,62'52] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'50 crt=62'50 lcod 62'50 active mbc={} ps=[20~1,23~1]] add_log_entry 62'52 (62'50) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:457 2019-12-07T19:34:39.631098+0000 0 ObjectCleanRegions clean_offsets: [0~2818048,3538944~18446744073706012671], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.633+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'52 (0'0,62'52] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'50 crt=62'50 lcod 62'50 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 52 2019-12-07T19:34:39.633+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'52 (0'0,62'52] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'50 crt=62'50 lcod 62'50 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.633+0000 7f8f07187700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.633+0000 7f8f07187700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'51, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58bb4000 seq 64 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) get_onode oid 1#1:d07269f7:::smithi13140796-28:26# key 0x818000000000000001d07269f721736d'ithi13140796-28!='0x0000000000000026ffffffffffffffff'o' 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) r -2 v.len 0 2019-12-07T19:34:39.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:26# 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1290 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.onode(0x558c54e44780).flush flush done 2019-12-07T19:34:39.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.bs1_head 1#1:d07269f7:::smithi13140796-28:26# 0x0 2019-12-07T19:34:39.633+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:26# 0x0~ac000 -> 0x0~ac000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x0~2000: 0x0~2000 Blob(0x558c5840f2d0 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58766af0 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766af0) get_ref 0x0~2000 Blob(0x558c58766af0 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766af0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x0~2000: 0x0~2000 Blob(0x558c58766af0 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2000~2000: 0x0~2000 Blob(0x558c583eac40 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58226230 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58226230) get_ref 0x0~2000 Blob(0x558c58226230 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58226230) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c58226230 blob([0x99b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4000~2000: 0x0~2000 Blob(0x558c5840f3b0 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c585e1260 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c585e1260) get_ref 0x0~2000 Blob(0x558c585e1260 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c585e1260) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c585e1260 blob([0x99d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x6000~2000: 0x0~2000 Blob(0x558c578f1570 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58766850 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766850) get_ref 0x0~2000 Blob(0x558c58766850 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766850) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c58766850 blob([0x99f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x8000~2000: 0x0~2000 Blob(0x558c550b2e00 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c5792cc40 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c5792cc40) get_ref 0x0~2000 Blob(0x558c5792cc40 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c5792cc40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5792cc40 blob([0x9a1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xa000~2000: 0x0~2000 Blob(0x558c583eb960 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58766b60 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766b60) get_ref 0x0~2000 Blob(0x558c58766b60 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766b60) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c58766b60 blob([0x9a3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57970fc0 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58766c40 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766c40) get_ref 0x0~2000 Blob(0x558c58766c40 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766c40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c58766c40 blob([0x9a5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xe000~2000: 0x0~2000 Blob(0x558c57970ee0 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58766d20 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766d20) get_ref 0x0~2000 Blob(0x558c58766d20 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766d20) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c58766d20 blob([0x9a7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x10000~2000: 0x0~2000 Blob(0x558c578199d0 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58766d90 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766d90) get_ref 0x0~2000 Blob(0x558c58766d90 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c58766d90) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c58766d90 blob([0x9a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x12000~2000: 0x0~2000 Blob(0x558c57970e00 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe000 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe000) get_ref 0x0~2000 Blob(0x558c587fe000 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe000) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c587fe000 blob([0x9ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x14000~2000: 0x0~2000 Blob(0x558c57818d20 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe070 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe070) get_ref 0x0~2000 Blob(0x558c587fe070 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe070) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c587fe070 blob([0x9ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x16000~2000: 0x0~2000 Blob(0x558c57970d20 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe0e0 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe0e0) get_ref 0x0~2000 Blob(0x558c587fe0e0 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c587fe0e0 blob([0x9af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x18000~2000: 0x0~2000 Blob(0x558c57818cb0 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe150 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe150) get_ref 0x0~2000 Blob(0x558c587fe150 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe150) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c587fe150 blob([0x9b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c57970c40 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe1c0 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe1c0) get_ref 0x0~2000 Blob(0x558c587fe1c0 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c587fe1c0 blob([0x9b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c57f50ee0 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe230 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe230) get_ref 0x0~2000 Blob(0x558c587fe230 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe230) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c587fe230 blob([0x9b5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c57970b60 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe2a0 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe2a0) get_ref 0x0~2000 Blob(0x558c587fe2a0 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c587fe2a0 blob([0x9b7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x20000~2000: 0x0~2000 Blob(0x558c57819a40 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe310 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe310) get_ref 0x0~2000 Blob(0x558c587fe310 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe310) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c587fe310 blob([0x9b9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x22000~1000: 0x0~1000 Blob(0x558c57970a80 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe380 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe380) get_ref 0x0~1000 Blob(0x558c587fe380 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe380) get_ref init 0x1000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x22000~1000: 0x0~1000 Blob(0x558c587fe380 blob([0x9bb000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x23000~2000: 0x0~2000 Blob(0x558c550b36c0 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe3f0 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe3f0) get_ref 0x0~2000 Blob(0x558c587fe3f0 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c587fe3f0 blob([0x9bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x25000~2000: 0x0~2000 Blob(0x558c579702a0 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe460 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe460) get_ref 0x0~2000 Blob(0x558c587fe460 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe460) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c587fe460 blob([0x9be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x27000~2000: 0x0~2000 Blob(0x558c57970230 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe4d0 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe4d0) get_ref 0x0~2000 Blob(0x558c587fe4d0 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c587fe4d0 blob([0x9c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x29000~2000: 0x0~2000 Blob(0x558c574d2fc0 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe540 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe540) get_ref 0x0~2000 Blob(0x558c587fe540 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe540) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c587fe540 blob([0x9c2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c578f1810 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe5b0 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe5b0) get_ref 0x0~2000 Blob(0x558c587fe5b0 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c587fe5b0 blob([0x9c4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c574fd570 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe620 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe620) get_ref 0x0~2000 Blob(0x558c587fe620 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe620) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c587fe620 blob([0x9c6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x2f000~2000: 0x0~2000 Blob(0x558c55106e00 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe690 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe690) get_ref 0x0~2000 Blob(0x558c587fe690 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe690) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x2f000~2000: 0x0~2000 Blob(0x558c587fe690 blob([0x9c8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x31000~2000: 0x0~2000 Blob(0x558c55106ee0 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe700 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe700) get_ref 0x0~2000 Blob(0x558c587fe700 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe700) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x31000~2000: 0x0~2000 Blob(0x558c587fe700 blob([0x9ca000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x33000~2000: 0x0~2000 Blob(0x558c55106fc0 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe770 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe770) get_ref 0x0~2000 Blob(0x558c587fe770 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe770) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x33000~2000: 0x0~2000 Blob(0x558c587fe770 blob([0x9cc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x35000~2000: 0x0~2000 Blob(0x558c551070a0 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe7e0 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe7e0) get_ref 0x0~2000 Blob(0x558c587fe7e0 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x35000~2000: 0x0~2000 Blob(0x558c587fe7e0 blob([0x9ce000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x37000~2000: 0x0~2000 Blob(0x558c55107180 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe850 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe850) get_ref 0x0~2000 Blob(0x558c587fe850 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe850) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x37000~2000: 0x0~2000 Blob(0x558c587fe850 blob([0x9d0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x39000~2000: 0x0~2000 Blob(0x558c55107260 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe8c0 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe8c0) get_ref 0x0~2000 Blob(0x558c587fe8c0 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x39000~2000: 0x0~2000 Blob(0x558c587fe8c0 blob([0x9d2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x3b000~2000: 0x0~2000 Blob(0x558c55107340 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe930 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe930) get_ref 0x0~2000 Blob(0x558c587fe930 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe930) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x3b000~2000: 0x0~2000 Blob(0x558c587fe930 blob([0x9d4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x3d000~2000: 0x0~2000 Blob(0x558c57e633b0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fe9a0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe9a0) get_ref 0x0~2000 Blob(0x558c587fe9a0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fe9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x3d000~2000: 0x0~2000 Blob(0x558c587fe9a0 blob([0x9d6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x3f000~2000: 0x0~2000 Blob(0x558c551079d0 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fea10 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fea10) get_ref 0x0~2000 Blob(0x558c587fea10 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fea10) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x3f000~2000: 0x0~2000 Blob(0x558c587fea10 blob([0x9d8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x41000~2000: 0x0~2000 Blob(0x558c57ed1e30 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fea80 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fea80) get_ref 0x0~2000 Blob(0x558c587fea80 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fea80) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x41000~2000: 0x0~2000 Blob(0x558c587fea80 blob([0x9da000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x43000~2000: 0x0~2000 Blob(0x558c55107ab0 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587feaf0 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587feaf0) get_ref 0x0~2000 Blob(0x558c587feaf0 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587feaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x43000~2000: 0x0~2000 Blob(0x558c587feaf0 blob([0x9dc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x45000~2000: 0x0~2000 Blob(0x558c55107b90 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587feb60 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587feb60) get_ref 0x0~2000 Blob(0x558c587feb60 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587feb60) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x45000~2000: 0x0~2000 Blob(0x558c587feb60 blob([0x9de000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x47000~2000: 0x0~2000 Blob(0x558c55107c70 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587febd0 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587febd0) get_ref 0x0~2000 Blob(0x558c587febd0 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587febd0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x47000~2000: 0x0~2000 Blob(0x558c587febd0 blob([0x9e0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x49000~2000: 0x0~2000 Blob(0x558c55107d50 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fec40 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fec40) get_ref 0x0~2000 Blob(0x558c587fec40 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fec40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x49000~2000: 0x0~2000 Blob(0x558c587fec40 blob([0x9e2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4b000~2000: 0x0~2000 Blob(0x558c55107e30 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fecb0 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fecb0) get_ref 0x0~2000 Blob(0x558c587fecb0 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fecb0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4b000~2000: 0x0~2000 Blob(0x558c587fecb0 blob([0x9e4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4d000~2000: 0x0~2000 Blob(0x558c55107f10 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fed20 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fed20) get_ref 0x0~2000 Blob(0x558c587fed20 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fed20) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4d000~2000: 0x0~2000 Blob(0x558c587fed20 blob([0x9e6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x4f000~2000: 0x0~2000 Blob(0x558c580a2000 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fed90 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fed90) get_ref 0x0~2000 Blob(0x558c587fed90 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fed90) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x4f000~2000: 0x0~2000 Blob(0x558c587fed90 blob([0x9e8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x51000~1000: 0x1000~1000 Blob(0x558c580a20e0 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fee00 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fee00) get_ref 0x1000~1000 Blob(0x558c587fee00 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fee00) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x51000~1000: 0x1000~1000 Blob(0x558c587fee00 blob([!~1000,0x9ea000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=3)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x52000~2000: 0x0~2000 Blob(0x558c587645b0 blob([0x2a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764620 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10961 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c587645b0 blob([0x2a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764620 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c587645b0 blob([0x2a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764620 loaded (sbid 0x2ad1 ref_map(0x2a9000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fee70 blob([0x2a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764620 loaded (sbid 0x2ad1 ref_map(0x2a9000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fee70) get_ref 0x0~2000 Blob(0x558c587fee70 blob([0x2a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764620 loaded (sbid 0x2ad1 ref_map(0x2a9000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fee70) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c587fee70 blob([0x2a9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764620 loaded (sbid 0x2ad1 ref_map(0x2a9000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x54000~2000: 0x0~2000 Blob(0x558c58764690 blob([0x2ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764700 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10962 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764690 blob([0x2ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764700 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764690 blob([0x2ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764700 loaded (sbid 0x2ad2 ref_map(0x2ab000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587feee0 blob([0x2ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764700 loaded (sbid 0x2ad2 ref_map(0x2ab000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587feee0) get_ref 0x0~2000 Blob(0x558c587feee0 blob([0x2ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764700 loaded (sbid 0x2ad2 ref_map(0x2ab000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587feee0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c587feee0 blob([0x2ab000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764700 loaded (sbid 0x2ad2 ref_map(0x2ab000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x56000~2000: 0x0~2000 Blob(0x558c58764770 blob([0x2ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587647e0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10963 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764770 blob([0x2ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587647e0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764770 blob([0x2ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587647e0 loaded (sbid 0x2ad3 ref_map(0x2ad000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fef50 blob([0x2ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587647e0 loaded (sbid 0x2ad3 ref_map(0x2ad000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fef50) get_ref 0x0~2000 Blob(0x558c587fef50 blob([0x2ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587647e0 loaded (sbid 0x2ad3 ref_map(0x2ad000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fef50) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c587fef50 blob([0x2ad000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587647e0 loaded (sbid 0x2ad3 ref_map(0x2ad000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x58000~2000: 0x0~2000 Blob(0x558c58764850 blob([0x2af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587648c0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10964 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764850 blob([0x2af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587648c0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764850 blob([0x2af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587648c0 loaded (sbid 0x2ad4 ref_map(0x2af000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fefc0 blob([0x2af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587648c0 loaded (sbid 0x2ad4 ref_map(0x2af000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fefc0) get_ref 0x0~2000 Blob(0x558c587fefc0 blob([0x2af000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587648c0 loaded (sbid 0x2ad4 ref_map(0x2af000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587fefc0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c587fefc0 blob([0x2af000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587648c0 loaded (sbid 0x2ad4 ref_map(0x2af000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c58764930 blob([0x2b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587649a0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10965 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764930 blob([0x2b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587649a0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764930 blob([0x2b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587649a0 loaded (sbid 0x2ad5 ref_map(0x2b1000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff030 blob([0x2b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587649a0 loaded (sbid 0x2ad5 ref_map(0x2b1000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff030) get_ref 0x0~2000 Blob(0x558c587ff030 blob([0x2b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587649a0 loaded (sbid 0x2ad5 ref_map(0x2b1000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff030) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c587ff030 blob([0x2b1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587649a0 loaded (sbid 0x2ad5 ref_map(0x2b1000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c58764a10 blob([0x2b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764a80 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10966 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764a10 blob([0x2b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764a80 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764a10 blob([0x2b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764a80 loaded (sbid 0x2ad6 ref_map(0x2b3000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff0a0 blob([0x2b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764a80 loaded (sbid 0x2ad6 ref_map(0x2b3000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff0a0) get_ref 0x0~2000 Blob(0x558c587ff0a0 blob([0x2b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764a80 loaded (sbid 0x2ad6 ref_map(0x2b3000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c587ff0a0 blob([0x2b3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764a80 loaded (sbid 0x2ad6 ref_map(0x2b3000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x5e000~2000: 0x0~2000 Blob(0x558c58764af0 blob([0x2b5000~1000,0x350000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764b60 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10967 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764af0 blob([0x2b5000~1000,0x350000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764b60 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764af0 blob([0x2b5000~1000,0x350000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764b60 loaded (sbid 0x2ad7 ref_map(0x2b5000~1000=1,0x350000~1000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff110 blob([0x2b5000~1000,0x350000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764b60 loaded (sbid 0x2ad7 ref_map(0x2b5000~1000=2,0x350000~1000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff110) get_ref 0x0~2000 Blob(0x558c587ff110 blob([0x2b5000~1000,0x350000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764b60 loaded (sbid 0x2ad7 ref_map(0x2b5000~1000=2,0x350000~1000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff110) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x5e000~2000: 0x0~2000 Blob(0x558c587ff110 blob([0x2b5000~1000,0x350000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764b60 loaded (sbid 0x2ad7 ref_map(0x2b5000~1000=2,0x350000~1000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x60000~2000: 0x0~2000 Blob(0x558c58764bd0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764c40 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10968 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764bd0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764c40 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764bd0 blob([0x351000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764c40 loaded (sbid 0x2ad8 ref_map(0x351000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff180 blob([0x351000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764c40 loaded (sbid 0x2ad8 ref_map(0x351000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff180) get_ref 0x0~2000 Blob(0x558c587ff180 blob([0x351000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764c40 loaded (sbid 0x2ad8 ref_map(0x351000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff180) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x60000~2000: 0x0~2000 Blob(0x558c587ff180 blob([0x351000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764c40 loaded (sbid 0x2ad8 ref_map(0x351000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x62000~2000: 0x0~2000 Blob(0x558c58764cb0 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764d20 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10969 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764cb0 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764d20 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764cb0 blob([0x353000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764d20 loaded (sbid 0x2ad9 ref_map(0x353000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff1f0 blob([0x353000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764d20 loaded (sbid 0x2ad9 ref_map(0x353000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff1f0) get_ref 0x0~2000 Blob(0x558c587ff1f0 blob([0x353000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764d20 loaded (sbid 0x2ad9 ref_map(0x353000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x62000~2000: 0x0~2000 Blob(0x558c587ff1f0 blob([0x353000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764d20 loaded (sbid 0x2ad9 ref_map(0x353000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x64000~2000: 0x0~2000 Blob(0x558c58764d90 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764e00 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10970 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764d90 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764e00 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764d90 blob([0x355000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764e00 loaded (sbid 0x2ada ref_map(0x355000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff260 blob([0x355000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764e00 loaded (sbid 0x2ada ref_map(0x355000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff260) get_ref 0x0~2000 Blob(0x558c587ff260 blob([0x355000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764e00 loaded (sbid 0x2ada ref_map(0x355000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff260) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x64000~2000: 0x0~2000 Blob(0x558c587ff260 blob([0x355000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764e00 loaded (sbid 0x2ada ref_map(0x355000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x66000~2000: 0x0~2000 Blob(0x558c58764e70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764ee0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10971 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764e70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764ee0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764e70 blob([0x357000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764ee0 loaded (sbid 0x2adb ref_map(0x357000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff2d0 blob([0x357000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764ee0 loaded (sbid 0x2adb ref_map(0x357000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff2d0) get_ref 0x0~2000 Blob(0x558c587ff2d0 blob([0x357000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764ee0 loaded (sbid 0x2adb ref_map(0x357000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x66000~2000: 0x0~2000 Blob(0x558c587ff2d0 blob([0x357000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764ee0 loaded (sbid 0x2adb ref_map(0x357000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x68000~2000: 0x0~2000 Blob(0x558c58764f50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764fc0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10972 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58764f50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764fc0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58764f50 blob([0x359000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764fc0 loaded (sbid 0x2adc ref_map(0x359000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff340 blob([0x359000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764fc0 loaded (sbid 0x2adc ref_map(0x359000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff340) get_ref 0x0~2000 Blob(0x558c587ff340 blob([0x359000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58764fc0 loaded (sbid 0x2adc ref_map(0x359000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff340) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x68000~2000: 0x0~2000 Blob(0x558c587ff340 blob([0x359000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58764fc0 loaded (sbid 0x2adc ref_map(0x359000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x6a000~2000: 0x0~2000 Blob(0x558c58765030 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587650a0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10973 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58765030 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587650a0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58765030 blob([0x35b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587650a0 loaded (sbid 0x2add ref_map(0x35b000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff3b0 blob([0x35b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587650a0 loaded (sbid 0x2add ref_map(0x35b000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff3b0) get_ref 0x0~2000 Blob(0x558c587ff3b0 blob([0x35b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587650a0 loaded (sbid 0x2add ref_map(0x35b000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x6a000~2000: 0x0~2000 Blob(0x558c587ff3b0 blob([0x35b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587650a0 loaded (sbid 0x2add ref_map(0x35b000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x6c000~2000: 0x0~2000 Blob(0x558c58765110 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765180 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10974 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58765110 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765180 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58765110 blob([0x35d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765180 loaded (sbid 0x2ade ref_map(0x35d000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff420 blob([0x35d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765180 loaded (sbid 0x2ade ref_map(0x35d000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff420) get_ref 0x0~2000 Blob(0x558c587ff420 blob([0x35d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765180 loaded (sbid 0x2ade ref_map(0x35d000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff420) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x6c000~2000: 0x0~2000 Blob(0x558c587ff420 blob([0x35d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765180 loaded (sbid 0x2ade ref_map(0x35d000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x6e000~2000: 0x0~2000 Blob(0x558c587651f0 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765260 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10975 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c587651f0 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765260 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c587651f0 blob([0x35f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765260 loaded (sbid 0x2adf ref_map(0x35f000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff490 blob([0x35f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765260 loaded (sbid 0x2adf ref_map(0x35f000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff490) get_ref 0x0~2000 Blob(0x558c587ff490 blob([0x35f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765260 loaded (sbid 0x2adf ref_map(0x35f000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff490) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x6e000~2000: 0x0~2000 Blob(0x558c587ff490 blob([0x35f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765260 loaded (sbid 0x2adf ref_map(0x35f000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x70000~2000: 0x0~2000 Blob(0x558c587652d0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765340 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10976 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c587652d0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765340 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c587652d0 blob([0x361000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765340 loaded (sbid 0x2ae0 ref_map(0x361000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff500 blob([0x361000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765340 loaded (sbid 0x2ae0 ref_map(0x361000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff500) get_ref 0x0~2000 Blob(0x558c587ff500 blob([0x361000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765340 loaded (sbid 0x2ae0 ref_map(0x361000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff500) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x70000~2000: 0x0~2000 Blob(0x558c587ff500 blob([0x361000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765340 loaded (sbid 0x2ae0 ref_map(0x361000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x72000~2000: 0x0~2000 Blob(0x558c587653b0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765420 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10977 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c587653b0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765420 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c587653b0 blob([0x363000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765420 loaded (sbid 0x2ae1 ref_map(0x363000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff570 blob([0x363000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765420 loaded (sbid 0x2ae1 ref_map(0x363000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff570) get_ref 0x0~2000 Blob(0x558c587ff570 blob([0x363000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765420 loaded (sbid 0x2ae1 ref_map(0x363000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff570) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x72000~2000: 0x0~2000 Blob(0x558c587ff570 blob([0x363000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765420 loaded (sbid 0x2ae1 ref_map(0x363000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x74000~2000: 0x0~2000 Blob(0x558c58765490 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765500 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10978 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58765490 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765500 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58765490 blob([0x365000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765500 loaded (sbid 0x2ae2 ref_map(0x365000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff5e0 blob([0x365000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765500 loaded (sbid 0x2ae2 ref_map(0x365000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff5e0) get_ref 0x0~2000 Blob(0x558c587ff5e0 blob([0x365000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765500 loaded (sbid 0x2ae2 ref_map(0x365000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x74000~2000: 0x0~2000 Blob(0x558c587ff5e0 blob([0x365000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58765500 loaded (sbid 0x2ae2 ref_map(0x365000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x76000~2000: 0x0~2000 Blob(0x558c58765570 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587655e0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10979 2019-12-07T19:34:39.633+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58765570 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587655e0 sbid 0x0)) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58765570 blob([0x367000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587655e0 loaded (sbid 0x2ae3 ref_map(0x367000~2000=1)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff650 blob([0x367000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587655e0 loaded (sbid 0x2ae3 ref_map(0x367000~2000=2)))) 2019-12-07T19:34:39.633+0000 7f8f07187700 20 bluestore.blob(0x558c587ff650) get_ref 0x0~2000 Blob(0x558c587ff650 blob([0x367000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587655e0 loaded (sbid 0x2ae3 ref_map(0x367000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff650) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x76000~2000: 0x0~2000 Blob(0x558c587ff650 blob([0x367000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587655e0 loaded (sbid 0x2ae3 ref_map(0x367000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x78000~2000: 0x0~2000 Blob(0x558c58765650 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587656c0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10980 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58765650 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587656c0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58765650 blob([0x369000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587656c0 loaded (sbid 0x2ae4 ref_map(0x369000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff6c0 blob([0x369000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587656c0 loaded (sbid 0x2ae4 ref_map(0x369000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff6c0) get_ref 0x0~2000 Blob(0x558c587ff6c0 blob([0x369000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587656c0 loaded (sbid 0x2ae4 ref_map(0x369000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x78000~2000: 0x0~2000 Blob(0x558c587ff6c0 blob([0x369000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587656c0 loaded (sbid 0x2ae4 ref_map(0x369000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x7a000~2000: 0x0~2000 Blob(0x558c58765730 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587657a0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10981 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58765730 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587657a0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58765730 blob([0x36b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587657a0 loaded (sbid 0x2ae5 ref_map(0x36b000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff730 blob([0x36b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587657a0 loaded (sbid 0x2ae5 ref_map(0x36b000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff730) get_ref 0x0~2000 Blob(0x558c587ff730 blob([0x36b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587657a0 loaded (sbid 0x2ae5 ref_map(0x36b000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff730) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x7a000~2000: 0x0~2000 Blob(0x558c587ff730 blob([0x36b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587657a0 loaded (sbid 0x2ae5 ref_map(0x36b000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x7c000~1000: 0x0~1000 Blob(0x558c58765810 blob([0x36d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58765880 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10982 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58765810 blob([0x36d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58765880 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58765810 blob([0x36d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58765880 loaded (sbid 0x2ae6 ref_map(0x36d000~1000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff7a0 blob([0x36d000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765880 loaded (sbid 0x2ae6 ref_map(0x36d000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff7a0) get_ref 0x0~1000 Blob(0x558c587ff7a0 blob([0x36d000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58765880 loaded (sbid 0x2ae6 ref_map(0x36d000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff7a0) get_ref init 0x1000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x7c000~1000: 0x0~1000 Blob(0x558c587ff7a0 blob([0x36d000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58765880 loaded (sbid 0x2ae6 ref_map(0x36d000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x7d000~2000: 0x0~2000 Blob(0x558c579d77a0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3030 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10983 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c579d77a0 blob([0x36e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3030 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c579d77a0 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3030 loaded (sbid 0x2ae7 ref_map(0x36e000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff810 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3030 loaded (sbid 0x2ae7 ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff810) get_ref 0x0~2000 Blob(0x558c587ff810 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e3030 loaded (sbid 0x2ae7 ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff810) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x7d000~2000: 0x0~2000 Blob(0x558c587ff810 blob([0x36e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e3030 loaded (sbid 0x2ae7 ref_map(0x36e000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x7f000~2000: 0x0~2000 Blob(0x558c57e63650 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551372d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10984 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57e63650 blob([0x370000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551372d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57e63650 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551372d0 loaded (sbid 0x2ae8 ref_map(0x370000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff880 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551372d0 loaded (sbid 0x2ae8 ref_map(0x370000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff880) get_ref 0x0~2000 Blob(0x558c587ff880 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c551372d0 loaded (sbid 0x2ae8 ref_map(0x370000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff880) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x7f000~2000: 0x0~2000 Blob(0x558c587ff880 blob([0x370000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c551372d0 loaded (sbid 0x2ae8 ref_map(0x370000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x81000~2000: 0x0~2000 Blob(0x558c579705b0 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb90 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10985 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c579705b0 blob([0x372000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb90 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c579705b0 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb90 loaded (sbid 0x2ae9 ref_map(0x372000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff8f0 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb90 loaded (sbid 0x2ae9 ref_map(0x372000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff8f0) get_ref 0x0~2000 Blob(0x558c587ff8f0 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb90 loaded (sbid 0x2ae9 ref_map(0x372000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x81000~2000: 0x0~2000 Blob(0x558c587ff8f0 blob([0x372000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb90 loaded (sbid 0x2ae9 ref_map(0x372000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x83000~2000: 0x0~2000 Blob(0x558c551371f0 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10986 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c551371f0 blob([0x374000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c551371f0 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 loaded (sbid 0x2aea ref_map(0x374000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff960 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 loaded (sbid 0x2aea ref_map(0x374000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff960) get_ref 0x0~2000 Blob(0x558c587ff960 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad9d0 loaded (sbid 0x2aea ref_map(0x374000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff960) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x83000~2000: 0x0~2000 Blob(0x558c587ff960 blob([0x374000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad9d0 loaded (sbid 0x2aea ref_map(0x374000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x85000~2000: 0x0~2000 Blob(0x558c55137180 blob([0x376000~1000,0xb9f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2d20 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10987 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55137180 blob([0x376000~1000,0xb9f000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2d20 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55137180 blob([0x376000~1000,0xb9f000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2d20 loaded (sbid 0x2aeb ref_map(0x376000~1000=1,0xb9f000~1000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ff9d0 blob([0x376000~1000,0xb9f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2d20 loaded (sbid 0x2aeb ref_map(0x376000~1000=2,0xb9f000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff9d0) get_ref 0x0~2000 Blob(0x558c587ff9d0 blob([0x376000~1000,0xb9f000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2d20 loaded (sbid 0x2aeb ref_map(0x376000~1000=2,0xb9f000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ff9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x85000~2000: 0x0~2000 Blob(0x558c587ff9d0 blob([0x376000~1000,0xb9f000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2d20 loaded (sbid 0x2aeb ref_map(0x376000~1000=2,0xb9f000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x87000~2000: 0x0~2000 Blob(0x558c551370a0 blob([0xba0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3730 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10988 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c551370a0 blob([0xba0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3730 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c551370a0 blob([0xba0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3730 loaded (sbid 0x2aec ref_map(0xba0000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffa40 blob([0xba0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3730 loaded (sbid 0x2aec ref_map(0xba0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffa40) get_ref 0x0~2000 Blob(0x558c587ffa40 blob([0xba0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3730 loaded (sbid 0x2aec ref_map(0xba0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffa40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x87000~2000: 0x0~2000 Blob(0x558c587ffa40 blob([0xba0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3730 loaded (sbid 0x2aec ref_map(0xba0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x89000~2000: 0x0~2000 Blob(0x558c55137030 blob([0xba2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a9a0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10989 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55137030 blob([0xba2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a9a0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55137030 blob([0xba2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a9a0 loaded (sbid 0x2aed ref_map(0xba2000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffab0 blob([0xba2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a9a0 loaded (sbid 0x2aed ref_map(0xba2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffab0) get_ref 0x0~2000 Blob(0x558c587ffab0 blob([0xba2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a9a0 loaded (sbid 0x2aed ref_map(0xba2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffab0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x89000~2000: 0x0~2000 Blob(0x558c587ffab0 blob([0xba2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a9a0 loaded (sbid 0x2aed ref_map(0xba2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x8b000~2000: 0x0~2000 Blob(0x558c550b3110 blob([0xba4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10990 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c550b3110 blob([0xba4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c550b3110 blob([0xba4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 loaded (sbid 0x2aee ref_map(0xba4000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffb20 blob([0xba4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 loaded (sbid 0x2aee ref_map(0xba4000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffb20) get_ref 0x0~2000 Blob(0x558c587ffb20 blob([0xba4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b37a0 loaded (sbid 0x2aee ref_map(0xba4000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffb20) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x8b000~2000: 0x0~2000 Blob(0x558c587ffb20 blob([0xba4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b37a0 loaded (sbid 0x2aee ref_map(0xba4000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x8d000~2000: 0x0~2000 Blob(0x558c574e2930 blob([0xba6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb20 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10991 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574e2930 blob([0xba6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb20 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574e2930 blob([0xba6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb20 loaded (sbid 0x2aef ref_map(0xba6000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffb90 blob([0xba6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb20 loaded (sbid 0x2aef ref_map(0xba6000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffb90) get_ref 0x0~2000 Blob(0x558c587ffb90 blob([0xba6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3bb20 loaded (sbid 0x2aef ref_map(0xba6000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffb90) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x8d000~2000: 0x0~2000 Blob(0x558c587ffb90 blob([0xba6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3bb20 loaded (sbid 0x2aef ref_map(0xba6000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x8f000~2000: 0x0~2000 Blob(0x558c55136f50 blob([0xba8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970540 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10992 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c55136f50 blob([0xba8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970540 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c55136f50 blob([0xba8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970540 loaded (sbid 0x2af0 ref_map(0xba8000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffc00 blob([0xba8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970540 loaded (sbid 0x2af0 ref_map(0xba8000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffc00) get_ref 0x0~2000 Blob(0x558c587ffc00 blob([0xba8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970540 loaded (sbid 0x2af0 ref_map(0xba8000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffc00) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x8f000~2000: 0x0~2000 Blob(0x558c587ffc00 blob([0xba8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970540 loaded (sbid 0x2af0 ref_map(0xba8000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x91000~2000: 0x0~2000 Blob(0x558c5829ccb0 blob([0xbaa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10993 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c5829ccb0 blob([0xbaa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c5829ccb0 blob([0xbaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 loaded (sbid 0x2af1 ref_map(0xbaa000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffc70 blob([0xbaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 loaded (sbid 0x2af1 ref_map(0xbaa000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffc70) get_ref 0x0~2000 Blob(0x558c587ffc70 blob([0xbaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3880 loaded (sbid 0x2af1 ref_map(0xbaa000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffc70) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x91000~2000: 0x0~2000 Blob(0x558c587ffc70 blob([0xbaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3880 loaded (sbid 0x2af1 ref_map(0xbaa000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x93000~2000: 0x0~2000 Blob(0x558c574e2d90 blob([0xbac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3810 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10994 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574e2d90 blob([0xbac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3810 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574e2d90 blob([0xbac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3810 loaded (sbid 0x2af2 ref_map(0xbac000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffce0 blob([0xbac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3810 loaded (sbid 0x2af2 ref_map(0xbac000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffce0) get_ref 0x0~2000 Blob(0x558c587ffce0 blob([0xbac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3810 loaded (sbid 0x2af2 ref_map(0xbac000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffce0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x93000~2000: 0x0~2000 Blob(0x558c587ffce0 blob([0xbac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3810 loaded (sbid 0x2af2 ref_map(0xbac000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x95000~2000: 0x0~2000 Blob(0x558c57f3bc00 blob([0xbae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2000 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10995 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f3bc00 blob([0xbae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2000 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f3bc00 blob([0xbae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2000 loaded (sbid 0x2af3 ref_map(0xbae000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffd50 blob([0xbae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2000 loaded (sbid 0x2af3 ref_map(0xbae000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffd50) get_ref 0x0~2000 Blob(0x558c587ffd50 blob([0xbae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b2000 loaded (sbid 0x2af3 ref_map(0xbae000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffd50) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x95000~2000: 0x0~2000 Blob(0x558c587ffd50 blob([0xbae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b2000 loaded (sbid 0x2af3 ref_map(0xbae000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x97000~2000: 0x0~2000 Blob(0x558c580a2850 blob([0xbb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970620 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10996 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c580a2850 blob([0xbb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970620 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c580a2850 blob([0xbb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970620 loaded (sbid 0x2af4 ref_map(0xbb0000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffdc0 blob([0xbb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970620 loaded (sbid 0x2af4 ref_map(0xbb0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffdc0) get_ref 0x0~2000 Blob(0x558c587ffdc0 blob([0xbb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970620 loaded (sbid 0x2af4 ref_map(0xbb0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x97000~2000: 0x0~2000 Blob(0x558c587ffdc0 blob([0xbb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970620 loaded (sbid 0x2af4 ref_map(0xbb0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x99000~2000: 0x0~2000 Blob(0x558c574e2e00 blob([0xbb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10997 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574e2e00 blob([0xbb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574e2e00 blob([0xbb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 loaded (sbid 0x2af5 ref_map(0xbb2000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffe30 blob([0xbb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 loaded (sbid 0x2af5 ref_map(0xbb2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffe30) get_ref 0x0~2000 Blob(0x558c587ffe30 blob([0xbb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3960 loaded (sbid 0x2af5 ref_map(0xbb2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffe30) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x99000~2000: 0x0~2000 Blob(0x558c587ffe30 blob([0xbb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3960 loaded (sbid 0x2af5 ref_map(0xbb2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x9b000~2000: 0x0~2000 Blob(0x558c57f3bc70 blob([0xbb4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b38f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10998 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f3bc70 blob([0xbb4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b38f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f3bc70 blob([0xbb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b38f0 loaded (sbid 0x2af6 ref_map(0xbb4000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587ffea0 blob([0xbb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b38f0 loaded (sbid 0x2af6 ref_map(0xbb4000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffea0) get_ref 0x0~2000 Blob(0x558c587ffea0 blob([0xbb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b38f0 loaded (sbid 0x2af6 ref_map(0xbb4000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587ffea0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x9b000~2000: 0x0~2000 Blob(0x558c587ffea0 blob([0xbb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b38f0 loaded (sbid 0x2af6 ref_map(0xbb4000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x9d000~2000: 0x0~2000 Blob(0x558c580a28c0 blob([0xbb6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 10999 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c580a28c0 blob([0xbb6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c580a28c0 blob([0xbb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 loaded (sbid 0x2af7 ref_map(0xbb6000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fff10 blob([0xbb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 loaded (sbid 0x2af7 ref_map(0xbb6000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587fff10) get_ref 0x0~2000 Blob(0x558c587fff10 blob([0xbb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57818c40 loaded (sbid 0x2af7 ref_map(0xbb6000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587fff10) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x9d000~2000: 0x0~2000 Blob(0x558c587fff10 blob([0xbb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57818c40 loaded (sbid 0x2af7 ref_map(0xbb6000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0x9f000~2000: 0x0~2000 Blob(0x558c574e2e70 blob([0xbb8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970690 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11000 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574e2e70 blob([0xbb8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970690 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574e2e70 blob([0xbb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970690 loaded (sbid 0x2af8 ref_map(0xbb8000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c587fff80 blob([0xbb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970690 loaded (sbid 0x2af8 ref_map(0xbb8000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587fff80) get_ref 0x0~2000 Blob(0x558c587fff80 blob([0xbb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57970690 loaded (sbid 0x2af8 ref_map(0xbb8000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c587fff80) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0x9f000~2000: 0x0~2000 Blob(0x558c587fff80 blob([0xbb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57970690 loaded (sbid 0x2af8 ref_map(0xbb8000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xa1000~2000: 0x0~2000 Blob(0x558c57f3bce0 blob([0xbba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11001 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c57f3bce0 blob([0xbba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c57f3bce0 blob([0xbba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 loaded (sbid 0x2af9 ref_map(0xbba000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58804000 blob([0xbba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 loaded (sbid 0x2af9 ref_map(0xbba000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804000) get_ref 0x0~2000 Blob(0x558c58804000 blob([0xbba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b3a40 loaded (sbid 0x2af9 ref_map(0xbba000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804000) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xa1000~2000: 0x0~2000 Blob(0x558c58804000 blob([0xbba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b3a40 loaded (sbid 0x2af9 ref_map(0xbba000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xa3000~2000: 0x0~2000 Blob(0x558c580a2bd0 blob([0xbbc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b39d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11002 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c580a2bd0 blob([0xbbc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b39d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c580a2bd0 blob([0xbbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b39d0 loaded (sbid 0x2afa ref_map(0xbbc000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58804070 blob([0xbbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b39d0 loaded (sbid 0x2afa ref_map(0xbbc000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804070) get_ref 0x0~2000 Blob(0x558c58804070 blob([0xbbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c579b39d0 loaded (sbid 0x2afa ref_map(0xbbc000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804070) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xa3000~2000: 0x0~2000 Blob(0x558c58804070 blob([0xbbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c579b39d0 loaded (sbid 0x2afa ref_map(0xbbc000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xa5000~2000: 0x0~2000 Blob(0x558c574e2ee0 blob([0xbbe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad880 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11003 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c574e2ee0 blob([0xbbe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad880 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c574e2ee0 blob([0xbbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad880 loaded (sbid 0x2afb ref_map(0xbbe000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c588040e0 blob([0xbbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad880 loaded (sbid 0x2afb ref_map(0xbbe000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588040e0) get_ref 0x0~2000 Blob(0x558c588040e0 blob([0xbbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ad880 loaded (sbid 0x2afb ref_map(0xbbe000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588040e0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xa5000~2000: 0x0~2000 Blob(0x558c588040e0 blob([0xbbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ad880 loaded (sbid 0x2afb ref_map(0xbbe000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xa7000~2000: 0x0~2000 Blob(0x558c58766700 blob([0xbc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58766690 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11004 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58766700 blob([0xbc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58766690 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58766700 blob([0xbc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58766690 loaded (sbid 0x2afc ref_map(0xbc0000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58804150 blob([0xbc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766690 loaded (sbid 0x2afc ref_map(0xbc0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804150) get_ref 0x0~2000 Blob(0x558c58804150 blob([0xbc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766690 loaded (sbid 0x2afc ref_map(0xbc0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804150) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xa7000~2000: 0x0~2000 Blob(0x558c58804150 blob([0xbc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58766690 loaded (sbid 0x2afc ref_map(0xbc0000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xa9000~2000: 0x0~2000 Blob(0x558c58766770 blob([0xbc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587667e0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11005 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c58766770 blob([0xbc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587667e0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c58766770 blob([0xbc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587667e0 loaded (sbid 0x2afd ref_map(0xbc2000~2000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c588041c0 blob([0xbc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587667e0 loaded (sbid 0x2afd ref_map(0xbc2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588041c0) get_ref 0x0~2000 Blob(0x558c588041c0 blob([0xbc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c587667e0 loaded (sbid 0x2afd ref_map(0xbc2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588041c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xa9000~2000: 0x0~2000 Blob(0x558c588041c0 blob([0xbc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c587667e0 loaded (sbid 0x2afd ref_map(0xbc2000~2000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup src 0xab000~1000: 0x1000~1000 Blob(0x558c587668c0 blob([!~1000,0xbc4000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c58766930 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11006 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared Blob(0x558c587668c0 blob([!~1000,0xbc4000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c58766930 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.bs1_head 0x558c54d9e3c0) make_blob_shared now Blob(0x558c587668c0 blob([!~1000,0xbc4000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c58766930 loaded (sbid 0x2afe ref_map(0xbc4000~1000=1)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup new Blob(0x558c58804230 blob([!~1000,0xbc4000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766930 loaded (sbid 0x2afe ref_map(0xbc4000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804230) get_ref 0x1000~1000 Blob(0x558c58804230 blob([!~1000,0xbc4000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58766930 loaded (sbid 0x2afe ref_map(0xbc4000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804230) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dup dst 0xab000~1000: 0x1000~1000 Blob(0x558c58804230 blob([!~1000,0xbc4000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c58766930 loaded (sbid 0x2afe ref_map(0xbc4000~1000=2)))) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark shard 0x4f000 dirty 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark shard 0x76000 dirty 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) dirty_range mark inline shard dirty 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:26# = 0 2019-12-07T19:34:39.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:26# snapset 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:26# snapset = 0 2019-12-07T19:34:39.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:26# 1 keys 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:26# 1 keys = 0 2019-12-07T19:34:39.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:26# hinfo_key (42 bytes) 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:26# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:39.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:39.634+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xac000~2c000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0xac000~2c000 - have 0xac000 (704512) bytes fadvise_flags 0x0 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:39.634+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0xac000~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58bb4000 22 blobs 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.634+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2c000/1000,2c000,0 2019-12-07T19:34:39.634+0000 7f8f07187700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbd1000~2c000/1000,2c000,0 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbd1000~2c000] 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c588042a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c588042a0 blob([0xbd1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804310 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588042a0) get_ref 0x0~2000 Blob(0x558c588042a0 blob([0xbd1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804310 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588042a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xac000~2000: 0x0~2000 Blob(0x558c588042a0 blob([0xbd1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804310 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804328 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c588043f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804380 blob([0xbd3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588043f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804380) get_ref 0x0~2000 Blob(0x558c58804380 blob([0xbd3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588043f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804380) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xae000~2000: 0x0~2000 Blob(0x558c58804380 blob([0xbd3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c588043f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804408 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c588044d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804460 blob([0xbd5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588044d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804460) get_ref 0x0~2000 Blob(0x558c58804460 blob([0xbd5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588044d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804460) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xb0000~2000: 0x0~2000 Blob(0x558c58804460 blob([0xbd5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c588044d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c588044e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c588045b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804540 blob([0xbd7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588045b0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804540) get_ref 0x0~2000 Blob(0x558c58804540 blob([0xbd7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588045b0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804540) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xb2000~2000: 0x0~2000 Blob(0x558c58804540 blob([0xbd7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c588045b0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c588045c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804620 blob([0xbd9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804690 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804620) get_ref 0x0~2000 Blob(0x558c58804620 blob([0xbd9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804690 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804620) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xb4000~2000: 0x0~2000 Blob(0x558c58804620 blob([0xbd9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804690 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c588046a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804700 blob([0xbdb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804770 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804700) get_ref 0x0~2000 Blob(0x558c58804700 blob([0xbdb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804770 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804700) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xb6000~2000: 0x0~2000 Blob(0x558c58804700 blob([0xbdb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804770 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804788 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c588047e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c588047e0 blob([0xbdd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804850 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588047e0) get_ref 0x0~2000 Blob(0x558c588047e0 blob([0xbdd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804850 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588047e0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xb8000~2000: 0x0~2000 Blob(0x558c588047e0 blob([0xbdd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804850 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804868 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c588048c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c588048c0 blob([0xbdf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804930 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588048c0) get_ref 0x0~2000 Blob(0x558c588048c0 blob([0xbdf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804930 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588048c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xba000~2000: 0x0~2000 Blob(0x558c588048c0 blob([0xbdf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804930 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804948 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c588049a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c588049a0 blob([0xbe1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804a10 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588049a0) get_ref 0x0~2000 Blob(0x558c588049a0 blob([0xbe1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804a10 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588049a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xbc000~2000: 0x0~2000 Blob(0x558c588049a0 blob([0xbe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804a10 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804a80 blob([0xbe3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804af0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804a80) get_ref 0x0~2000 Blob(0x558c58804a80 blob([0xbe3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804af0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804a80) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xbe000~2000: 0x0~2000 Blob(0x558c58804a80 blob([0xbe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804af0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804b08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804b60 blob([0xbe5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804bd0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804b60) get_ref 0x0~2000 Blob(0x558c58804b60 blob([0xbe5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804bd0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804b60) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc0000~2000: 0x0~2000 Blob(0x558c58804b60 blob([0xbe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804bd0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804be8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804c40 blob([0xbe7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804cb0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804c40) get_ref 0x0~2000 Blob(0x558c58804c40 blob([0xbe7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804cb0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804c40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc2000~2000: 0x0~2000 Blob(0x558c58804c40 blob([0xbe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804cb0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804cc8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804d20 blob([0xbe9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804d90 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804d20) get_ref 0x0~2000 Blob(0x558c58804d20 blob([0xbe9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804d90 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804d20) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc4000~2000: 0x0~2000 Blob(0x558c58804d20 blob([0xbe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804d90 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804da8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804e00 blob([0xbeb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804e70 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804e00) get_ref 0x0~2000 Blob(0x558c58804e00 blob([0xbeb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804e70 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804e00) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc6000~2000: 0x0~2000 Blob(0x558c58804e00 blob([0xbeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804e70 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804e88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58804f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804ee0 blob([0xbed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804f50 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804ee0) get_ref 0x0~2000 Blob(0x558c58804ee0 blob([0xbed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58804f50 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc8000~2000: 0x0~2000 Blob(0x558c58804ee0 blob([0xbed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58804f50 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58804f68 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58804fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58805030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58804fc0 blob([0xbef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805030 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804fc0) get_ref 0x0~2000 Blob(0x558c58804fc0 blob([0xbef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805030 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58804fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xca000~2000: 0x0~2000 Blob(0x558c58804fc0 blob([0xbef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58805030 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58805048 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c588050a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58805110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c588050a0 blob([0xbf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805110 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588050a0) get_ref 0x0~2000 Blob(0x558c588050a0 blob([0xbf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805110 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c588050a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xcc000~2000: 0x0~2000 Blob(0x558c588050a0 blob([0xbf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58805110 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58805128 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58805180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c588051f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58805180 blob([0xbf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588051f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805180) get_ref 0x0~2000 Blob(0x558c58805180 blob([0xbf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588051f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805180) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xce000~2000: 0x0~2000 Blob(0x558c58805180 blob([0xbf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c588051f0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58805208 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58805260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c588052d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58805260 blob([0xbf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588052d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805260) get_ref 0x0~2000 Blob(0x558c58805260 blob([0xbf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588052d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805260) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xd0000~2000: 0x0~2000 Blob(0x558c58805260 blob([0xbf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c588052d0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c588052e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58805340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c588053b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58805340 blob([0xbf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588053b0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805340) get_ref 0x0~2000 Blob(0x558c58805340 blob([0xbf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c588053b0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805340) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xd2000~2000: 0x0~2000 Blob(0x558c58805340 blob([0xbf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c588053b0 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c588053c8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58805420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58805490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58805420 blob([0xbf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805490 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805420) get_ref 0x0~2000 Blob(0x558c58805420 blob([0xbf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805490 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805420) get_ref init 0x2000, 1000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xd4000~2000: 0x0~2000 Blob(0x558c58805420 blob([0xbf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58805490 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c588054a8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58805500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58805570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58805500 blob([0xbfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805570 sbid 0x0)) 2019-12-07T19:34:39.634+0000 7f8f07187700 20 bluestore.blob(0x558c58805500) get_ref 0x0~2000 Blob(0x558c58805500 blob([0xbfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58805570 sbid 0x0)) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.blob(0x558c58805500) get_ref init 0x2000, 1000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xd6000~2000: 0x0~2000 Blob(0x558c58805500 blob([0xbfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58805570 sbid 0x0)) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.BufferSpace(0x558c58805588 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xd8000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xac000~2c000 = 0 2019-12-07T19:34:39.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.B0E469FE.26.smithi13140796-28..' <- OBJ_.1_0000000000000001.B0E469FE.26.smithi13140796-28.. 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000025_.1_0000000000000001.B0E469FE.26.smithi13140796-28..' <- SNA_1_0000000000000025_.1_0000000000000001.B0E469FE.26.smithi13140796-28.. 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000026_.1_0000000000000001.B0E469FE.26.smithi13140796-28..' <- SNA_1_0000000000000026_.1_0000000000000001.B0E469FE.26.smithi13140796-28.. 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:39.635+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000062.00000000000000000051' <- 0000000062.00000000000000000051 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000062.00000000000000000052' <- 0000000062.00000000000000000052 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bb4000 cost 276887 (23 ios * 4000 + 184887 bytes) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bb4000 onodes 0x558c54e44780,0x558c58768f00 shared_blobs 0x558c54677810,0x558c54cf96c0,0x558c550b2ee0,0x558c550b3650,0x558c55106e70,0x558c55106f50,0x558c55107030,0x558c55107110,0x558c551071f0,0x558c551072d0,0x558c551073b0,0x558c55107a40,0x558c55107b20,0x558c55107c00,0x558c55107ce0,0x558c55107dc0,0x558c55107ea0,0x558c55107f80,0x558c551372d0,0x558c552790a0,0x558c574e2d20,0x558c574e3030,0x558c574fd960,0x558c57567110,0x558c576ad650,0x558c576ad880,0x558c576ad9d0,0x558c576adb90,0x558c57818c40,0x558c57897c70,0x558c57970540,0x558c57970620,0x558c57970690,0x558c57970af0,0x558c57970bd0,0x558c57970cb0,0x558c57970d90,0x558c57970e70,0x558c579718f0,0x558c57971960,0x558c579719d0,0x558c57971a40,0x558c57971ab0,0x558c57971b20,0x558c57971b90,0x558c579b2000,0x558c579b22a0,0x558c579b3730,0x558c579b37a0,0x558c579b3810,0x558c579b3880,0x558c579b38f0,0x558c579b3960,0x558c579b39d0,0x558c579b3a40,0x558c57f3bb20,0x558c57f3bb90,0x558c5809a9a0,0x558c580a2070,0x558c580a2150,0x558c580d79d0,0x558c583ead20,0x558c5840f420,0x558c58764620,0x558c58764700,0x558c587647e0,0x558c587648c0,0x558c587649a0,0x558c58764a80,0x558c58764b60,0x558c58764c40,0x558c58764d20,0x558c58764e00,0x558c58764ee0,0x558c58764fc0,0x558c587650a0,0x558c58765180,0x558c58765260,0x558c58765340,0x558c58765420,0x558c58765500,0x558c587655e0,0x558c587656c0,0x558c587657a0,0x558c58765880,0x558c58766690,0x558c587667e0,0x558c58766930 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update shard 0x4f000 is 564 bytes (was 420) from 20 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c54e44888) update shard 0x76000 is 1196 bytes (was 553) from 50 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 666 (664 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update 1#1:d07269f7:::smithi13140796-28:26# 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update inline shard 2412 bytes from 88 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore.extentmap(0x558c58769008) reshard 0x[0,ffffffff) of 0 shards on 1#1:d07269f7:::smithi13140796-28:26# 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard new shard 0x0 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard new shard 0x23000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard new shard 0x47000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard new shard 0x6a000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard new shard 0x8d000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes),0x6a000(0x0 bytes),0x8d000(0x0 bytes)] 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard old [] 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x47000(0x0 bytes),0x6a000(0x0 bytes),0x8d000(0x0 bytes)] 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update 1#1:d07269f7:::smithi13140796-28:26# force 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update shard 0x0 is 486 bytes (was 0) from 18 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update shard 0x23000 is 491 bytes (was 0) from 18 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update shard 0x47000 is 510 bytes (was 0) from 18 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update shard 0x6a000 is 494 bytes (was 0) from 18 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore.extentmap(0x558c58769008) update shard 0x8d000 is 450 bytes (was 0) from 16 extents 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:26# is 438 (436 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a23 is 11 SharedBlob(0x558c54677810 loaded (sbid 0x2a23 ref_map(0x9c2000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2f is 11 SharedBlob(0x558c54cf96c0 loaded (sbid 0x2a2f ref_map(0x9da000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a25 is 11 SharedBlob(0x558c550b2ee0 loaded (sbid 0x2a25 ref_map(0x9c6000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a20 is 11 SharedBlob(0x558c550b3650 loaded (sbid 0x2a20 ref_map(0x9bc000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a26 is 11 SharedBlob(0x558c55106e70 loaded (sbid 0x2a26 ref_map(0x9c8000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a27 is 11 SharedBlob(0x558c55106f50 loaded (sbid 0x2a27 ref_map(0x9ca000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a28 is 11 SharedBlob(0x558c55107030 loaded (sbid 0x2a28 ref_map(0x9cc000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a29 is 11 SharedBlob(0x558c55107110 loaded (sbid 0x2a29 ref_map(0x9ce000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2a is 11 SharedBlob(0x558c551071f0 loaded (sbid 0x2a2a ref_map(0x9d0000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2b is 11 SharedBlob(0x558c551072d0 loaded (sbid 0x2a2b ref_map(0x9d2000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2c is 11 SharedBlob(0x558c551073b0 loaded (sbid 0x2a2c ref_map(0x9d4000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2e is 11 SharedBlob(0x558c55107a40 loaded (sbid 0x2a2e ref_map(0x9d8000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a30 is 11 SharedBlob(0x558c55107b20 loaded (sbid 0x2a30 ref_map(0x9dc000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a31 is 11 SharedBlob(0x558c55107c00 loaded (sbid 0x2a31 ref_map(0x9de000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a32 is 11 SharedBlob(0x558c55107ce0 loaded (sbid 0x2a32 ref_map(0x9e0000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a33 is 11 SharedBlob(0x558c55107dc0 loaded (sbid 0x2a33 ref_map(0x9e2000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a34 is 11 SharedBlob(0x558c55107ea0 loaded (sbid 0x2a34 ref_map(0x9e4000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a35 is 11 SharedBlob(0x558c55107f80 loaded (sbid 0x2a35 ref_map(0x9e6000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae8 is 11 SharedBlob(0x558c551372d0 loaded (sbid 0x2ae8 ref_map(0x370000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a22 is 11 SharedBlob(0x558c552790a0 loaded (sbid 0x2a22 ref_map(0x9c0000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aeb is 15 SharedBlob(0x558c574e2d20 loaded (sbid 0x2aeb ref_map(0x376000~1000=2,0xb9f000~1000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae7 is 11 SharedBlob(0x558c574e3030 loaded (sbid 0x2ae7 ref_map(0x36e000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a24 is 11 SharedBlob(0x558c574fd960 loaded (sbid 0x2a24 ref_map(0x9c4000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a2d is 11 SharedBlob(0x558c57567110 loaded (sbid 0x2a2d ref_map(0x9d6000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a13 is 11 SharedBlob(0x558c576ad650 loaded (sbid 0x2a13 ref_map(0x9a3000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2afb is 11 SharedBlob(0x558c576ad880 loaded (sbid 0x2afb ref_map(0xbbe000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aea is 11 SharedBlob(0x558c576ad9d0 loaded (sbid 0x2aea ref_map(0x374000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a21 is 11 SharedBlob(0x558c576adb90 loaded (sbid 0x2a21 ref_map(0x9be000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af7 is 11 SharedBlob(0x558c57818c40 loaded (sbid 0x2af7 ref_map(0xbb6000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0f is 11 SharedBlob(0x558c57897c70 loaded (sbid 0x2a0f ref_map(0x99b000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af0 is 11 SharedBlob(0x558c57970540 loaded (sbid 0x2af0 ref_map(0xba8000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af4 is 11 SharedBlob(0x558c57970620 loaded (sbid 0x2af4 ref_map(0xbb0000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af8 is 11 SharedBlob(0x558c57970690 loaded (sbid 0x2af8 ref_map(0xbb8000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1c is 11 SharedBlob(0x558c57970af0 loaded (sbid 0x2a1c ref_map(0x9b5000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1a is 11 SharedBlob(0x558c57970bd0 loaded (sbid 0x2a1a ref_map(0x9b1000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a18 is 11 SharedBlob(0x558c57970cb0 loaded (sbid 0x2a18 ref_map(0x9ad000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a16 is 11 SharedBlob(0x558c57970d90 loaded (sbid 0x2a16 ref_map(0x9a9000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a14 is 11 SharedBlob(0x558c57970e70 loaded (sbid 0x2a14 ref_map(0x9a5000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1f is 11 SharedBlob(0x558c579718f0 loaded (sbid 0x2a1f ref_map(0x9bb000~1000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1d is 11 SharedBlob(0x558c57971960 loaded (sbid 0x2a1d ref_map(0x9b7000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1b is 11 SharedBlob(0x558c579719d0 loaded (sbid 0x2a1b ref_map(0x9b3000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a19 is 11 SharedBlob(0x558c57971a40 loaded (sbid 0x2a19 ref_map(0x9af000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a17 is 11 SharedBlob(0x558c57971ab0 loaded (sbid 0x2a17 ref_map(0x9ab000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a15 is 11 SharedBlob(0x558c57971b20 loaded (sbid 0x2a15 ref_map(0x9a7000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a10 is 11 SharedBlob(0x558c57971b90 loaded (sbid 0x2a10 ref_map(0x99d000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af3 is 11 SharedBlob(0x558c579b2000 loaded (sbid 0x2af3 ref_map(0xbae000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a1e is 11 SharedBlob(0x558c579b22a0 loaded (sbid 0x2a1e ref_map(0x9b9000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aec is 11 SharedBlob(0x558c579b3730 loaded (sbid 0x2aec ref_map(0xba0000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aee is 11 SharedBlob(0x558c579b37a0 loaded (sbid 0x2aee ref_map(0xba4000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af2 is 11 SharedBlob(0x558c579b3810 loaded (sbid 0x2af2 ref_map(0xbac000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af1 is 11 SharedBlob(0x558c579b3880 loaded (sbid 0x2af1 ref_map(0xbaa000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af6 is 11 SharedBlob(0x558c579b38f0 loaded (sbid 0x2af6 ref_map(0xbb4000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af5 is 11 SharedBlob(0x558c579b3960 loaded (sbid 0x2af5 ref_map(0xbb2000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2afa is 11 SharedBlob(0x558c579b39d0 loaded (sbid 0x2afa ref_map(0xbbc000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2af9 is 11 SharedBlob(0x558c579b3a40 loaded (sbid 0x2af9 ref_map(0xbba000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aef is 11 SharedBlob(0x558c57f3bb20 loaded (sbid 0x2aef ref_map(0xba6000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae9 is 11 SharedBlob(0x558c57f3bb90 loaded (sbid 0x2ae9 ref_map(0x372000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aed is 11 SharedBlob(0x558c5809a9a0 loaded (sbid 0x2aed ref_map(0xba2000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a36 is 11 SharedBlob(0x558c580a2070 loaded (sbid 0x2a36 ref_map(0x9e8000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a37 is 11 SharedBlob(0x558c580a2150 loaded (sbid 0x2a37 ref_map(0x9ea000~1000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a0e is 11 SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a11 is 11 SharedBlob(0x558c583ead20 loaded (sbid 0x2a11 ref_map(0x99f000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2a12 is 11 SharedBlob(0x558c5840f420 loaded (sbid 0x2a12 ref_map(0x9a1000~2000=3))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad1 is 11 SharedBlob(0x558c58764620 loaded (sbid 0x2ad1 ref_map(0x2a9000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad2 is 11 SharedBlob(0x558c58764700 loaded (sbid 0x2ad2 ref_map(0x2ab000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad3 is 11 SharedBlob(0x558c587647e0 loaded (sbid 0x2ad3 ref_map(0x2ad000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad4 is 11 SharedBlob(0x558c587648c0 loaded (sbid 0x2ad4 ref_map(0x2af000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad5 is 11 SharedBlob(0x558c587649a0 loaded (sbid 0x2ad5 ref_map(0x2b1000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad6 is 11 SharedBlob(0x558c58764a80 loaded (sbid 0x2ad6 ref_map(0x2b3000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad7 is 15 SharedBlob(0x558c58764b60 loaded (sbid 0x2ad7 ref_map(0x2b5000~1000=2,0x350000~1000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad8 is 11 SharedBlob(0x558c58764c40 loaded (sbid 0x2ad8 ref_map(0x351000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad9 is 11 SharedBlob(0x558c58764d20 loaded (sbid 0x2ad9 ref_map(0x353000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ada is 11 SharedBlob(0x558c58764e00 loaded (sbid 0x2ada ref_map(0x355000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2adb is 11 SharedBlob(0x558c58764ee0 loaded (sbid 0x2adb ref_map(0x357000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2adc is 11 SharedBlob(0x558c58764fc0 loaded (sbid 0x2adc ref_map(0x359000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2add is 11 SharedBlob(0x558c587650a0 loaded (sbid 0x2add ref_map(0x35b000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ade is 11 SharedBlob(0x558c58765180 loaded (sbid 0x2ade ref_map(0x35d000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2adf is 11 SharedBlob(0x558c58765260 loaded (sbid 0x2adf ref_map(0x35f000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae0 is 11 SharedBlob(0x558c58765340 loaded (sbid 0x2ae0 ref_map(0x361000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae1 is 11 SharedBlob(0x558c58765420 loaded (sbid 0x2ae1 ref_map(0x363000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae2 is 11 SharedBlob(0x558c58765500 loaded (sbid 0x2ae2 ref_map(0x365000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae3 is 11 SharedBlob(0x558c587655e0 loaded (sbid 0x2ae3 ref_map(0x367000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae4 is 11 SharedBlob(0x558c587656c0 loaded (sbid 0x2ae4 ref_map(0x369000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae5 is 11 SharedBlob(0x558c587657a0 loaded (sbid 0x2ae5 ref_map(0x36b000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ae6 is 11 SharedBlob(0x558c58765880 loaded (sbid 0x2ae6 ref_map(0x36d000~1000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2afc is 11 SharedBlob(0x558c58766690 loaded (sbid 0x2afc ref_map(0xbc0000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2afd is 11 SharedBlob(0x558c587667e0 loaded (sbid 0x2afd ref_map(0xbc2000~2000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2afe is 11 SharedBlob(0x558c58766930 loaded (sbid 0x2afe ref_map(0xbc4000~1000=2))) 2019-12-07T19:34:39.635+0000 7f8f07187700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bb4000 allocated 0x[bd1000~2c000] released 0x[] 2019-12-07T19:34:39.635+0000 7f8f07187700 10 freelist allocate 0xbd1000~2c000 2019-12-07T19:34:39.635+0000 7f8f07187700 20 freelist _xor first_key 0xb80000 last_key 0xb80000 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb4000 prepare 2019-12-07T19:34:39.635+0000 7f8f07187700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58bb4000 2019-12-07T19:34:39.635+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.635+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'52 (0'0,62'52] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'50 crt=62'50 lcod 62'50 active mbc={} ps=[20~1,23~1]] op_applied version 62'52 2019-12-07T19:34:39.635+0000 7f8f07187700 10 osd.7 62 dequeue_op 0x558c54e78840 finish 2019-12-07T19:34:39.635+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.636+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb4000 aio_wait 2019-12-07T19:34:39.636+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bb4000 2019-12-07T19:34:39.636+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb4000 io_done 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10669 < min_flush_size 524288 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58768f00 had 1 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10692 < min_flush_size 524288 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 10692 < min_flush_size 524288 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53cb3c~29c4 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53cb3c 0x53cb3c~29c4 to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~510000 x_off 0x50cb3c 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb3c 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x500 and padding block with 0xb00 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x53f500 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.636+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000213276s (0.000046060s flush + 0.000167216s kv commit) 2019-12-07T19:34:39.636+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58bb4000> 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb4000 kv_submitted 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bb4000 2019-12-07T19:34:39.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb4000 finishing 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58bb4000 onodes 0x558c54e44780,0x558c58768f00 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804328 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8f050 space 0x558c58804328 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804408 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8ef30 space 0x558c58804408 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c588044e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8eea0 space 0x558c588044e8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c588045c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2d80 space 0x558c588045c8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c588046a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2cf0 space 0x558c588046a8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804788 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8f3b0 space 0x558c58804788 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804868 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8f320 space 0x558c58804868 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804948 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8f290 space 0x558c58804948 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804a28 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c30e0 space 0x558c58804a28 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804b08 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c3050 space 0x558c58804b08 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804be8 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c3170 space 0x558c58804be8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804cc8 in 0x558c53a59c00) _finish_write discard buffer(0x558c580f2d80 space 0x558c58804cc8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804da8 in 0x558c53a59c00) _finish_write discard buffer(0x558c580f2cf0 space 0x558c58804da8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804e88 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8f200 space 0x558c58804e88 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58804f68 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8f170 space 0x558c58804f68 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58805048 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2f30 space 0x558c58805048 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58805128 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2fc0 space 0x558c58805128 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58805208 in 0x558c53a59c00) _finish_write discard buffer(0x558c551a8750 space 0x558c58805208 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c588052e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2e10 space 0x558c588052e8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c588053c8 in 0x558c53a59c00) _finish_write discard buffer(0x558c586c2ea0 space 0x558c588053c8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c588054a8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e142d0 space 0x558c588054a8 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58805588 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535440 space 0x558c58805588 0x0~2000 writing nocache) 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bb4000 done 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:39.636+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58bb4000 [] 2019-12-07T19:34:39.636+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.636+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.636+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 62/41 ECSubWriteReply(tid=122, last_complete=62'52, committed=1, applied=1)) v2 -- 0x558c58115400 con 0x558c54c5b400 2019-12-07T19:34:39.636+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.636+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 335 ==== MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=123, reqid=client.4540.0:458, at_version=62'53, trim_to=0'0, roll_forward_to=62'50)) v2 ==== 113240+0+0 (unknown 0 0 0) 0x558c58bce000 con 0x558c54c5b400 2019-12-07T19:34:39.636+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000043 epoch 62 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=123, reqid=client.4540.0:458, at_version=62'53, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.636+0000 7f8f24c58700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=123, reqid=client.4540.0:458, at_version=62'53, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=123, reqid=client.4540.0:458, at_version=62'53, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=123, reqid=client.4540.0:458, at_version=62'53, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c54e78b00 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=123, reqid=client.4540.0:458, at_version=62'53, trim_to=0'0, roll_forward_to=62'50)) v2 pg pg[1.bs1( v 62'52 (0'0,62'52] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'52 (0'0,62'52] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=123, reqid=client.4540.0:458, at_version=62'53, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'52 (0'0,62'52] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'52], crt=62'50) [62'53 (62'52) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:458 2019-12-07T19:34:39.633634+0000 0 ObjectCleanRegions clean_offsets: [0~3538944,3981312~18446744073705570303], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'53 (0'0,62'53] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'52 crt=62'50 lcod 62'52 active mbc={} ps=[20~1,23~1]] add_log_entry 62'53 (62'52) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:458 2019-12-07T19:34:39.633634+0000 0 ObjectCleanRegions clean_offsets: [0~3538944,3981312~18446744073705570303], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'53 (0'0,62'53] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'52 crt=62'50 lcod 62'52 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 53 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'53 (0'0,62'53] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'52 crt=62'50 lcod 62'52 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.636+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'53, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58bc2600 seq 65 2019-12-07T19:34:39.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:39.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:39.636+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xd8000~1b000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0xd8000~1b000 - have 0xd8000 (884736) bytes fadvise_flags 0x0 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0xd8000~1b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58bc2600 14 blobs 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1b000/1000,1b000,0 2019-12-07T19:34:39.636+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xbfd000~1b000/1000,1b000,0 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xbfd000~1b000] 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578808c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58226070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578808c0 blob([0xbfd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58226070 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c578808c0) get_ref 0x0~2000 Blob(0x558c578808c0 blob([0xbfd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58226070 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c578808c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xd8000~2000: 0x0~2000 Blob(0x558c578808c0 blob([0xbfd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58226070 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c58226088 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c582260e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2c40 blob([0xbff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c582260e0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2c40) get_ref 0x0~2000 Blob(0x558c574e2c40 blob([0xbff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c582260e0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2c40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xda000~2000: 0x0~2000 Blob(0x558c574e2c40 blob([0xbff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c582260e0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c582260f8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3ba40 blob([0xc01000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2cb0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3ba40) get_ref 0x0~2000 Blob(0x558c57f3ba40 blob([0xc01000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2cb0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57f3ba40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xdc000~2000: 0x0~2000 Blob(0x558c57f3ba40 blob([0xc01000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2cb0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2cc8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d180 blob([0xc03000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac540 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d180) get_ref 0x0~2000 Blob(0x558c5829d180 blob([0xc03000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac540 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d180) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xde000~2000: 0x0~2000 Blob(0x558c5829d180 blob([0xc03000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac540 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ac558 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57971e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d1f0 blob([0xc05000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971e30 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d1f0) get_ref 0x0~2000 Blob(0x558c5829d1f0 blob([0xc05000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57971e30 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe0000~2000: 0x0~2000 Blob(0x558c5829d1f0 blob([0xc05000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57971e30 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57971e48 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e636c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2b60 blob([0xc07000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e636c0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2b60) get_ref 0x0~2000 Blob(0x558c574e2b60 blob([0xc07000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e636c0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe2000~2000: 0x0~2000 Blob(0x558c574e2b60 blob([0xc07000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e636c0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57e636d8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54cf81c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54cf81c0 blob([0xc09000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2af0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf81c0) get_ref 0x0~2000 Blob(0x558c54cf81c0 blob([0xc09000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2af0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c54cf81c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe4000~2000: 0x0~2000 Blob(0x558c54cf81c0 blob([0xc09000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2af0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2b08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e63a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c576ac3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e63a40 blob([0xc0b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac3f0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57e63a40) get_ref 0x0~2000 Blob(0x558c57e63a40 blob([0xc0b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c576ac3f0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c57e63a40) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe6000~2000: 0x0~2000 Blob(0x558c57e63a40 blob([0xc0b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c576ac3f0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c576ac408 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e639d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2a80 blob([0xc0d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e639d0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2a80) get_ref 0x0~2000 Blob(0x558c574e2a80 blob([0xc0d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e639d0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.blob(0x558c574e2a80) get_ref init 0x2000, 1000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe8000~2000: 0x0~2000 Blob(0x558c574e2a80 blob([0xc0d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e639d0 sbid 0x0)) 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c57e639e8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.636+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57971ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57971ce0 blob([0xc0f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2a10 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c57971ce0) get_ref 0x0~2000 Blob(0x558c57971ce0 blob([0xc0f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2a10 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c57971ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xea000~2000: 0x0~2000 Blob(0x558c57971ce0 blob([0xc0f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2a10 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2a28 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829c540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a2b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829c540 blob([0xc11000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2b60 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c5829c540) get_ref 0x0~2000 Blob(0x558c5829c540 blob([0xc11000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2b60 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c5829c540) get_ref init 0x2000, 1000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xec000~2000: 0x0~2000 Blob(0x558c5829c540 blob([0xc11000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2b60 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580a2b78 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a2cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c55063260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a2cb0 blob([0xc13000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063260 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2cb0) get_ref 0x0~2000 Blob(0x558c580a2cb0 blob([0xc13000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c55063260 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xee000~2000: 0x0~2000 Blob(0x558c580a2cb0 blob([0xc13000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c55063260 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c55063278 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a2690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a2690 blob([0xc15000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2af0 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2690) get_ref 0x0~2000 Blob(0x558c580a2690 blob([0xc15000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a2af0 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c580a2690) get_ref init 0x2000, 1000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf0000~2000: 0x0~2000 Blob(0x558c580a2690 blob([0xc15000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a2af0 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c580a2b08 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829cc40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d030 blob([0xc17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829cc40 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d030) get_ref 0x0~1000 Blob(0x558c5829d030 blob([0xc17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829cc40 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d030) get_ref init 0x1000, 1000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf2000~1000: 0x0~1000 Blob(0x558c5829d030 blob([0xc17000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5829cc40 sbid 0x0)) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829cc58 in 0x558c53a59c00) _discard 0x0~1000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0xf3000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark shard 0x76000 dirty 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xd8000~1b000 = 0 2019-12-07T19:34:39.637+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.637+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000062.00000000000000000053' <- 0000000062.00000000000000000053 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bc2600 cost 172890 (15 ios * 4000 + 112890 bytes) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bc2600 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0x76000 is 1459 bytes (was 1196) from 64 extents 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 bluestore.extentmap(0x558c54e44888) reshard 0x[0,ffffffff) of 4 shards on 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard shards [0,4) over 0x[0,ffffffff) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard extent_avg 22, target 500, slop 100 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x0 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x2b000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x56000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0x81000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0xac000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard new shard 0xd8000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard new [0x0(0x0 bytes),0x2b000(0x0 bytes),0x56000(0x0 bytes),0x81000(0x0 bytes),0xac000(0x0 bytes),0xd8000(0x0 bytes)] 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard old [0x0(0x21d bytes),0x27000(0x221 bytes),0x4f000(0x234 bytes),0x76000(0x4ac bytes)] 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard fin [0x0(0x0 bytes),0x2b000(0x0 bytes),0x56000(0x0 bytes),0x81000(0x0 bytes),0xac000(0x0 bytes),0xd8000(0x0 bytes)] 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# force 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0x0 is 595 bytes (was 0) from 22 extents 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0x2b000 is 613 bytes (was 0) from 22 extents 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0x56000 is 602 bytes (was 0) from 22 extents 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0x81000 is 617 bytes (was 0) from 22 extents 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0xac000 is 423 bytes (was 0) from 22 extents 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0xd8000 is 268 bytes (was 0) from 14 extents 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 676 (674 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bc2600 allocated 0x[bfd000~1b000] released 0x[] 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 freelist allocate 0xbfd000~1b000 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 freelist _xor first_key 0xb80000 last_key 0xc00000 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2600 prepare 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58bc2600 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'53 (0'0,62'53] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'52 crt=62'50 lcod 62'52 active mbc={} ps=[20~1,23~1]] op_applied version 62'53 2019-12-07T19:34:39.637+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c54e78b00 finish 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2600 aio_wait 2019-12-07T19:34:39.637+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bc2600 2019-12-07T19:34:39.637+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2600 io_done 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2816 < min_flush_size 524288 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5736 < min_flush_size 524288 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5759 < min_flush_size 524288 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 5759 < min_flush_size 524288 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x53f500~167f to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x53f500 0x53f500~167f to file(ino 16 size 0x540000 mtime 2019-12-07T19:34:39.618069+0000 allocated 540000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~510000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _allocate len 0xb7f from 0 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _allocate len 0xb7f from 1 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd670000 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd670000~10000/10000,10000,acd670000 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x550000 to include allocated 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 79 (was clean) 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x50f500 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x500 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xb7f and padding block with 0x481 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x550000 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x540b7f 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (79) on file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]), flushing log 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 79 len 0x4a crc 0x359372c5) 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x4e000~1000 to file(ino 1 size 0x4e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x4e000 0x4e000~1000 to file(ino 1 size 0x4e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x4f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4e000 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x4f000 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:39.637+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000017 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 79 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000327697s (0.000036761s flush + 0.000290936s kv commit) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.637+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58bc2600> 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2600 kv_submitted 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bc2600 2019-12-07T19:34:39.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2600 finishing 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58bc2600 onodes 0x558c54e44780 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c55063278 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535d40 space 0x558c55063278 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2a28 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbb9e0 space 0x558c574e2a28 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2b08 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e141b0 space 0x558c574e2b08 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2cc8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e14510 space 0x558c574e2cc8 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac408 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e14120 space 0x558c576ac408 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c576ac558 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e14480 space 0x558c576ac558 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57971e48 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e143f0 space 0x558c57971e48 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e636d8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e14360 space 0x558c57e636d8 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e639e8 in 0x558c53a59c00) _finish_write discard buffer(0x558c54e14090 space 0x558c57e639e8 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 62/41 ECSubWriteReply(tid=123, last_complete=62'53, committed=1, applied=1)) v2 -- 0x558c58115600 con 0x558c54c5b400 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a2b08 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535dd0 space 0x558c580a2b08 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a2b78 in 0x558c53a59c00) _finish_write discard buffer(0x558c58534090 space 0x558c580a2b78 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58226088 in 0x558c53a59c00) _finish_write discard buffer(0x558c54dbbef0 space 0x558c58226088 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c582260f8 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535cb0 space 0x558c582260f8 0x0~2000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829cc58 in 0x558c53a59c00) _finish_write discard buffer(0x558c58535e60 space 0x558c5829cc58 0x0~1000 writing nocache) 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bc2600 done 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:39.637+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58bc2600 [] 2019-12-07T19:34:39.637+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.637+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.637+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.638+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 336 ==== MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=124, reqid=client.4540.0:459, at_version=62'54, trim_to=0'0, roll_forward_to=62'50)) v2 ==== 84568+0+0 (unknown 0 0 0) 0x558c58bce800 con 0x558c54c5b400 2019-12-07T19:34:39.638+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c54e79600 prio 127 cost 0 latency 0.000036 epoch 62 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=124, reqid=client.4540.0:459, at_version=62'54, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.638+0000 7f8f24c58700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=124, reqid=client.4540.0:459, at_version=62'54, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=124, reqid=client.4540.0:459, at_version=62'54, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=124, reqid=client.4540.0:459, at_version=62'54, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c54e79600 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=124, reqid=client.4540.0:459, at_version=62'54, trim_to=0'0, roll_forward_to=62'50)) v2 pg pg[1.bs1( v 62'53 (0'0,62'53] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'53 (0'0,62'53] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=124, reqid=client.4540.0:459, at_version=62'54, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'53 (0'0,62'53] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'53], crt=62'50) [62'54 (62'53) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:459 2019-12-07T19:34:39.635824+0000 0 ObjectCleanRegions clean_offsets: [0~3981312,4308992~18446744073705242623], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'54 (0'0,62'54] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'53 crt=62'50 lcod 62'53 active mbc={} ps=[20~1,23~1]] add_log_entry 62'54 (62'53) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:459 2019-12-07T19:34:39.635824+0000 0 ObjectCleanRegions clean_offsets: [0~3981312,4308992~18446744073705242623], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'54 (0'0,62'54] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'53 crt=62'50 lcod 62'53 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 54 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'54 (0'0,62'54] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'53 crt=62'50 lcod 62'53 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.638+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'54, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58bc2900 seq 66 2019-12-07T19:34:39.638+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:34:39.638+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:39.638+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xf3000~14000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0xf3000~14000 - have 0xf3000 (995328) bytes fadvise_flags 0x0 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0xf3000~14000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58bc2900 10 blobs 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate 0x14000/1000,14000,0 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc18000~14000/1000,14000,0 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc18000~14000] 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585e11f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e2bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585e11f0 blob([0xc18000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2bd0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c585e11f0) get_ref 0x0~2000 Blob(0x558c585e11f0 blob([0xc18000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e2bd0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c585e11f0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf3000~2000: 0x0~2000 Blob(0x558c585e11f0 blob([0xc18000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e2bd0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c574e2be8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829cfc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d500 blob([0xc1a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829cfc0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d500) get_ref 0x0~2000 Blob(0x558c5829d500 blob([0xc1a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829cfc0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d500) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf5000~2000: 0x0~2000 Blob(0x558c5829d500 blob([0xc1a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829cfc0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829cfd8 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829ce00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829d8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829ce00 blob([0xc1c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d8f0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829ce00) get_ref 0x0~2000 Blob(0x558c5829ce00 blob([0xc1c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d8f0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829ce00) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf7000~2000: 0x0~2000 Blob(0x558c5829ce00 blob([0xc1c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829d8f0 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829d908 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829d810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d880 blob([0xc1e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d810 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d880) get_ref 0x0~2000 Blob(0x558c5829d880 blob([0xc1e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d810 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d880) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xf9000~2000: 0x0~2000 Blob(0x558c5829d880 blob([0xc1e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829d810 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829d828 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829d730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d7a0 blob([0xc20000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d730 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d7a0) get_ref 0x0~2000 Blob(0x558c5829d7a0 blob([0xc20000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d730 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xfb000~2000: 0x0~2000 Blob(0x558c5829d7a0 blob([0xc20000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829d730 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829d748 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829d650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d6c0 blob([0xc22000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d650 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d6c0) get_ref 0x0~2000 Blob(0x558c5829d6c0 blob([0xc22000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d650 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xfd000~2000: 0x0~2000 Blob(0x558c5829d6c0 blob([0xc22000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829d650 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829d668 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829d5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829d570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829d5e0 blob([0xc24000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d570 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d5e0) get_ref 0x0~2000 Blob(0x558c5829d5e0 blob([0xc24000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d570 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829d5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xff000~2000: 0x0~2000 Blob(0x558c5829d5e0 blob([0xc24000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829d570 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829d588 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5809bb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829ce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5809bb20 blob([0xc26000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829ce70 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5809bb20) get_ref 0x0~2000 Blob(0x558c5809bb20 blob([0xc26000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829ce70 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5809bb20) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x101000~2000: 0x0~2000 Blob(0x558c5809bb20 blob([0xc26000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829ce70 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829ce88 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5829cee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829d960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5829cee0 blob([0xc28000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d960 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829cee0) get_ref 0x0~2000 Blob(0x558c5829cee0 blob([0xc28000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829d960 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c5829cee0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x103000~2000: 0x0~2000 Blob(0x558c5829cee0 blob([0xc28000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829d960 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c5829d978 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ea9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58bc4e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ea9a0 blob([0xc2a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc4e00 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c583ea9a0) get_ref 0x0~2000 Blob(0x558c583ea9a0 blob([0xc2a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc4e00 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.blob(0x558c583ea9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x105000~2000: 0x0~2000 Blob(0x558c583ea9a0 blob([0xc2a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58bc4e00 sbid 0x0)) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.BufferSpace(0x558c58bc4e18 in 0x558c53a59c00) _discard 0x0~2000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x107000 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) dirty_range mark shard 0xd8000 dirty 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0xf3000~14000 = 0 2019-12-07T19:34:39.638+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.638+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000062.00000000000000000054' <- 0000000062.00000000000000000054 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.638+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bc2900 cost 128218 (11 ios * 4000 + 84218 bytes) 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bc2900 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:39.638+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update shard 0xd8000 is 459 bytes (was 268) from 24 extents 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 676 (674 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bc2900 allocated 0x[c18000~14000] released 0x[] 2019-12-07T19:34:39.639+0000 7f8f0317f700 10 freelist allocate 0xc18000~14000 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 freelist _xor first_key 0xc00000 last_key 0xc00000 2019-12-07T19:34:39.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2900 prepare 2019-12-07T19:34:39.639+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58bc2900 2019-12-07T19:34:39.639+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'54 (0'0,62'54] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'53 crt=62'50 lcod 62'53 active mbc={} ps=[20~1,23~1]] op_applied version 62'54 2019-12-07T19:34:39.639+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c54e79600 finish 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2900 aio_wait 2019-12-07T19:34:39.639+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bc2900 2019-12-07T19:34:39.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2900 io_done 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2496 < min_flush_size 524288 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2519 < min_flush_size 524288 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2519 < min_flush_size 524288 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x540b7f~9d7 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x540b7f 0x540b7f~9d7 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x510b7f 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xb7f 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x556 and padding block with 0xaaa 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x541556 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000145098s (0.000040590s flush + 0.000104508s kv commit) 2019-12-07T19:34:39.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58bc2900> 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2900 kv_submitted 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bc2900 2019-12-07T19:34:39.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2900 finishing 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58bc2900 onodes 0x558c54e44780 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e2be8 in 0x558c53a59c00) _finish_write discard buffer(0x558c57c45950 space 0x558c574e2be8 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829ce88 in 0x558c53a59c00) _finish_write discard buffer(0x558c585ea3f0 space 0x558c5829ce88 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829cfd8 in 0x558c53a59c00) _finish_write discard buffer(0x558c580f3710 space 0x558c5829cfd8 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829d588 in 0x558c53a59c00) _finish_write discard buffer(0x558c585ea480 space 0x558c5829d588 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829d668 in 0x558c53a59c00) _finish_write discard buffer(0x558c585ea510 space 0x558c5829d668 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829d748 in 0x558c53a59c00) _finish_write discard buffer(0x558c58c8f710 space 0x558c5829d748 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829d828 in 0x558c53a59c00) _finish_write discard buffer(0x558c550e4630 space 0x558c5829d828 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829d908 in 0x558c53a59c00) _finish_write discard buffer(0x558c550e47e0 space 0x558c5829d908 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829d978 in 0x558c53a59c00) _finish_write discard buffer(0x558c585ea360 space 0x558c5829d978 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58bc4e18 in 0x558c53a59c00) _finish_write discard buffer(0x558c585ea2d0 space 0x558c58bc4e18 0x0~2000 writing nocache) 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bc2900 done 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:39.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58bc2900 [] 2019-12-07T19:34:39.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.639+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 62/41 ECSubWriteReply(tid=124, last_complete=62'54, committed=1, applied=1)) v2 -- 0x558c5795d000 con 0x558c54c5b400 2019-12-07T19:34:39.639+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.639+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 337 ==== MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=125, reqid=client.4540.0:460, at_version=62'55, trim_to=0'0, roll_forward_to=62'50)) v2 ==== 2616+0+0 (unknown 0 0 0) 0x558c58bcf000 con 0x558c54c5b400 2019-12-07T19:34:39.639+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000021 epoch 62 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=125, reqid=client.4540.0:460, at_version=62'55, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.639+0000 7f8f24c58700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=125, reqid=client.4540.0:460, at_version=62'55, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=125, reqid=client.4540.0:460, at_version=62'55, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.639+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=125, reqid=client.4540.0:460, at_version=62'55, trim_to=0'0, roll_forward_to=62'50)) v2) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.639+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c54e794a0 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=125, reqid=client.4540.0:460, at_version=62'55, trim_to=0'0, roll_forward_to=62'50)) v2 pg pg[1.bs1( v 62'54 (0'0,62'54] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'54 (0'0,62'54] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=125, reqid=client.4540.0:460, at_version=62'55, trim_to=0'0, roll_forward_to=62'50)) v2 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'54 (0'0,62'54] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'54], crt=62'50) [62'55 (62'54) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:460 2019-12-07T19:34:39.635848+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'54 crt=62'50 lcod 62'54 active mbc={} ps=[20~1,23~1]] add_log_entry 62'55 (62'54) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:460 2019-12-07T19:34:39.635848+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'54 crt=62'50 lcod 62'54 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 55 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'54 crt=62'50 lcod 62'54 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.640+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 62'55, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c579d2300 seq 67 2019-12-07T19:34:39.640+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys = 0 2019-12-07T19:34:39.640+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:39.640+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.0000000062.00000000000000000055' <- 0000000062.00000000000000000055 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._fastinfo' <- _fastinfo 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c579d2300 cost 6266 (1 ios * 4000 + 2266 bytes) 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c579d2300 onodes 0x558c54e44780 shared_blobs 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore.extentmap(0x558c54e44888) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 676 (674 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c579d2300 allocated 0x[] released 0x[] 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2300 prepare 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c579d2300 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2300 io_done 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 lua=62'54 crt=62'50 lcod 62'54 active mbc={} ps=[20~1,23~1]] op_applied version 62'55 2019-12-07T19:34:39.640+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c54e794a0 finish 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1937 < min_flush_size 524288 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c54e44780 had 1 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1960 < min_flush_size 524288 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1960 < min_flush_size 524288 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x541556~7a8 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x541556 0x541556~7a8 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x511556 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x556 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xcfe and padding block with 0x302 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x541cfe 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000572180s (0.000000112s flush + 0.000572068s kv commit) 2019-12-07T19:34:39.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c579d2300> 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2300 kv_submitted 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c579d2300 2019-12-07T19:34:39.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c579d2300 finishing 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c579d2300 onodes 0x558c54e44780 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c579d2300 done 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:39.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c579d2300 [] 2019-12-07T19:34:39.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.640+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.640+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 62/41 ECSubWriteReply(tid=125, last_complete=62'55, committed=1, applied=1)) v2 -- 0x558c5795d800 con 0x558c54c5b400 2019-12-07T19:34:39.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.643+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 338 ==== MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=126, reqid=client.4540.0:460, at_version=0'0, trim_to=0'0, roll_forward_to=62'55)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558c58bcf800 con 0x558c54c5b400 2019-12-07T19:34:39.643+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c58bd4000 prio 127 cost 0 latency 0.000023 epoch 62 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=126, reqid=client.4540.0:460, at_version=0'0, trim_to=0'0, roll_forward_to=62'55)) v2 2019-12-07T19:34:39.643+0000 7f8f24c58700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=126, reqid=client.4540.0:460, at_version=0'0, trim_to=0'0, roll_forward_to=62'55)) v2) prio 127 cost 0 e62) 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=126, reqid=client.4540.0:460, at_version=0'0, trim_to=0'0, roll_forward_to=62'55)) v2) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=126, reqid=client.4540.0:460, at_version=0'0, trim_to=0'0, roll_forward_to=62'55)) v2) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.643+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c58bd4000 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=126, reqid=client.4540.0:460, at_version=0'0, trim_to=0'0, roll_forward_to=62'55)) v2 pg pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.bs1 62/41 ECSubWrite(tid=126, reqid=client.4540.0:460, at_version=0'0, trim_to=0'0, roll_forward_to=62'55)) v2 2019-12-07T19:34:39.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'55], crt=62'50) [] 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'51 (62'50) clone 1:d07269f7:::smithi13140796-28:26 by unknown.0.0:0 2019-12-07T19:34:35.633784+0000 0 snaps [26,25] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'52 (62'50) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:457 2019-12-07T19:34:39.631098+0000 0 ObjectCleanRegions clean_offsets: [0~2818048,3538944~18446744073706012671], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'53 (62'52) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:458 2019-12-07T19:34:39.633634+0000 0 ObjectCleanRegions clean_offsets: [0~3538944,3981312~18446744073705570303], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'54 (62'53) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:459 2019-12-07T19:34:39.635824+0000 0 ObjectCleanRegions clean_offsets: [0~3981312,4308992~18446744073705242623], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.643+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] rollforward: entry=62'55 (62'54) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:460 2019-12-07T19:34:39.635848+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:39.643+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 55 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:39.644+0000 7f8f0317f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d9e3c0 1.bs1_head 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57c6f540 = 0x558c58bc2f00 seq 68 2019-12-07T19:34:39.644+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'._info' <- _info 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000499'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bc2f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bc2f00 onodes shared_blobs 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bc2f00 allocated 0x[] released 0x[] 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2f00 prepare 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bc2f00 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2f00 io_done 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c58bd4000 finish 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57572500 had 1 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x541cfe~471 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x541cfe 0x541cfe~471 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x511cfe 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xcfe 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x16f and padding block with 0xe91 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x54216f 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:39.644+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000119883s (0.000000082s flush + 0.000119801s kv commit) 2019-12-07T19:34:39.644+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58bc2f00> 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:39.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2f00 kv_submitted 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bc2f00 2019-12-07T19:34:39.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bc2f00 finishing 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58bc2f00 onodes 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bc2f00 done 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57c6f540 q now empty 2019-12-07T19:34:39.644+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58bc2f00 [] 2019-12-07T19:34:39.644+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.644+0000 7f8f07187700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 62/41 ECSubWriteReply(tid=126, last_complete=62'55, committed=1, applied=1)) v2 -- 0x558c5795d200 con 0x558c54c5b400 2019-12-07T19:34:39.644+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:39.644+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.644+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 339 ==== MOSDECSubOpRead(1.bs1 62/41 ECSubRead(tid=127, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558c57976280 con 0x558c54c5b400 2019-12-07T19:34:39.644+0000 7f8f24c58700 15 osd.7 62 enqueue_op 0x558c58bd42c0 prio 127 cost 0 latency 0.000021 epoch 62 MOSDECSubOpRead(1.bs1 62/41 ECSubRead(tid=127, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:39.644+0000 7f8f24c58700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 62/41 ECSubRead(tid=127, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e62) 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 62/41 ECSubRead(tid=127, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e62) queued 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 62/41 ECSubRead(tid=127, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e62) pg 0x558c575ef000 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c58bd42c0 prio 127 cost 0 latency 0.000052 MOSDECSubOpRead(1.bs1 62/41 ECSubRead(tid=127, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpRead(1.bs1 62/41 ECSubRead(tid=127, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:39.644+0000 7f8f0317f700 25 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:39.644+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1000 size 0x107000 (1077248) 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5840f2d0 blob([0x999000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580d79d0 loaded (sbid 0x2a0e ref_map(0x999000~2000=3)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 = 4096 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] get_hash_info: found on disk, size 1077248 2019-12-07T19:34:39.644+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:34:39.644+0000 7f8f0317f700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 62/41 ECSubReadReply(tid=127, attrs_read=0)) v2 -- 0x558c5895c000 con 0x558c54c5b400 2019-12-07T19:34:39.644+0000 7f8f0317f700 10 osd.7 62 dequeue_op 0x558c58bd42c0 finish 2019-12-07T19:34:39.644+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:39.651+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.702+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:34:39.702+0000 7f8f191ab700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock osr 0x558c54ea4b40 2 ios pending 2019-12-07T19:34:39.702+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 52 0x2000~1000 2019-12-07T19:34:39.702+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x2000~1000 crc 72d8b7b6 2019-12-07T19:34:39.702+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock seq 52 0x1b000~1000 2019-12-07T19:34:39.702+0000 7f8f191ab700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_submit_unlock write 0x1b000~1000 crc ada932ca 2019-12-07T19:34:39.702+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.702+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish osr 0x558c54ea4b40 2019-12-07T19:34:39.702+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_aio_finish dequeueing 2019-12-07T19:34:39.752+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 10 osd.7 62 tick_without_osd_lock 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f3d4000/0x40000000/0x1680000000, data 0xeac0ca/0xc2a000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 25 osd.7 62 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476453+0000 last_rx_front 2019-12-07T19:34:38.476418+0000 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 25 osd.7 62 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476352+0000 last_rx_front 2019-12-07T19:34:38.476394+0000 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 25 osd.7 62 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476378+0000 last_rx_front 2019-12-07T19:34:38.476470+0000 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 25 osd.7 62 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476367+0000 last_rx_front 2019-12-07T19:34:38.476505+0000 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 25 osd.7 62 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476409+0000 last_rx_front 2019-12-07T19:34:38.476432+0000 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 20 osd.7 62 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 10 osd.7 62 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 20 osd.7 62 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:39.782+0000 7f8f1fbbe700 10 osd.7 62 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:39.802+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.852+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.903+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:39.953+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.003+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.053+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.103+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.153+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.186+0000 7f8f21451700 10 osd.7 62 tick 2019-12-07T19:34:40.186+0000 7f8f21451700 10 osd.7 62 do_waiters -- start 2019-12-07T19:34:40.187+0000 7f8f21451700 10 osd.7 62 do_waiters -- finish 2019-12-07T19:34:40.187+0000 7f8f21451700 20 osd.7 62 tick last_purged_snaps_scrub 2019-12-07T19:34:38.205126+0000 next 2019-12-07T19:35:38.205126+0000 2019-12-07T19:34:40.202+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.253+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.304+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.330+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:40.354+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.404+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.454+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.503+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.524+0000 7f8f1219d700 20 osd.7 62 reports for 0 queries 2019-12-07T19:34:40.524+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.7 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558c58c69c00 con 0x558c5476d800 2019-12-07T19:34:40.524+0000 7f8f1219d700 20 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfs pool 1 2019-12-07T19:34:40.524+0000 7f8f1219d700 10 bluestore(/var/lib/ceph/osd/ceph-7) pool_statfsstore_statfs(0x0/0x0/0x0, data 0xe5f000/0xb7a000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:40.524+0000 7f8f1219d700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(3 pgs tid 0 v 0) v2 -- 0x558c5489f500 con 0x558c5476d800 2019-12-07T19:34:40.546+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.2 172.21.15.131:0/37077 20 ==== osd_ping(ping e62 up_from 13 ping_stamp 2019-12-07T19:34:40.546975+0000/64.018694132s send_stamp 64.018694132s delta_ub 0.036551779s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c54d76400 con 0x558c552d1c00 2019-12-07T19:34:40.546+0000 7f8f24457700 20 osd.7 62 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036104986s,0.036551779s]) 2019-12-07T19:34:40.546+0000 7f8f24457700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e62 up_from 13 ping_stamp 2019-12-07T19:34:40.546975+0000/64.018694132s send_stamp 63.982589146s delta_ub 18446744073.673446630s) v5 -- 0x558c58513c00 con 0x558c552d1c00 2019-12-07T19:34:40.546+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.2 172.21.15.131:0/37077 20 ==== osd_ping(ping e62 up_from 13 ping_stamp 2019-12-07T19:34:40.546975+0000/64.018694132s send_stamp 64.018694132s delta_ub 0.036551779s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c552d1800 2019-12-07T19:34:40.546+0000 7f8f24c58700 20 osd.7 62 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.036019612s,0.036551779s]) 2019-12-07T19:34:40.546+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e62 up_from 13 ping_stamp 2019-12-07T19:34:40.546975+0000/64.018694132s send_stamp 63.982674520s delta_ub 18446744073.673532004s) v5 -- 0x558c5795d800 con 0x558c552d1800 2019-12-07T19:34:40.554+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.604+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1552581 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.638+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 319 ==== osd_map(63..63 src has 1..63) v4 ==== 633+0+0 (crc 0 0 0) 0x558c58bb6d80 con 0x558c54c5bc00 2019-12-07T19:34:40.638+0000 7f8f1199c700 20 osd.7 62 OSD::ms_dispatch: osd_map(63..63 src has 1..63) v4 2019-12-07T19:34:40.638+0000 7f8f1199c700 10 osd.7 62 do_waiters -- start 2019-12-07T19:34:40.638+0000 7f8f1199c700 10 osd.7 62 do_waiters -- finish 2019-12-07T19:34:40.638+0000 7f8f1199c700 20 osd.7 62 _dispatch 0x558c58bb6d80 osd_map(63..63 src has 1..63) v4 2019-12-07T19:34:40.638+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 320 ==== MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=163, reqid=client.4540.0:466, at_version=63'79, trim_to=0'0, roll_forward_to=62'77)) v2 ==== 3319+0+0 (crc 0 0 0) 0x558c575c1800 con 0x558c54c5bc00 2019-12-07T19:34:40.638+0000 7f8f23c56700 15 osd.7 62 enqueue_op 0x558c5511f8c0 prio 127 cost 0 latency 0.000041 epoch 63 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=163, reqid=client.4540.0:466, at_version=63'79, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:40.638+0000 7f8f1199c700 3 osd.7 62 handle_osd_map epochs [63,63], i have 62, src has [1,63] 2019-12-07T19:34:40.638+0000 7f8f1199c700 10 osd.7 62 handle_osd_map got inc map for epoch 63 2019-12-07T19:34:40.638+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=163, reqid=client.4540.0:466, at_version=63'79, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.638+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3404e826:::osdmap.62:0# 0x0~0 2019-12-07T19:34:40.638+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:40.638+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:40.638+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c58226000 blob([0xbcf000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57a07260 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:40.638+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57a07490 blob([!~1000,0xbd0000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558c57a076c0 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:40.638+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3404e826:::osdmap.62:0# 0x0~1454 = 5204 2019-12-07T19:34:40.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.638+0000 7f8f1199c700 10 osd.7 62 add_map_bl 62 5204 bytes 2019-12-07T19:34:40.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=163, reqid=client.4540.0:466, at_version=63'79, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.638+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=163, reqid=client.4540.0:466, at_version=63'79, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) pg 0x558c54d5c000 2019-12-07T19:34:40.638+0000 7f8f05984700 10 osd.7 62 dequeue_op 0x558c5511f8c0 prio 127 cost 0 latency 0.000093 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=163, reqid=client.4540.0:466, at_version=63'79, trim_to=0'0, roll_forward_to=62'77)) v2 pg pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.638+0000 7f8f05984700 10 osd.7 62 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 62 -> 63 (as per caller) 2019-12-07T19:34:40.638+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=163, reqid=client.4540.0:466, at_version=63'79, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:40.639+0000 7f8f05984700 20 snap_mapper.add_oid 1:6c205177:::smithi13140796-8:28 28 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 snap_mapper.get_snaps 1:6c205177:::smithi13140796-8:28 got.empty() 2019-12-07T19:34:40.639+0000 7f8f05984700 20 snap_mapper.set_snaps 1:6c205177:::smithi13140796-8:28 28 2019-12-07T19:34:40.639+0000 7f8f05984700 20 snap_mapper.set_snaps set OBJ_.2_0000000000000001.6340A8EE.28.smithi13140796-8.. 2019-12-07T19:34:40.639+0000 7f8f05984700 20 snap_mapper.add_oid set SNA_1_0000000000000028_.2_0000000000000001.6340A8EE.28.smithi13140796-8.. 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 osd.7 62 got_full_map 63, nothing requested 2019-12-07T19:34:40.639+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 62'77 (0'0,62'77] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'77], crt=62'77) [63'78 (62'77) clone 1:6c205177:::smithi13140796-8:28 by unknown.0.0:0 2019-12-07T19:34:38.611669+0000 0 snaps [28] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,63'79 (62'77) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:466 2019-12-07T19:34:40.638217+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0] 2019-12-07T19:34:40.639+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 63'78 (0'0,63'78] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=62'77 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] add_log_entry 63'78 (62'77) clone 1:6c205177:::smithi13140796-8:28 by unknown.0.0:0 2019-12-07T19:34:38.611669+0000 0 snaps [28] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 osd.7 62 handle_osd_map pg_num_history pg_num_history(e63 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps purged_snaps {63={}} 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:40.639+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=62'77 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] add_log_entry 63'79 (62'77) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:466 2019-12-07T19:34:40.638217+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 osd.7 62 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e63 [1,63] lci=[13,63]) 2019-12-07T19:34:40.639+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=62'77 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 79 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c58bb2000 seq 55 2019-12-07T19:34:40.639+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=62'77 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.639+0000 7f8f05984700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.639+0000 7f8f05984700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'78, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:c71f972f:::inc_osdmap.63:0# key 0x7f7fffffffffffffffc71f97'/!inc_osdmap.63!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:40.639+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c71f972f:::inc_osdmap.63:0# 0x0~24d 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1291 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:c71f972f:::inc_osdmap.63:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58bb2000 1 blobs 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc2c000~1000/1000,1000,0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc2c000~1000] 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58b79110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792dab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58b79110 blob([0xc2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792dab0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c58b79110) get_ref 0x0~24d Blob(0x558c58b79110 blob([0xc2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792dab0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c58b79110) get_ref init 0x1000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c58b79110 blob([0xc2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5792dab0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c5792dac8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58bd7c88) dirty_range mark inline shard dirty 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:c71f972f:::inc_osdmap.63:0# 0x0~24d = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c5876c300 seq 119 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3f84e826:::osdmap.63:0# key 0x7f7fffffffffffffff3f84e8'&!osdmap.63!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:28# key 0x8280000000000000016c2051'w!smithi13140796-8!='0x0000000000000028ffffffffffffffff'o' 2019-12-07T19:34:40.639+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3f84e826:::osdmap.63:0# 0x0~1438 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1292 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3f84e826:::osdmap.63:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~438 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58bb2000 2 blobs 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc2d000~2000/1000,2000,0 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:28# 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc2d000~2000] 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1293 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.onode(0x558c579df680).flush flush done 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580f7e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f7ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:28# 0x0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580f7e30 blob([0xc2d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7ea0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c580f7e30) get_ref 0x0~1000 Blob(0x558c580f7e30 blob([0xc2d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7ea0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c580f7e30) get_ref init 0x1000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:28# 0x0~5f000 -> 0x0~5f000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c580f7e30 blob([0xc2d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580f7ea0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c580f7eb8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579dc310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579dc310 blob([!~1000,0xc2e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c70 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x0~2000: 0x0~2000 Blob(0x558c54e56f50 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c579dc310) get_ref 0x1000~438 Blob(0x558c579dc310 blob([!~1000,0xc2e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c57ff5c70 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c579dc310) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558c579dc310 blob([!~1000,0xc2e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c57ff5c70 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c57ff5c88 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c58766e70 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1438 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58766e70) get_ref 0x0~2000 Blob(0x558c58766e70 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58bd8108) dirty_range mark inline shard dirty 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58766e70) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3f84e826:::osdmap.63:0# 0x0~1438 = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x0~2000: 0x0~2000 Blob(0x558c58766e70 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x2000~2000: 0x0~2000 Blob(0x558c54d3c310 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c57881500 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c57881500) get_ref 0x0~2000 Blob(0x558c57881500 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c57881500) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57881500 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x4000~2000: 0x0~2000 Blob(0x558c5792dce0 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58bb2000 0 blobs 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c585e0a80 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c585e0a80) get_ref 0x0~2000 Blob(0x558c585e0a80 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c585e0a80) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c585e0a80 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x6000~2000: 0x0~2000 Blob(0x558c550b3730 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c55279260 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c55279260) get_ref 0x0~2000 Blob(0x558c55279260 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c55279260) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c55279260 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x8000~2000: 0x0~2000 Blob(0x558c54d750a0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58bb2000 0 blobs 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c58766ee0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58766ee0) get_ref 0x0~2000 Blob(0x558c58766ee0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58766ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58bb2000 cost 23246 (4 ios * 4000 + 7246 bytes) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58bb2000 onodes 0x558c53a91900,0x558c55276c80,0x558c58bd7b80,0x558c58bd8000 shared_blobs 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c58766ee0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0xa000~2000: 0x0~2000 Blob(0x558c580d7e30 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c580f7f80 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c580f7f80) get_ref 0x0~2000 Blob(0x558c580f7f80 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c580f7f80) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58bd7c88) update #-1:c71f972f:::inc_osdmap.63:0# 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58bd7c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:c71f972f:::inc_osdmap.63:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c580f7f80 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58bd8108) update #-1:3f84e826:::osdmap.63:0# 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore.extentmap(0x558c58bd8108) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3f84e826:::osdmap.63:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0xc000~2000: 0x0~2000 Blob(0x558c574d28c0 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579d6f50 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579d6f50) get_ref 0x0~2000 Blob(0x558c579d6f50 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58bb2000 allocated 0x[c2c000~3000] released 0x[] 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579d6f50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 freelist allocate 0xc2c000~3000 2019-12-07T19:34:40.639+0000 7f8f1199c700 20 freelist _xor first_key 0xc00000 last_key 0xc00000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c579d6f50 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0xe000~2000: 0x0~2000 Blob(0x558c575f7c70 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c5794c9a0 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c5794c9a0) get_ref 0x0~2000 Blob(0x558c5794c9a0 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c5794c9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb2000 prepare 2019-12-07T19:34:40.639+0000 7f8f1199c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58bb2000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c5794c9a0 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x10000~2000: 0x0~2000 Blob(0x558c5829c000 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c58b79030 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58b79030) get_ref 0x0~2000 Blob(0x558c58b79030 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58b79030) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c58b79030 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x12000~1000: 0x0~1000 Blob(0x558c547402a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c58b78a80 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58b78a80) get_ref 0x0~1000 Blob(0x558c58b78a80 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c58b78a80) get_ref init 0x1000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x12000~1000: 0x0~1000 Blob(0x558c58b78a80 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x13000~2000: 0x0~2000 Blob(0x558c57efd420 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c550b27e0 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b27e0) get_ref 0x0~2000 Blob(0x558c550b27e0 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b27e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x13000~2000: 0x0~2000 Blob(0x558c550b27e0 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x15000~2000: 0x0~2000 Blob(0x558c57c42930 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579d6ee0 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579d6ee0) get_ref 0x0~2000 Blob(0x558c579d6ee0 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579d6ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x15000~2000: 0x0~2000 Blob(0x558c579d6ee0 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x17000~2000: 0x0~2000 Blob(0x558c581d6bd0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c5792d9d0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c5792d9d0) get_ref 0x0~2000 Blob(0x558c5792d9d0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c5792d9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x17000~2000: 0x0~2000 Blob(0x558c5792d9d0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x19000~2000: 0x0~2000 Blob(0x558c57e63e30 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c5792da40 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c5792da40) get_ref 0x0~2000 Blob(0x558c5792da40 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c5792da40) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x19000~2000: 0x0~2000 Blob(0x558c5792da40 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x1b000~2000: 0x0~2000 Blob(0x558c57efd490 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c550b29a0 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b29a0) get_ref 0x0~2000 Blob(0x558c550b29a0 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b29a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x1b000~2000: 0x0~2000 Blob(0x558c550b29a0 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x1d000~2000: 0x0~2000 Blob(0x558c57a07dc0 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c580f7f10 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c580f7f10) get_ref 0x0~2000 Blob(0x558c580f7f10 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c580f7f10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x1d000~2000: 0x0~2000 Blob(0x558c580f7f10 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x1f000~2000: 0x0~2000 Blob(0x558c581d6cb0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c578f1dc0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c578f1dc0) get_ref 0x0~2000 Blob(0x558c578f1dc0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c578f1dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x1f000~2000: 0x0~2000 Blob(0x558c578f1dc0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x21000~2000: 0x0~2000 Blob(0x558c54d3df10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c550b2a10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b2a10) get_ref 0x0~2000 Blob(0x558c550b2a10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b2a10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x21000~2000: 0x0~2000 Blob(0x558c550b2a10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x23000~2000: 0x0~2000 Blob(0x558c57efd500 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c550b2b60 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b2b60) get_ref 0x0~2000 Blob(0x558c550b2b60 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c550b2b60 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x25000~2000: 0x0~2000 Blob(0x558c57efd1f0 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c580d7500 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c580d7500) get_ref 0x0~2000 Blob(0x558c580d7500 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c580d7500) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c580d7500 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x27000~2000: 0x0~2000 Blob(0x558c580d7f80 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c581d7e30 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c581d7e30) get_ref 0x0~2000 Blob(0x558c581d7e30 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c581d7e30) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c581d7e30 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x29000~2000: 0x0~2000 Blob(0x558c57efd260 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c581d7ea0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c581d7ea0) get_ref 0x0~2000 Blob(0x558c581d7ea0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c581d7ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c581d7ea0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb2000 aio_wait 2019-12-07T19:34:40.639+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58bb2000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c5829c7e0 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c581d7f80 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c581d7f80) get_ref 0x0~2000 Blob(0x558c581d7f80 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c581d7f80) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb2000 io_done 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c581d7f80 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c581d6d90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dce00 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dce00) get_ref 0x0~2000 Blob(0x558c579dce00 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dce00) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c579dce00 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x2f000~1000: 0x1000~1000 Blob(0x558c585e16c0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dce70 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dce70) get_ref 0x1000~1000 Blob(0x558c579dce70 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dce70) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x2f000~1000: 0x1000~1000 Blob(0x558c579dce70 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x30000~2000: 0x0~2000 Blob(0x558c585e1730 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dcee0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dcee0) get_ref 0x0~2000 Blob(0x558c579dcee0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dcee0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c579dcee0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x32000~2000: 0x0~2000 Blob(0x558c585e17a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dcf50 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dcf50) get_ref 0x0~2000 Blob(0x558c579dcf50 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dcf50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c579dcf50 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x34000~2000: 0x0~2000 Blob(0x558c585e1810 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dcfc0 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dcfc0) get_ref 0x0~2000 Blob(0x558c579dcfc0 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dcfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c579dcfc0 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x36000~2000: 0x0~2000 Blob(0x558c585e1880 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd030 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd030) get_ref 0x0~2000 Blob(0x558c579dd030 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd030) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c579dd030 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x38000~2000: 0x0~2000 Blob(0x558c585e18f0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd0a0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd0a0) get_ref 0x0~2000 Blob(0x558c579dd0a0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c579dd0a0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c585e1960 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd110 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd110) get_ref 0x0~2000 Blob(0x558c579dd110 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd110) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c579dd110 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c585e19d0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd180 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd180) get_ref 0x0~2000 Blob(0x558c579dd180 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd180) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c579dd180 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c585e1a40 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c550b2930 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b2930) get_ref 0x0~2000 Blob(0x558c550b2930 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c550b2930) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c550b2930 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x40000~2000: 0x0~2000 Blob(0x558c585e1ab0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd1f0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd1f0) get_ref 0x0~2000 Blob(0x558c579dd1f0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c579dd1f0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58bd7b80 had 1 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c58bd8000 had 1 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x42000~2000: 0x0~2000 Blob(0x558c585e1b20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd260 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd260) get_ref 0x0~2000 Blob(0x558c579dd260 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd260) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c579dd260 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x44000~2000: 0x0~2000 Blob(0x558c585e1b90 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3855286272 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd2d0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd2d0) get_ref 0x0~2000 Blob(0x558c579dd2d0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c579dd2d0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x46000~2000: 0x0~2000 Blob(0x558c585e1c00 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd340 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd340) get_ref 0x0~2000 Blob(0x558c579dd340 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd340) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997355) bluestore 86 GiB free (0.959867), bluefs_ratio 0.0399041 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c579dd340 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x48000~2000: 0x0~2000 Blob(0x558c585e1c70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd3b0 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd3b0) get_ref 0x0~2000 Blob(0x558c579dd3b0 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c579dd3b0 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c585e1ce0 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd420 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd420) get_ref 0x0~2000 Blob(0x558c579dd420 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd420) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c579dd420 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8860 < min_flush_size 524288 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c585e1d50 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8860 < min_flush_size 524288 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd490 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd490) get_ref 0x0~2000 Blob(0x558c579dd490 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd490) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c579dd490 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c585e1dc0 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd500 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd500) get_ref 0x0~2000 Blob(0x558c579dd500 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd500) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x54216f~229c to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c579dd500 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x50000~2000: 0x0~2000 Blob(0x558c585e1e30 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x54216f 0x54216f~229c to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd570 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd570) get_ref 0x0~2000 Blob(0x558c579dd570 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd570) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c579dd570 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x51216f 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x16f 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x52000~2000: 0x0~2000 Blob(0x558c585e1ea0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x40b and padding block with 0xbf5 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd5e0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd5e0) get_ref 0x0~2000 Blob(0x558c579dd5e0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c579dd5e0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x54000~2000: 0x0~2000 Blob(0x558c585e1f10 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd650 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd650) get_ref 0x0~2000 Blob(0x558c579dd650 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd650) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c579dd650 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x56000~2000: 0x0~2000 Blob(0x558c585e1f80 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd6c0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd6c0) get_ref 0x0~2000 Blob(0x558c579dd6c0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c579dd6c0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x58000~2000: 0x0~2000 Blob(0x558c550b0000 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd730 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd730) get_ref 0x0~2000 Blob(0x558c579dd730 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd730) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c579dd730 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c550b0070 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd7a0 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd7a0) get_ref 0x0~2000 Blob(0x558c579dd7a0 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c579dd7a0 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c550b00e0 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd810 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd810) get_ref 0x0~2000 Blob(0x558c579dd810 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd810) get_ref init 0x2000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c579dd810 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup src 0x5e000~1000: 0x0~1000 Blob(0x558c550b0150 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=2)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup new Blob(0x558c579dd880 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd880) get_ref 0x0~1000 Blob(0x558c579dd880 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.blob(0x558c579dd880) get_ref init 0x1000, 1000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) dup dst 0x5e000~1000: 0x0~1000 Blob(0x558c579dd880 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) dirty_range mark inline shard dirty 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:28# = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:28# snapset 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x54440b 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:28# snapset = 0 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.639+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:28# 1 keys 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:28# 1 keys = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:28# hinfo_key (42 bytes) 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:28# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 1 in 0.000210148s (0.000044092s flush + 0.000166056s kv commit) 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:40.639+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.2_0000000000000001.6340A8EE.28.smithi13140796-8..' <- OBJ_.2_0000000000000001.6340A8EE.28.smithi13140796-8.. 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000028_.2_0000000000000001.6340A8EE.28.smithi13140796-8..' <- SNA_1_0000000000000028_.2_0000000000000001.6340A8EE.28.smithi13140796-8.. 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58bb2000> 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <0x558c58141400> 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58bb2000 kv_submitted 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58bb2000 2019-12-07T19:34:40.639+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000063.00000000000000000078' <- 0000000063.00000000000000000078 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c58bb2000 osr 0x558c54ea4b40 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000063.00000000000000000079' <- 0000000063.00000000000000000079 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 53 0x1b000~1000 crc b7769e37 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 53 0x2000~1000 crc c9e3a38b 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6c00 deferred_cleanup 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5876c300 cost 8345 (1 ios * 4000 + 4345 bytes) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b6c00 onodes 0x558c53a91900,0x558c55276c80,0x558c58768280,0x558c58768500 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7 62 _committed_osd_maps 63..63 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5757e168 in 0x558c53a59340) _finish_write discard buffer(0x558c580f2f30 space 0x558c5757e168 0x0~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5876c300 onodes 0x558c579de780,0x558c579df680 shared_blobs 0x558c54df8000,0x558c54df8af0,0x558c54df8bd0,0x558c54df8cb0,0x558c54df8d90,0x558c54df8e70,0x558c54df8f50,0x558c54df9030,0x558c54df9110,0x558c54df91f0,0x558c54df9ab0,0x558c54df9c00,0x558c54df9ce0,0x558c54df9dc0,0x558c54e43ce0,0x558c54e56690,0x558c54e56bd0,0x558c54e56cb0,0x558c54e56d90,0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c573970a0,0x558c575bd3b0,0x558c577f3a40,0x558c577f3c70,0x558c57e63490,0x558c57efd5e0,0x558c57efd6c0,0x558c57efd7a0,0x558c57efd880,0x558c57efd960,0x558c57efda40,0x558c57efdb20,0x558c57efdc00,0x558c5809b490,0x558c5809b500,0x558c5809b570,0x558c5809bc70,0x558c5809bd50,0x558c5809bdc0,0x558c5809bf80,0x558c580a4a10 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7 62 advance to epoch 63 (<= last 63 <= newest_map 63) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) update 2#1:6c205177:::smithi13140796-8:28# 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c579d7978 in 0x558c53a59340) _finish_write discard buffer(0x558c580f2fc0 space 0x558c579d7978 0x0~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f70b8 in 0x558c53a59340) _finish_write discard buffer(0x558c580f3050 space 0x558c580f70b8 0x1000~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6c00 done 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584fc000 deferred_cleanup 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) update inline shard 1335 bytes from 49 extents 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b6c00 [] 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore.extentmap(0x558c579de888) reshard 0x[0,ffffffff) of 0 shards on 2#1:6c205177:::smithi13140796-8:28# 2019-12-07T19:34:40.639+0000 7f8f141a1700 7 osd.7 63 consume_map version 63 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard new shard 0x0 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.8s5 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard new shard 0x23000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard new shard 0x46000 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.0s3 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard old [] 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.1s1 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.9s4 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584fc000 deferred_cleanup 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.as5 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584fc000 onodes 0x558c53a91900 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.2s0 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) update 2#1:6c205177:::smithi13140796-8:28# force 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584fc000 done 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.bs1 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6f00 deferred_cleanup 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.3s4 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584fc000 [] 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.cs0 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) update shard 0x23000 is 505 bytes (was 0) from 18 extents 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.4s0 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c582b6f00 deferred_cleanup 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579de888) update shard 0x46000 is 353 bytes (was 0) from 13 extents 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.5s4 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c582b6f00 onodes 0x558c53a91900,0x558c55276c80,0x558c579dec80,0x558c579def00 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.ds2 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f7278 in 0x558c53a59340) _finish_write discard buffer(0x558c580f3290 space 0x558c580f7278 0x0~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.es4 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.6s2 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f7358 in 0x558c53a59340) _finish_write discard buffer(0x558c580f3320 space 0x558c580f7358 0x1000~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.fs5 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f7c88 in 0x558c53a59340) _finish_write discard buffer(0x558c580f3200 space 0x558c580f7c88 0x0~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c582b6f00 done 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578aa600 deferred_cleanup 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c582b6f00 [] 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 identify_splits_and_merges 1.7s4 e62 to e63 pg_nums {18=16} 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:28# is 425 (423 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 63 prune_pg_created 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:0.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore.extentmap(0x558c579df788) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:1.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 666 (664 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c578aa600 deferred_cleanup 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aba is 11 SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c578aa600 onodes 0x558c53a91900,0x558c55276c80,0x558c5851f680,0x558c5851f900 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:2.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abd is 11 SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:3.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:4.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:5.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abe is 11 SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a07278 in 0x558c53a59340) _finish_write discard buffer(0x558c58534f30 space 0x558c57a07278 0x0~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abf is 11 SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:6.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a076d8 in 0x558c53a59340) _finish_write discard buffer(0x558c58534fc0 space 0x558c57a076d8 0x1000~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac0 is 11 SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac1 is 11 SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829cb78 in 0x558c53a59340) _finish_write discard buffer(0x558c58534ea0 space 0x558c5829cb78 0x0~1000 writing nocache) 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7:7.consume_map 63 (was 62) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac2 is 11 SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c578aa600 done 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac3 is 11 SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58bb2000 deferred_queued 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac4 is 11 SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c578aa600 [] 2019-12-07T19:34:40.639+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac5 is 11 SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acd is 11 SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ace is 11 SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acf is 11 SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad0 is 11 SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7 63 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab8 is 11 SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab4 is 11 SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab7 is 11 SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab6 is 11 SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab5 is 11 SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.8s5 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.0s3 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.1s1 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac7 is 11 SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.9s4 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac6 is 11 SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab9 is 11 SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab2 is 11 SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c57380000 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.as5 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaa is 11 SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aab is 11 SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aac is 11 SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aad is 11 SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.2s0 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aae is 11 SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaf is 11 SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab0 is 11 SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab1 is 11 SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abc is 11 SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abb is 11 SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acc is 11 SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c57e8b000 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aca is 11 SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac8 is 11 SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac9 is 11 SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acb is 11 SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.8s5 62 -> 63 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab3 is 11 SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3))) 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5876c300 allocated 0x[] released 0x[] 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 62 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c300 prepare 2019-12-07T19:34:40.639+0000 7f8f05984700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5876c300 2019-12-07T19:34:40.639+0000 7f8f05984700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c300 io_done 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.639+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.bs1 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.as5 62 -> 63 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.639+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:40.639+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=62'77 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] op_applied version 63'79 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.3s4 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f07988700 10 osd.7 pg_epoch: 62 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.cs0 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f05984700 10 osd.7 63 dequeue_op 0x558c5511f8c0 finish 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.639+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.4s0 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.5s4 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.639+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.ds2 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.es4 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.639+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c57676000 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.6s2 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.fs5 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f141a1700 15 osd.7 63 enqueue_peering_evt 1.7s4 epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.639+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7 63 maybe_update_heartbeat_peers forcing update after 20.228714 seconds 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7 63 reset_heartbeat_peers 2019-12-07T19:34:40.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.639+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.cs0 62 -> 63 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.639+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:40.639+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.639+0000 7f8f141a1700 10 osd.7 63 maybe_update_heartbeat_peers updating 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.639+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:40.639+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.639+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c5500d000 2019-12-07T19:34:40.639+0000 7f8f07988700 20 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c57e88000 2019-12-07T19:34:40.639+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: Active advmap 2019-12-07T19:34:40.640+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f0898a700 20 osd.7:0.update_pg_epoch 1.0s3 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 62 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.2s0 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 62 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6400 < min_flush_size 524288 2019-12-07T19:34:40.640+0000 7f8f0898a700 20 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: Active advmap 2019-12-07T19:34:40.640+0000 7f8f0898a700 20 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0898a700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c5736a000 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f0297e700 7 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:40.640+0000 7f8f0898a700 20 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f0898a700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f0297e700 15 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:217 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579de780 had 1 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f141a1700 10 osd.7 63 adding neighbor peer osd.1 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c548ab000 2019-12-07T19:34:40.640+0000 7f8f03980700 7 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:40.640+0000 7f8f141a1700 10 osd.7 63 adding neighbor peer osd.3 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f141a1700 10 osd.7 63 adding neighbor peer osd.5 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f141a1700 10 osd.7 63 adding random peer osd.2 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:40.640+0000 7f8f141a1700 10 osd.7 63 adding random peer osd.4 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f141a1700 10 osd.7 63 maybe_update_heartbeat_peers 5 peers, extras 1,2,3,4,5 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f141a1700 7 osd.7 63 activate_map version 63 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c57383000 2019-12-07T19:34:40.640+0000 7f8f03980700 15 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:180 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579df680 had 1 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:40.640+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6423 < min_flush_size 524288 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 62 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:40.640+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6423 < min_flush_size 524288 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: Active advmap 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x54440b~1917 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x54440b 0x54440b~1917 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x51440b 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x40b 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd22 and padding block with 0x2de 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c57522000 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c5500a000 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.5s4 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0297e700 7 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 62 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f0297e700 15 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:168 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:40.640+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 62 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c54d98000 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:40.640+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.es4 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c55141000 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7:5.update_pg_epoch 1.ds2 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 62 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f0217d700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f0918b700 20 osd.7 63 agent_entry empty queue 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c575ef000 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c54d5c000 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f0217d700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f0217d700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.bs1 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7:6.update_pg_epoch 1.6s2 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 62 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f05984700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f05984700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 lcod 62'77 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f05984700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) queued 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 63 epoch_requested: 63 NullEvt) prio 255 cost 10 e63) pg 0x558c5513e000 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] handle_advance_map: 63 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7:3.update_pg_epoch 1.3s4 62 -> 63 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 62 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:40.640+0000 7f8f07187700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 63 advance_pg new pool opts old pool opts 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] handle_activate_map: 63 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 63 2019-12-07T19:34:40.640+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f07187700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 63 epoch_requested: 63 NullEvt 2019-12-07T19:34:40.640+0000 7f8f07187700 10 log is not dirty 2019-12-07T19:34:40.640+0000 7f8f07187700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x545d22 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.640+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000722174s (0.000000135s flush + 0.000722039s kv commit) 2019-12-07T19:34:40.640+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5876c300> 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c300 kv_submitted 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5876c300 2019-12-07T19:34:40.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5876c300 finishing 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5876c300 onodes 0x558c579de780,0x558c579df680 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5876c300 done 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:40.640+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5876c300 [] 2019-12-07T19:34:40.640+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.640+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.640+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 63/28 ECSubWriteReply(tid=163, last_complete=63'79, committed=1, applied=1)) v2 -- 0x558c58512c00 con 0x558c54c5bc00 2019-12-07T19:34:40.640+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.640+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 321 ==== MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=164, reqid=client.4540.0:467, at_version=63'80, trim_to=0'0, roll_forward_to=62'77)) v2 ==== 2206+0+0 (crc 0 0 0) 0x558c58231800 con 0x558c54c5bc00 2019-12-07T19:34:40.641+0000 7f8f23c56700 15 osd.7 63 enqueue_op 0x558c5511f600 prio 127 cost 0 latency 0.000034 epoch 63 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=164, reqid=client.4540.0:467, at_version=63'80, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:40.641+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=164, reqid=client.4540.0:467, at_version=63'80, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=164, reqid=client.4540.0:467, at_version=63'80, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=164, reqid=client.4540.0:467, at_version=63'80, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) pg 0x558c54d5c000 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 63 dequeue_op 0x558c5511f600 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=164, reqid=client.4540.0:467, at_version=63'80, trim_to=0'0, roll_forward_to=62'77)) v2 pg pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=164, reqid=client.4540.0:467, at_version=63'80, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'79 (0'0,63'79] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'79], crt=62'77) [63'80 (63'79) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:467 2019-12-07T19:34:40.638240+0000 0 ObjectCleanRegions clean_offsets: [1556480~18446744073707995135], clean_omap: 0, new_object: 0] 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'80 (0'0,63'80] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'79 crt=62'77 lcod 63'79 active mbc={} ps=[20~1,23~1]] add_log_entry 63'80 (63'79) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:467 2019-12-07T19:34:40.638240+0000 0 ObjectCleanRegions clean_offsets: [1556480~18446744073707995135], clean_omap: 0, new_object: 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'80 (0'0,63'80] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'79 crt=62'77 lcod 63'79 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 80 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'80 (0'0,63'80] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'79 crt=62'77 lcod 63'79 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.641+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'80, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c550a3800 seq 120 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) get_onode oid 2#1:6c205177:::smithi13140796-8:head#50 key 0x8280000000000000016c2051'w!smithi13140796-8!='0xfffffffffffffffe0000000000000050'o' 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.6s2_head 0x558c54d38b40) r -2 v.len 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:head#50 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# -> 2#1:6c205177:::smithi13140796-8:head#50 = 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# -> 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1294 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.onode(0x558c57c05b80).flush flush done 2019-12-07T19:34:40.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 0x0 2019-12-07T19:34:40.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# -> 2#1:6c205177:::smithi13140796-8:head# 0x0~5f000 -> 0x0~5f000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x0~2000: 0x0~2000 Blob(0x558c57896380 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585e01c0 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c585e01c0) get_ref 0x0~2000 Blob(0x558c585e01c0 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c585e01c0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x0~2000: 0x0~2000 Blob(0x558c585e01c0 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2000~2000: 0x0~2000 Blob(0x558c578962a0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b15e0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b15e0) get_ref 0x0~2000 Blob(0x558c550b15e0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b15e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c550b15e0 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4000~2000: 0x0~2000 Blob(0x558c57f51570 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df9b90 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9b90) get_ref 0x0~2000 Blob(0x558c54df9b90 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9b90) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c54df9b90 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x6000~2000: 0x0~2000 Blob(0x558c55278a10 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c585d5260 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c585d5260) get_ref 0x0~2000 Blob(0x558c585d5260 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c585d5260) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c585d5260 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x8000~2000: 0x0~2000 Blob(0x558c550607e0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5794c850 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5794c850) get_ref 0x0~2000 Blob(0x558c5794c850 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5794c850) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5794c850 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0xa000~2000: 0x0~2000 Blob(0x558c55060770 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5898bb20 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5898bb20) get_ref 0x0~2000 Blob(0x558c5898bb20 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5898bb20) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c5898bb20 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0xc000~2000: 0x0~2000 Blob(0x558c57efd180 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df9d50 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9d50) get_ref 0x0~2000 Blob(0x558c54df9d50 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9d50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c54df9d50 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0xe000~2000: 0x0~2000 Blob(0x558c550b2700 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1570 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1570) get_ref 0x0~2000 Blob(0x558c550b1570 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1570) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c550b1570 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x10000~2000: 0x0~2000 Blob(0x558c579d6e70 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df9a40 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9a40) get_ref 0x0~2000 Blob(0x558c54df9a40 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9a40) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c54df9a40 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x12000~1000: 0x0~1000 Blob(0x558c55060690 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1490 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1490) get_ref 0x0~1000 Blob(0x558c550b1490 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1490) get_ref init 0x1000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x12000~1000: 0x0~1000 Blob(0x558c550b1490 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x13000~2000: 0x0~2000 Blob(0x558c5898b180 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df80e0 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df80e0) get_ref 0x0~2000 Blob(0x558c54df80e0 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df80e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x13000~2000: 0x0~2000 Blob(0x558c54df80e0 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x15000~2000: 0x0~2000 Blob(0x558c57654f50 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1420 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1420) get_ref 0x0~2000 Blob(0x558c550b1420 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1420) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x15000~2000: 0x0~2000 Blob(0x558c550b1420 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x17000~2000: 0x0~2000 Blob(0x558c579d6fc0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5809bc00 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bc00) get_ref 0x0~2000 Blob(0x558c5809bc00 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bc00) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x17000~2000: 0x0~2000 Blob(0x558c5809bc00 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x19000~2000: 0x0~2000 Blob(0x558c575bc620 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b13b0 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b13b0) get_ref 0x0~2000 Blob(0x558c550b13b0 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b13b0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x19000~2000: 0x0~2000 Blob(0x558c550b13b0 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x1b000~2000: 0x0~2000 Blob(0x558c54677880 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5809be30 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809be30) get_ref 0x0~2000 Blob(0x558c5809be30 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809be30) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x1b000~2000: 0x0~2000 Blob(0x558c5809be30 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x1d000~2000: 0x0~2000 Blob(0x558c575bc700 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1340 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1340) get_ref 0x0~2000 Blob(0x558c550b1340 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1340) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x1d000~2000: 0x0~2000 Blob(0x558c550b1340 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x1f000~2000: 0x0~2000 Blob(0x558c57efd340 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5809bce0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bce0) get_ref 0x0~2000 Blob(0x558c5809bce0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bce0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x1f000~2000: 0x0~2000 Blob(0x558c5809bce0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x21000~2000: 0x0~2000 Blob(0x558c57396cb0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b12d0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b12d0) get_ref 0x0~2000 Blob(0x558c550b12d0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b12d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x21000~2000: 0x0~2000 Blob(0x558c550b12d0 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x23000~2000: 0x0~2000 Blob(0x558c5523a1c0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5809bea0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bea0) get_ref 0x0~2000 Blob(0x558c5809bea0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5809bea0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x23000~2000: 0x0~2000 Blob(0x558c5809bea0 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x25000~2000: 0x0~2000 Blob(0x558c5742ae70 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1260 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1260) get_ref 0x0~2000 Blob(0x558c550b1260 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1260) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x25000~2000: 0x0~2000 Blob(0x558c550b1260 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x27000~2000: 0x0~2000 Blob(0x558c581d6c40 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c577f3b20 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3b20) get_ref 0x0~2000 Blob(0x558c577f3b20 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3b20) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x27000~2000: 0x0~2000 Blob(0x558c577f3b20 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x29000~2000: 0x0~2000 Blob(0x558c574d3a40 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b11f0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b11f0) get_ref 0x0~2000 Blob(0x558c550b11f0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b11f0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x29000~2000: 0x0~2000 Blob(0x558c550b11f0 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2b000~2000: 0x0~2000 Blob(0x558c55264070 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df9260 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9260) get_ref 0x0~2000 Blob(0x558c54df9260 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9260) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2b000~2000: 0x0~2000 Blob(0x558c54df9260 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2d000~2000: 0x0~2000 Blob(0x558c5742ad90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1180 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1180) get_ref 0x0~2000 Blob(0x558c550b1180 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1180) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2d000~2000: 0x0~2000 Blob(0x558c550b1180 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x2f000~1000: 0x1000~1000 Blob(0x558c57efd2d0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df9180 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9180) get_ref 0x1000~1000 Blob(0x558c54df9180 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df9180) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x2f000~1000: 0x1000~1000 Blob(0x558c54df9180 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x30000~2000: 0x0~2000 Blob(0x558c575bd6c0 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1110 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1110) get_ref 0x0~2000 Blob(0x558c550b1110 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1110) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c550b1110 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x32000~2000: 0x0~2000 Blob(0x558c579d6e00 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df90a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df90a0) get_ref 0x0~2000 Blob(0x558c54df90a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df90a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x32000~2000: 0x0~2000 Blob(0x558c54df90a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x34000~2000: 0x0~2000 Blob(0x558c55278f50 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b10a0 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b10a0) get_ref 0x0~2000 Blob(0x558c550b10a0 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b10a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x34000~2000: 0x0~2000 Blob(0x558c550b10a0 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x36000~2000: 0x0~2000 Blob(0x558c575f6070 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df8fc0 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8fc0) get_ref 0x0~2000 Blob(0x558c54df8fc0 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x36000~2000: 0x0~2000 Blob(0x558c54df8fc0 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x38000~2000: 0x0~2000 Blob(0x558c574d3030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b1030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1030) get_ref 0x0~2000 Blob(0x558c550b1030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b1030) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x38000~2000: 0x0~2000 Blob(0x558c550b1030 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x3a000~2000: 0x0~2000 Blob(0x558c581d6b60 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df8ee0 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8ee0) get_ref 0x0~2000 Blob(0x558c54df8ee0 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x3a000~2000: 0x0~2000 Blob(0x558c54df8ee0 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x3c000~2000: 0x0~2000 Blob(0x558c54cf82a0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0fc0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0fc0) get_ref 0x0~2000 Blob(0x558c550b0fc0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x3c000~2000: 0x0~2000 Blob(0x558c550b0fc0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x3e000~2000: 0x0~2000 Blob(0x558c575f6000 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df8e00 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8e00) get_ref 0x0~2000 Blob(0x558c54df8e00 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8e00) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x3e000~2000: 0x0~2000 Blob(0x558c54df8e00 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x40000~2000: 0x0~2000 Blob(0x558c574d30a0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0f50 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0f50) get_ref 0x0~2000 Blob(0x558c550b0f50 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0f50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x40000~2000: 0x0~2000 Blob(0x558c550b0f50 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x42000~2000: 0x0~2000 Blob(0x558c577f3ab0 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df8d20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8d20) get_ref 0x0~2000 Blob(0x558c54df8d20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8d20) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x42000~2000: 0x0~2000 Blob(0x558c54df8d20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x44000~2000: 0x0~2000 Blob(0x558c574e0690 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0ee0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0ee0) get_ref 0x0~2000 Blob(0x558c550b0ee0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x44000~2000: 0x0~2000 Blob(0x558c550b0ee0 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x46000~2000: 0x0~2000 Blob(0x558c575f6380 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df8c40 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8c40) get_ref 0x0~2000 Blob(0x558c54df8c40 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8c40) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x46000~2000: 0x0~2000 Blob(0x558c54df8c40 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x48000~2000: 0x0~2000 Blob(0x558c57566e00 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0e70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0e70) get_ref 0x0~2000 Blob(0x558c550b0e70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0e70) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x48000~2000: 0x0~2000 Blob(0x558c550b0e70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4a000~2000: 0x0~2000 Blob(0x558c581d6d20 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df8b60 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8b60) get_ref 0x0~2000 Blob(0x558c54df8b60 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8b60) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4a000~2000: 0x0~2000 Blob(0x558c54df8b60 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4c000~2000: 0x0~2000 Blob(0x558c573144d0 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0e00 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0e00) get_ref 0x0~2000 Blob(0x558c550b0e00 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0e00) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4c000~2000: 0x0~2000 Blob(0x558c550b0e00 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x4e000~2000: 0x0~2000 Blob(0x558c54df8070 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c54df8a80 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8a80) get_ref 0x0~2000 Blob(0x558c54df8a80 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c54df8a80) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x4e000~2000: 0x0~2000 Blob(0x558c54df8a80 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x50000~2000: 0x0~2000 Blob(0x558c574d2770 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0d90 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0d90) get_ref 0x0~2000 Blob(0x558c550b0d90 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0d90) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x50000~2000: 0x0~2000 Blob(0x558c550b0d90 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x52000~2000: 0x0~2000 Blob(0x558c5829cf50 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5792d650 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d650) get_ref 0x0~2000 Blob(0x558c5792d650 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d650) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x52000~2000: 0x0~2000 Blob(0x558c5792d650 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x54000~2000: 0x0~2000 Blob(0x558c574d27e0 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0d20 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0d20) get_ref 0x0~2000 Blob(0x558c550b0d20 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0d20) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x54000~2000: 0x0~2000 Blob(0x558c550b0d20 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x56000~2000: 0x0~2000 Blob(0x558c575f7ce0 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c5792d730 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d730) get_ref 0x0~2000 Blob(0x558c5792d730 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c5792d730) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x56000~2000: 0x0~2000 Blob(0x558c5792d730 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x58000~2000: 0x0~2000 Blob(0x558c574d39d0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0cb0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0cb0) get_ref 0x0~2000 Blob(0x558c550b0cb0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x58000~2000: 0x0~2000 Blob(0x558c550b0cb0 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x5a000~2000: 0x0~2000 Blob(0x558c575f7c00 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c577f3e30 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3e30) get_ref 0x0~2000 Blob(0x558c577f3e30 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c577f3e30) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x5a000~2000: 0x0~2000 Blob(0x558c577f3e30 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x5c000~2000: 0x0~2000 Blob(0x558c57314310 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c550b0c40 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0c40) get_ref 0x0~2000 Blob(0x558c550b0c40 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0c40) get_ref init 0x2000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x5c000~2000: 0x0~2000 Blob(0x558c550b0c40 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup src 0x5e000~1000: 0x0~1000 Blob(0x558c54df8230 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup new Blob(0x558c57efd3b0 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd3b0) get_ref 0x0~1000 Blob(0x558c57efd3b0 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd3b0) get_ref init 0x1000, 1000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57c05c88) dup dst 0x5e000~1000: 0x0~1000 Blob(0x558c57efd3b0 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=4)))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) dirty_range mark inline shard dirty 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.6s2_head 2#1:6c205177:::smithi13140796-8:26# -> 2#1:6c205177:::smithi13140796-8:head# = 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000063.00000000000000000080' <- 0000000063.00000000000000000080 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a3800 cost 5875 (1 ios * 4000 + 1875 bytes) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a3800 onodes 0x558c579df680,0x558c57f8ef00 shared_blobs 0x558c54df8000,0x558c54df8af0,0x558c54df8bd0,0x558c54df8cb0,0x558c54df8d90,0x558c54df8e70,0x558c54df8f50,0x558c54df9030,0x558c54df9110,0x558c54df91f0,0x558c54df9ab0,0x558c54df9c00,0x558c54df9ce0,0x558c54df9dc0,0x558c54e43ce0,0x558c54e56690,0x558c54e56bd0,0x558c54e56cb0,0x558c54e56d90,0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c573970a0,0x558c575bd3b0,0x558c577f3a40,0x558c577f3c70,0x558c57e63490,0x558c57efd5e0,0x558c57efd6c0,0x558c57efd7a0,0x558c57efd880,0x558c57efd960,0x558c57efda40,0x558c57efdb20,0x558c57efdc00,0x558c5809b490,0x558c5809b500,0x558c5809b570,0x558c5809bc70,0x558c5809bd50,0x558c5809bdc0,0x558c5809bf80,0x558c580a4a10 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update 2#1:6c205177:::smithi13140796-8:head#50 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update shard 0x0 is 487 bytes (was 487) from 18 extents 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update shard 0x23000 is 505 bytes (was 505) from 18 extents 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) update shard 0x46000 is 353 bytes (was 353) from 13 extents 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head#50 is 666 (664 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) update inline shard 1335 bytes from 49 extents 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore.extentmap(0x558c57f8f008) reshard 0x[0,ffffffff) of 0 shards on 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard extent_avg 27, target 500, slop 100 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard new shard 0x0 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard new shard 0x23000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard new shard 0x46000 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard new [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard old [] 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard fin [0x0(0x0 bytes),0x23000(0x0 bytes),0x46000(0x0 bytes)] 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) reshard checking spanning blobs 0x[0,ffffffff) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) update 2#1:6c205177:::smithi13140796-8:head# force 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) update shard 0x0 is 487 bytes (was 0) from 18 extents 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) update shard 0x23000 is 505 bytes (was 0) from 18 extents 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) update shard 0x46000 is 353 bytes (was 0) from 13 extents 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 666 (664 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aba is 11 SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abd is 11 SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abe is 11 SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abf is 11 SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac0 is 11 SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac1 is 11 SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac2 is 11 SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac3 is 11 SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac4 is 11 SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac5 is 11 SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acd is 11 SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ace is 11 SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acf is 11 SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad0 is 11 SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab8 is 11 SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab4 is 11 SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab7 is 11 SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab6 is 11 SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab5 is 11 SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac7 is 11 SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac6 is 11 SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab9 is 11 SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab2 is 11 SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaa is 11 SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aab is 11 SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aac is 11 SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aad is 11 SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aae is 11 SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaf is 11 SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab0 is 11 SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab1 is 11 SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abc is 11 SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abb is 11 SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acc is 11 SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aca is 11 SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac8 is 11 SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac9 is 11 SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acb is 11 SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab3 is 11 SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=4))) 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a3800 allocated 0x[] released 0x[] 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3800 prepare 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a3800 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3800 io_done 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'80 (0'0,63'80] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'79 crt=62'77 lcod 63'79 active mbc={} ps=[20~1,23~1]] op_applied version 63'80 2019-12-07T19:34:40.641+0000 7f8f0197c700 10 osd.7 63 dequeue_op 0x558c5511f600 finish 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6553 < min_flush_size 524288 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579df680 had 1 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8ef00 had 1 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6576 < min_flush_size 524288 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 6576 < min_flush_size 524288 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x545d22~19b0 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x545d22 0x545d22~19b0 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x515d22 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd22 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x6d2 and padding block with 0x92e 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5476d2 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.641+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000161958s (0.000000130s flush + 0.000161828s kv commit) 2019-12-07T19:34:40.641+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c550a3800> 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3800 kv_submitted 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a3800 2019-12-07T19:34:40.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3800 finishing 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a3800 onodes 0x558c579df680,0x558c57f8ef00 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a3800 done 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:40.641+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a3800 [] 2019-12-07T19:34:40.641+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.641+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.641+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.641+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 63/28 ECSubWriteReply(tid=164, last_complete=63'80, committed=1, applied=1)) v2 -- 0x558c57950e00 con 0x558c54c5bc00 2019-12-07T19:34:40.641+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.642+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 322 ==== MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=165, reqid=client.4540.0:468, at_version=63'81, trim_to=0'0, roll_forward_to=62'77)) v2 ==== 2496+0+0 (crc 0 0 0) 0x558c57a43000 con 0x558c54c5bc00 2019-12-07T19:34:40.642+0000 7f8f23c56700 15 osd.7 63 enqueue_op 0x558c57efedc0 prio 127 cost 0 latency 0.000021 epoch 63 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=165, reqid=client.4540.0:468, at_version=63'81, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:40.642+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=165, reqid=client.4540.0:468, at_version=63'81, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=165, reqid=client.4540.0:468, at_version=63'81, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=165, reqid=client.4540.0:468, at_version=63'81, trim_to=0'0, roll_forward_to=62'77)) v2) prio 127 cost 0 e63) pg 0x558c54d5c000 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 63 dequeue_op 0x558c57efedc0 prio 127 cost 0 latency 0.000044 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=165, reqid=client.4540.0:468, at_version=63'81, trim_to=0'0, roll_forward_to=62'77)) v2 pg pg[1.6s2( v 63'80 (0'0,63'80] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'80 (0'0,63'80] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=165, reqid=client.4540.0:468, at_version=63'81, trim_to=0'0, roll_forward_to=62'77)) v2 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'80 (0'0,63'80] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'80], crt=62'77) [63'81 (63'80) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:468 2019-12-07T19:34:40.638244+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0] 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'80 crt=62'77 lcod 63'80 active mbc={} ps=[20~1,23~1]] add_log_entry 63'81 (63'80) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:468 2019-12-07T19:34:40.638244+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'80 crt=62'77 lcod 63'80 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 81 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'80 crt=62'77 lcod 63'80 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.642+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'81, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c550a3b00 seq 121 2019-12-07T19:34:40.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:34:40.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.6s2_head 2#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.642+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.0000000063.00000000000000000081' <- 0000000063.00000000000000000081 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c550a3b00 cost 6147 (1 ios * 4000 + 2147 bytes) 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c550a3b00 onodes 0x558c57f8ef00 shared_blobs 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore.extentmap(0x558c57f8f008) update 2#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 2#1:6c205177:::smithi13140796-8:head# is 666 (664 bytes onode + 2 bytes spanning blobs + 0 bytes inline extents) 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c550a3b00 allocated 0x[] released 0x[] 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3b00 prepare 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c550a3b00 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3b00 io_done 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 lua=63'80 crt=62'77 lcod 63'80 active mbc={} ps=[20~1,23~1]] op_applied version 63'81 2019-12-07T19:34:40.642+0000 7f8f0197c700 10 osd.7 63 dequeue_op 0x558c57efedc0 finish 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1874 < min_flush_size 524288 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57f8ef00 had 1 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1897 < min_flush_size 524288 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1897 < min_flush_size 524288 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5476d2~769 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5476d2 0x5476d2~769 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x5176d2 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x6d2 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xe3b and padding block with 0x1c5 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x547e3b 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.642+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000108726s (0.000000072s flush + 0.000108654s kv commit) 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.642+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c550a3b00> 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3b00 kv_submitted 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c550a3b00 2019-12-07T19:34:40.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c550a3b00 finishing 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c550a3b00 onodes 0x558c57f8ef00 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c550a3b00 done 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:40.642+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c550a3b00 [] 2019-12-07T19:34:40.642+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.642+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.642+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 63/28 ECSubWriteReply(tid=165, last_complete=63'81, committed=1, applied=1)) v2 -- 0x558c57950c00 con 0x558c54c5bc00 2019-12-07T19:34:40.642+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.642+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.645+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 323 ==== MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=166, reqid=client.4540.0:468, at_version=0'0, trim_to=0'0, roll_forward_to=63'81)) v2 ==== 909+0+0 (crc 0 0 0) 0x558c57a44000 con 0x558c54c5bc00 2019-12-07T19:34:40.645+0000 7f8f23c56700 15 osd.7 63 enqueue_op 0x558c5740cf20 prio 127 cost 0 latency 0.000019 epoch 63 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=166, reqid=client.4540.0:468, at_version=0'0, trim_to=0'0, roll_forward_to=63'81)) v2 2019-12-07T19:34:40.645+0000 7f8f23c56700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=166, reqid=client.4540.0:468, at_version=0'0, trim_to=0'0, roll_forward_to=63'81)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=166, reqid=client.4540.0:468, at_version=0'0, trim_to=0'0, roll_forward_to=63'81)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGOpItem(op=MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=166, reqid=client.4540.0:468, at_version=0'0, trim_to=0'0, roll_forward_to=63'81)) v2) prio 127 cost 0 e63) pg 0x558c54d5c000 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 osd.7 63 dequeue_op 0x558c5740cf20 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=166, reqid=client.4540.0:468, at_version=0'0, trim_to=0'0, roll_forward_to=63'81)) v2 pg pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.6s2 63/28 ECSubWrite(tid=166, reqid=client.4540.0:468, at_version=0'0, trim_to=0'0, roll_forward_to=63'81)) v2 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=62'77 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'81], crt=62'77) [] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'78 (62'77) clone 1:6c205177:::smithi13140796-8:28 by unknown.0.0:0 2019-12-07T19:34:38.611669+0000 0 snaps [28] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'79 (62'77) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:466 2019-12-07T19:34:40.638217+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'80 (63'79) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:467 2019-12-07T19:34:40.638240+0000 0 ObjectCleanRegions clean_offsets: [1556480~18446744073707995135], clean_omap: 0, new_object: 0 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'81 (63'80) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:468 2019-12-07T19:34:40.638244+0000 0 ObjectCleanRegions clean_offsets: [0~1556480,1556480~18446744073707995135], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 81 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.645+0000 7f8f0197c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d38b40 1.6s2_head 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf77c0 = 0x558c584ca000 seq 122 2019-12-07T19:34:40.645+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#50 onode 0x558c579df680 txc 0x558c584ca000 2019-12-07T19:34:40.645+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#50 0x0 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c54e56f50) put_ref 0x0~2000 Blob(0x558c54e56f50 blob([0x26f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3c310) put_ref 0x0~2000 Blob(0x558c54d3c310 blob([0x271000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c5792dce0) put_ref 0x0~2000 Blob(0x558c5792dce0 blob([0x273000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c550b3730) put_ref 0x0~2000 Blob(0x558c550b3730 blob([0x275000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c54d750a0) put_ref 0x0~2000 Blob(0x558c54d750a0 blob([0x277000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c580d7e30) put_ref 0x0~2000 Blob(0x558c580d7e30 blob([0x279000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c574d28c0) put_ref 0x0~2000 Blob(0x558c574d28c0 blob([0x27b000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c575f7c70) put_ref 0x0~2000 Blob(0x558c575f7c70 blob([0x27d000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c5829c000) put_ref 0x0~2000 Blob(0x558c5829c000 blob([0x27f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c547402a0) put_ref 0x0~1000 Blob(0x558c547402a0 blob([0x281000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd420) put_ref 0x0~2000 Blob(0x558c57efd420 blob([0x5a6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57c42930) put_ref 0x0~2000 Blob(0x558c57c42930 blob([0x5a8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6bd0) put_ref 0x0~2000 Blob(0x558c581d6bd0 blob([0x5aa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57e63e30) put_ref 0x0~2000 Blob(0x558c57e63e30 blob([0x5ac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd490) put_ref 0x0~2000 Blob(0x558c57efd490 blob([0x5ae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57a07dc0) put_ref 0x0~2000 Blob(0x558c57a07dc0 blob([0x5b0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6cb0) put_ref 0x0~2000 Blob(0x558c581d6cb0 blob([0x5b2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c54d3df10) put_ref 0x0~2000 Blob(0x558c54d3df10 blob([0x5b4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd500) put_ref 0x0~2000 Blob(0x558c57efd500 blob([0x5b6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd1f0) put_ref 0x0~2000 Blob(0x558c57efd1f0 blob([0x5b8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c580d7f80) put_ref 0x0~2000 Blob(0x558c580d7f80 blob([0x5ba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c57efd260) put_ref 0x0~2000 Blob(0x558c57efd260 blob([0x5bc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c5829c7e0) put_ref 0x0~2000 Blob(0x558c5829c7e0 blob([0x5be000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c581d6d90) put_ref 0x0~2000 Blob(0x558c581d6d90 blob([0x5c0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e16c0) put_ref 0x1000~1000 Blob(0x558c585e16c0 blob([!~1000,0x5c2000~1000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1730) put_ref 0x0~2000 Blob(0x558c585e1730 blob([0x5c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e17a0) put_ref 0x0~2000 Blob(0x558c585e17a0 blob([0x5c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1810) put_ref 0x0~2000 Blob(0x558c585e1810 blob([0x5c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1880) put_ref 0x0~2000 Blob(0x558c585e1880 blob([0x5c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e18f0) put_ref 0x0~2000 Blob(0x558c585e18f0 blob([0x5cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1960) put_ref 0x0~2000 Blob(0x558c585e1960 blob([0x5cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e19d0) put_ref 0x0~2000 Blob(0x558c585e19d0 blob([0x5cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1a40) put_ref 0x0~2000 Blob(0x558c585e1a40 blob([0x5d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ab0) put_ref 0x0~2000 Blob(0x558c585e1ab0 blob([0x5d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1b20) put_ref 0x0~2000 Blob(0x558c585e1b20 blob([0x5d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1b90) put_ref 0x0~2000 Blob(0x558c585e1b90 blob([0x5d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1c00) put_ref 0x0~2000 Blob(0x558c585e1c00 blob([0x5d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1c70) put_ref 0x0~2000 Blob(0x558c585e1c70 blob([0x5db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ce0) put_ref 0x0~2000 Blob(0x558c585e1ce0 blob([0x5dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1d50) put_ref 0x0~2000 Blob(0x558c585e1d50 blob([0x5df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1dc0) put_ref 0x0~2000 Blob(0x558c585e1dc0 blob([0x5e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1e30) put_ref 0x0~2000 Blob(0x558c585e1e30 blob([0x5e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1ea0) put_ref 0x0~2000 Blob(0x558c585e1ea0 blob([0x5e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1f10) put_ref 0x0~2000 Blob(0x558c585e1f10 blob([0x5e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c585e1f80) put_ref 0x0~2000 Blob(0x558c585e1f80 blob([0x5e9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0000) put_ref 0x0~2000 Blob(0x558c550b0000 blob([0x5eb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0070) put_ref 0x0~2000 Blob(0x558c550b0070 blob([0x5ed000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c550b00e0) put_ref 0x0~2000 Blob(0x558c550b00e0 blob([0x5ef000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.blob(0x558c550b0150) put_ref 0x0~1000 Blob(0x558c550b0150 blob([0x5f1000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) dirty_range mark shard 0x0 dirty 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) dirty_range mark shard 0x23000 dirty 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore.extentmap(0x558c579df788) dirty_range mark shard 0x46000 dirty 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c54e56f50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x26f000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c54d3c310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x271000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c5792dce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x273000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c550b3730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x275000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c54d750a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x277000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c580d7e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x279000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c574d28c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27b000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c575f7c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27d000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c5829c000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x27f000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~1000: 0x0~1000 Blob(0x558c547402a0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x281000~1000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x13000~2000: 0x0~2000 Blob(0x558c57efd420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5a6000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x15000~2000: 0x0~2000 Blob(0x558c57c42930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5a8000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x17000~2000: 0x0~2000 Blob(0x558c581d6bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5aa000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x19000~2000: 0x0~2000 Blob(0x558c57e63e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ac000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1b000~2000: 0x0~2000 Blob(0x558c57efd490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ae000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1d000~2000: 0x0~2000 Blob(0x558c57a07dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b0000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1f000~2000: 0x0~2000 Blob(0x558c581d6cb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b2000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x21000~2000: 0x0~2000 Blob(0x558c54d3df10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b4000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x23000~2000: 0x0~2000 Blob(0x558c57efd500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b6000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x25000~2000: 0x0~2000 Blob(0x558c57efd1f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5b8000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x27000~2000: 0x0~2000 Blob(0x558c580d7f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ba000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x29000~2000: 0x0~2000 Blob(0x558c57efd260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5bc000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2b000~2000: 0x0~2000 Blob(0x558c5829c7e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5be000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2d000~2000: 0x0~2000 Blob(0x558c581d6d90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c0000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2f000~1000: 0x1000~1000 Blob(0x558c585e16c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c2000~1000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c585e1730 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c3000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~2000: 0x0~2000 Blob(0x558c585e17a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c5000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x34000~2000: 0x0~2000 Blob(0x558c585e1810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c7000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x36000~2000: 0x0~2000 Blob(0x558c585e1880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5c9000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x38000~2000: 0x0~2000 Blob(0x558c585e18f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5cb000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3a000~2000: 0x0~2000 Blob(0x558c585e1960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5cd000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3c000~2000: 0x0~2000 Blob(0x558c585e19d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5cf000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x3e000~2000: 0x0~2000 Blob(0x558c585e1a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d1000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x40000~2000: 0x0~2000 Blob(0x558c585e1ab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d3000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x42000~2000: 0x0~2000 Blob(0x558c585e1b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d5000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x44000~2000: 0x0~2000 Blob(0x558c585e1b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d7000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x46000~2000: 0x0~2000 Blob(0x558c585e1c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5d9000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x48000~2000: 0x0~2000 Blob(0x558c585e1c70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5db000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4a000~2000: 0x0~2000 Blob(0x558c585e1ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5dd000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4c000~2000: 0x0~2000 Blob(0x558c585e1d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5df000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4e000~2000: 0x0~2000 Blob(0x558c585e1dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e1000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x50000~2000: 0x0~2000 Blob(0x558c585e1e30 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e3000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x52000~2000: 0x0~2000 Blob(0x558c585e1ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e5000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x54000~2000: 0x0~2000 Blob(0x558c585e1f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e7000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x56000~2000: 0x0~2000 Blob(0x558c585e1f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5e9000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x58000~2000: 0x0~2000 Blob(0x558c550b0000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5eb000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5a000~2000: 0x0~2000 Blob(0x558c550b0070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ed000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5c000~2000: 0x0~2000 Blob(0x558c550b00e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5ef000~2000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x5e000~1000: 0x0~1000 Blob(0x558c550b0150 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=4)))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0x5f1000~1000] 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate request reshard past EOF 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x0 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x23000 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove removing shard 0x46000 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.6s2_head 2#1:6c205177:::smithi13140796-8:head#50 = 0 2019-12-07T19:34:40.645+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.6s2_head 2#1:60000000::::head# = 0 2019-12-07T19:34:40.645+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584ca000 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584ca000 onodes shared_blobs 0x558c54df8000,0x558c54df8af0,0x558c54df8bd0,0x558c54df8cb0,0x558c54df8d90,0x558c54df8e70,0x558c54df8f50,0x558c54df9030,0x558c54df9110,0x558c54df91f0,0x558c54df9ab0,0x558c54df9c00,0x558c54df9ce0,0x558c54df9dc0,0x558c54e43ce0,0x558c54e56690,0x558c54e56bd0,0x558c54e56cb0,0x558c54e56d90,0x558c573140e0,0x558c573141c0,0x558c573142a0,0x558c57314540,0x558c57314690,0x558c57314770,0x558c57314850,0x558c57314930,0x558c57314a10,0x558c573970a0,0x558c575bd3b0,0x558c577f3a40,0x558c577f3c70,0x558c57e63490,0x558c57efd5e0,0x558c57efd6c0,0x558c57efd7a0,0x558c57efd880,0x558c57efd960,0x558c57efda40,0x558c57efdb20,0x558c57efdc00,0x558c5809b490,0x558c5809b500,0x558c5809b570,0x558c5809bc70,0x558c5809bd50,0x558c5809bdc0,0x558c5809bf80,0x558c580a4a10 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aba is 11 SharedBlob(0x558c54df8000 loaded (sbid 0x2aba ref_map(0x5c5000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abd is 11 SharedBlob(0x558c54df8af0 loaded (sbid 0x2abd ref_map(0x5cb000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abe is 11 SharedBlob(0x558c54df8bd0 loaded (sbid 0x2abe ref_map(0x5cd000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abf is 11 SharedBlob(0x558c54df8cb0 loaded (sbid 0x2abf ref_map(0x5cf000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac0 is 11 SharedBlob(0x558c54df8d90 loaded (sbid 0x2ac0 ref_map(0x5d1000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac1 is 11 SharedBlob(0x558c54df8e70 loaded (sbid 0x2ac1 ref_map(0x5d3000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac2 is 11 SharedBlob(0x558c54df8f50 loaded (sbid 0x2ac2 ref_map(0x5d5000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac3 is 11 SharedBlob(0x558c54df9030 loaded (sbid 0x2ac3 ref_map(0x5d7000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac4 is 11 SharedBlob(0x558c54df9110 loaded (sbid 0x2ac4 ref_map(0x5d9000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac5 is 11 SharedBlob(0x558c54df91f0 loaded (sbid 0x2ac5 ref_map(0x5db000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acd is 11 SharedBlob(0x558c54df9ab0 loaded (sbid 0x2acd ref_map(0x5eb000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ace is 11 SharedBlob(0x558c54df9c00 loaded (sbid 0x2ace ref_map(0x5ed000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acf is 11 SharedBlob(0x558c54df9ce0 loaded (sbid 0x2acf ref_map(0x5ef000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ad0 is 11 SharedBlob(0x558c54df9dc0 loaded (sbid 0x2ad0 ref_map(0x5f1000~1000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab8 is 11 SharedBlob(0x558c54e43ce0 loaded (sbid 0x2ab8 ref_map(0x5c2000~1000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab4 is 11 SharedBlob(0x558c54e56690 loaded (sbid 0x2ab4 ref_map(0x5ba000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab7 is 11 SharedBlob(0x558c54e56bd0 loaded (sbid 0x2ab7 ref_map(0x5c0000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab6 is 11 SharedBlob(0x558c54e56cb0 loaded (sbid 0x2ab6 ref_map(0x5be000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab5 is 11 SharedBlob(0x558c54e56d90 loaded (sbid 0x2ab5 ref_map(0x5bc000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2899 is 11 SharedBlob(0x558c573140e0 loaded (sbid 0x2899 ref_map(0x275000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2898 is 11 SharedBlob(0x558c573141c0 loaded (sbid 0x2898 ref_map(0x273000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2897 is 11 SharedBlob(0x558c573142a0 loaded (sbid 0x2897 ref_map(0x271000~2000=3))) 2019-12-07T19:34:40.645+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289a is 11 SharedBlob(0x558c57314540 loaded (sbid 0x289a ref_map(0x277000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289b is 11 SharedBlob(0x558c57314690 loaded (sbid 0x289b ref_map(0x279000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289c is 11 SharedBlob(0x558c57314770 loaded (sbid 0x289c ref_map(0x27b000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289d is 11 SharedBlob(0x558c57314850 loaded (sbid 0x289d ref_map(0x27d000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289e is 11 SharedBlob(0x558c57314930 loaded (sbid 0x289e ref_map(0x27f000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x289f is 11 SharedBlob(0x558c57314a10 loaded (sbid 0x289f ref_map(0x281000~1000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac7 is 11 SharedBlob(0x558c573970a0 loaded (sbid 0x2ac7 ref_map(0x5df000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2896 is 11 SharedBlob(0x558c575bd3b0 loaded (sbid 0x2896 ref_map(0x26f000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac6 is 11 SharedBlob(0x558c577f3a40 loaded (sbid 0x2ac6 ref_map(0x5dd000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab9 is 11 SharedBlob(0x558c577f3c70 loaded (sbid 0x2ab9 ref_map(0x5c3000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab2 is 11 SharedBlob(0x558c57e63490 loaded (sbid 0x2ab2 ref_map(0x5b6000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaa is 11 SharedBlob(0x558c57efd5e0 loaded (sbid 0x2aaa ref_map(0x5a6000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aab is 11 SharedBlob(0x558c57efd6c0 loaded (sbid 0x2aab ref_map(0x5a8000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aac is 11 SharedBlob(0x558c57efd7a0 loaded (sbid 0x2aac ref_map(0x5aa000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aad is 11 SharedBlob(0x558c57efd880 loaded (sbid 0x2aad ref_map(0x5ac000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aae is 11 SharedBlob(0x558c57efd960 loaded (sbid 0x2aae ref_map(0x5ae000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aaf is 11 SharedBlob(0x558c57efda40 loaded (sbid 0x2aaf ref_map(0x5b0000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab0 is 11 SharedBlob(0x558c57efdb20 loaded (sbid 0x2ab0 ref_map(0x5b2000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab1 is 11 SharedBlob(0x558c57efdc00 loaded (sbid 0x2ab1 ref_map(0x5b4000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abc is 11 SharedBlob(0x558c5809b490 loaded (sbid 0x2abc ref_map(0x5c9000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2abb is 11 SharedBlob(0x558c5809b500 loaded (sbid 0x2abb ref_map(0x5c7000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acc is 11 SharedBlob(0x558c5809b570 loaded (sbid 0x2acc ref_map(0x5e9000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aca is 11 SharedBlob(0x558c5809bc70 loaded (sbid 0x2aca ref_map(0x5e5000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac8 is 11 SharedBlob(0x558c5809bd50 loaded (sbid 0x2ac8 ref_map(0x5e1000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ac9 is 11 SharedBlob(0x558c5809bdc0 loaded (sbid 0x2ac9 ref_map(0x5e3000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2acb is 11 SharedBlob(0x558c5809bf80 loaded (sbid 0x2acb ref_map(0x5e7000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2ab3 is 11 SharedBlob(0x558c580a4a10 loaded (sbid 0x2ab3 ref_map(0x5b8000~2000=3))) 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584ca000 allocated 0x[] released 0x[] 2019-12-07T19:34:40.646+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584ca000 prepare 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584ca000 2019-12-07T19:34:40.646+0000 7f8f0197c700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584ca000 io_done 2019-12-07T19:34:40.646+0000 7f8f0197c700 10 osd.7 63 dequeue_op 0x558c5740cf20 finish 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 453 < min_flush_size 524288 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2581 < min_flush_size 524288 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148a00 had 1 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c579df680 had 1 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2604 < min_flush_size 524288 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2604 < min_flush_size 524288 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x547e3b~a2c to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x547e3b 0x547e3b~a2c to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x517e3b 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xe3b 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x867 and padding block with 0x799 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x548867 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.646+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000166722s (0.000000097s flush + 0.000166625s kv commit) 2019-12-07T19:34:40.646+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584ca000> 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584ca000 kv_submitted 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584ca000 2019-12-07T19:34:40.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584ca000 finishing 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584ca000 onodes 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584ca000 done 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf77c0 q now empty 2019-12-07T19:34:40.646+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584ca000 [] 2019-12-07T19:34:40.646+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.646+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.646+0000 7f8f05984700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 63/28 ECSubWriteReply(tid=166, last_complete=63'81, committed=1, applied=1)) v2 -- 0x558c57950a00 con 0x558c54c5bc00 2019-12-07T19:34:40.646+0000 7f8f05984700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:40.648+0000 7f8f1199c700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 543 ==== osd_map(63..63 src has 1..63) v4 ==== 633+0+0 (crc 0 0 0) 0x558c58234480 con 0x558c54c64000 2019-12-07T19:34:40.648+0000 7f8f1199c700 20 osd.7 63 OSD::ms_dispatch: osd_map(63..63 src has 1..63) v4 2019-12-07T19:34:40.648+0000 7f8f1199c700 10 osd.7 63 do_waiters -- start 2019-12-07T19:34:40.648+0000 7f8f1199c700 10 osd.7 63 do_waiters -- finish 2019-12-07T19:34:40.648+0000 7f8f1199c700 20 osd.7 63 _dispatch 0x558c58234480 osd_map(63..63 src has 1..63) v4 2019-12-07T19:34:40.648+0000 7f8f1199c700 3 osd.7 63 handle_osd_map epochs [63,63], i have 63, src has [1,63] 2019-12-07T19:34:40.648+0000 7f8f1199c700 10 osd.7 63 no new maps here, dropping 2019-12-07T19:34:40.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 544 ==== MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=309, reqid=client.4540.0:469, at_version=63'44, trim_to=0'0, roll_forward_to=62'43)) v2 ==== 186744+0+0 (crc 0 0 0) 0x558c57a44800 con 0x558c54c64000 2019-12-07T19:34:40.648+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c58755ce0 prio 127 cost 0 latency 0.000031 epoch 63 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=309, reqid=client.4540.0:469, at_version=63'44, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.648+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=309, reqid=client.4540.0:469, at_version=63'44, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.648+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.648+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=309, reqid=client.4540.0:469, at_version=63'44, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.648+0000 7f8f05183700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.648+0000 7f8f05183700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=309, reqid=client.4540.0:469, at_version=63'44, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.648+0000 7f8f05183700 10 osd.7 63 dequeue_op 0x558c58755ce0 prio 127 cost 0 latency 0.000068 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=309, reqid=client.4540.0:469, at_version=63'44, trim_to=0'0, roll_forward_to=62'43)) v2 pg pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.648+0000 7f8f05183700 10 osd.7 63 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 62 -> 63 (as per caller) 2019-12-07T19:34:40.648+0000 7f8f05183700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=309, reqid=client.4540.0:469, at_version=63'44, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.649+0000 7f8f05183700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 62'43 (0'0,62'43] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] append_log log((0'0,62'43], crt=62'43) [63'44 (62'43) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:469 2019-12-07T19:34:40.642449+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 1, new_object: 0] 2019-12-07T19:34:40.649+0000 7f8f05183700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'44 (0'0,63'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=62'43 crt=62'43 lcod 62'43 active mbc={} ps=[20~1,23~1]] add_log_entry 63'44 (62'43) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:469 2019-12-07T19:34:40.642449+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.649+0000 7f8f05183700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'44 (0'0,63'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=62'43 crt=62'43 lcod 62'43 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 44 2019-12-07T19:34:40.649+0000 7f8f05183700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'44 (0'0,63'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=62'43 crt=62'43 lcod 62'43 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.649+0000 7f8f05183700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.649+0000 7f8f05183700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'44, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c583b7500 seq 94 2019-12-07T19:34:40.649+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:40.649+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:40.649+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~2d000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ec1285ab:::smithi13140796-19:head# 0x0~2d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c583b7500 23 blobs 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.649+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2d000/1000,2d000,0 2019-12-07T19:34:40.649+0000 7f8f05183700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc2f000~2d000/1000,2d000,0 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc2f000~2d000] 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57881dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f90ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57881dc0 blob([0xc2f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90ee0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c57881dc0) get_ref 0x0~2000 Blob(0x558c57881dc0 blob([0xc2f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f90ee0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c57881dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c57881dc0 blob([0xc2f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f90ee0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57f90ef8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580f7d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5809a8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580f7d50 blob([0xc31000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a8c0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c580f7d50) get_ref 0x0~2000 Blob(0x558c580f7d50 blob([0xc31000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5809a8c0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c580f7d50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c580f7d50 blob([0xc31000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5809a8c0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5809a8d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579ddea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580f7dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579ddea0 blob([0xc33000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7dc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c579ddea0) get_ref 0x0~2000 Blob(0x558c579ddea0 blob([0xc33000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580f7dc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c579ddea0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c579ddea0 blob([0xc33000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580f7dc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580f7dd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54ea3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574d3dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54ea3490 blob([0xc35000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3dc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c54ea3490) get_ref 0x0~2000 Blob(0x558c54ea3490 blob([0xc35000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574d3dc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c54ea3490) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c54ea3490 blob([0xc35000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574d3dc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c574d3dd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794fd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580bbb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794fd50 blob([0xc37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbb90 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5794fd50) get_ref 0x0~2000 Blob(0x558c5794fd50 blob([0xc37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bbb90 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5794fd50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5794fd50 blob([0xc37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bbb90 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c580bbba8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5750d340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5750d340 blob([0xc39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d3b0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5750d340) get_ref 0x0~2000 Blob(0x558c5750d340 blob([0xc39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d3b0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5750d340) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5750d340 blob([0xc39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d3b0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5800d3c8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580bbb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f91340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580bbb20 blob([0xc3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91340 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c580bbb20) get_ref 0x0~2000 Blob(0x558c580bbb20 blob([0xc3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f91340 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c580bbb20) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c580bbb20 blob([0xc3b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f91340 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c57f91358 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57583dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794fdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57583dc0 blob([0xc3d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794fdc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c57583dc0) get_ref 0x0~2000 Blob(0x558c57583dc0 blob([0xc3d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794fdc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c57583dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c57583dc0 blob([0xc3d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794fdc0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5794fdd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794fe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794fea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794fe30 blob([0xc3f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794fea0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5794fe30) get_ref 0x0~2000 Blob(0x558c5794fe30 blob([0xc3f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794fea0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5794fe30) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c5794fe30 blob([0xc3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794fea0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5794feb8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5794ff10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794ff80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5794ff10 blob([0xc41000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ff80 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5794ff10) get_ref 0x0~2000 Blob(0x558c5794ff10 blob([0xc41000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794ff80 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5794ff10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c5794ff10 blob([0xc41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794ff80 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5794ff98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58bc5c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58bc5c00 blob([0xc43000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5c70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5c00) get_ref 0x0~2000 Blob(0x558c58bc5c00 blob([0xc43000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5c70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5c00) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c58bc5c00 blob([0xc43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58bc5c70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58bc5c88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58bc5ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58bc5ce0 blob([0xc45000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5d50 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5ce0) get_ref 0x0~2000 Blob(0x558c58bc5ce0 blob([0xc45000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5d50 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c58bc5ce0 blob([0xc45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58bc5d50 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58bc5d68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58bc5dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58bc5dc0 blob([0xc47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5e30 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5dc0) get_ref 0x0~2000 Blob(0x558c58bc5dc0 blob([0xc47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5e30 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c58bc5dc0 blob([0xc47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58bc5e30 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58bc5e48 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58bc5ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58bc5ea0 blob([0xc49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5f10 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5ea0) get_ref 0x0~2000 Blob(0x558c58bc5ea0 blob([0xc49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58bc5f10 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c58bc5ea0 blob([0xc49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58bc5f10 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c58bc5f28 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58bc5f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585e0e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58bc5f80 blob([0xc4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585e0e70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5f80) get_ref 0x0~2000 Blob(0x558c58bc5f80 blob([0xc4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585e0e70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c58bc5f80) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c58bc5f80 blob([0xc4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585e0e70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c585e0e88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792ccb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792dc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792ccb0 blob([0xc4d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792dc70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5792ccb0) get_ref 0x0~2000 Blob(0x558c5792ccb0 blob([0xc4d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792dc70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c5792ccb0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c5792ccb0 blob([0xc4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792dc70 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c5792dc88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c573960e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad490 blob([0xc4f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573960e0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c576ad490) get_ref 0x0~2000 Blob(0x558c576ad490 blob([0xc4f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c573960e0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c576ad490) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c576ad490 blob([0xc4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c573960e0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c573960f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c584cc000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d5e0 blob([0xc51000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc000 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d5e0) get_ref 0x0~2000 Blob(0x558c54d3d5e0 blob([0xc51000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc000 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c54d3d5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c54d3d5e0 blob([0xc51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c584cc000 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c584cc018 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c584cc070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c584cc0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c584cc070 blob([0xc53000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc0e0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc070) get_ref 0x0~2000 Blob(0x558c584cc070 blob([0xc53000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc0e0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc070) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c584cc070 blob([0xc53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c584cc0e0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c584cc0f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c584cc150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c584cc1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c584cc150 blob([0xc55000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc1c0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc150) get_ref 0x0~2000 Blob(0x558c584cc150 blob([0xc55000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc1c0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc150) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c584cc150 blob([0xc55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c584cc1c0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c584cc1d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c584cc230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c584cc2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c584cc230 blob([0xc57000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc2a0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc230) get_ref 0x0~2000 Blob(0x558c584cc230 blob([0xc57000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc2a0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc230) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c584cc230 blob([0xc57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c584cc2a0 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c584cc2b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c584cc310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c584cc380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c584cc310 blob([0xc59000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc380 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc310) get_ref 0x0~2000 Blob(0x558c584cc310 blob([0xc59000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc380 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc310) get_ref init 0x2000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c584cc310 blob([0xc59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c584cc380 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c584cc398 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c584cc3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c584cc460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c584cc3f0 blob([0xc5b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc460 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc3f0) get_ref 0x0~1000 Blob(0x558c584cc3f0 blob([0xc5b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c584cc460 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.blob(0x558c584cc3f0) get_ref init 0x1000, 1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~1000: 0x0~1000 Blob(0x558c584cc3f0 blob([0xc5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c584cc460 sbid 0x0)) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.BufferSpace(0x558c584cc478 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2d000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce388) dirty_range mark inline shard dirty 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x0~2d000 = 0 2019-12-07T19:34:40.649+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.649+0000 7f8f05183700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000063.00000000000000000044' <- 0000000063.00000000000000000044 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b7500 cost 283202 (24 ios * 4000 + 187202 bytes) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b7500 onodes 0x558c585ce280 shared_blobs 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce388) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore.extentmap(0x558c585ce388) update inline shard 437 bytes from 23 extents 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 912 (469 bytes onode + 2 bytes spanning blobs + 441 bytes inline extents) 2019-12-07T19:34:40.649+0000 7f8f05183700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b7500 allocated 0x[c2f000~2d000] released 0x[] 2019-12-07T19:34:40.649+0000 7f8f05183700 10 freelist allocate 0xc2f000~2d000 2019-12-07T19:34:40.649+0000 7f8f05183700 20 freelist _xor first_key 0xc00000 last_key 0xc00000 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7500 prepare 2019-12-07T19:34:40.649+0000 7f8f05183700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c583b7500 2019-12-07T19:34:40.650+0000 7f8f05183700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'44 (0'0,63'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=62'43 crt=62'43 lcod 62'43 active mbc={} ps=[20~1,23~1]] op_applied version 63'44 2019-12-07T19:34:40.650+0000 7f8f05183700 10 osd.7 63 dequeue_op 0x558c58755ce0 finish 2019-12-07T19:34:40.650+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.650+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7500 aio_wait 2019-12-07T19:34:40.650+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b7500 2019-12-07T19:34:40.650+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7500 io_done 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2906 < min_flush_size 524288 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce280 had 1 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2929 < min_flush_size 524288 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2929 < min_flush_size 524288 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x548867~b71 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x548867 0x548867~b71 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x518867 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x867 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x3d8 and padding block with 0xc28 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5493d8 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.650+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000159844s (0.000041863s flush + 0.000117981s kv commit) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583b7500> 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.650+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7500 kv_submitted 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b7500 2019-12-07T19:34:40.650+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7500 finishing 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c583b7500 onodes 0x558c585ce280 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c573960f8 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce120 space 0x558c573960f8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574d3dd8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcd80 space 0x558c574d3dd8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792dc88 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce090 space 0x558c5792dc88 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794fdd8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcfc0 space 0x558c5794fdd8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794feb8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bd050 space 0x558c5794feb8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794ff98 in 0x558c53a59880) _finish_write discard buffer(0x558c580bd0e0 space 0x558c5794ff98 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f90ef8 in 0x558c53a59880) _finish_write discard buffer(0x558c580f37a0 space 0x558c57f90ef8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f91358 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcf30 space 0x558c57f91358 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d3c8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcea0 space 0x558c5800d3c8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5809a8d8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcc60 space 0x558c5809a8d8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580bbba8 in 0x558c53a59880) _finish_write discard buffer(0x558c580bce10 space 0x558c580bbba8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580f7dd8 in 0x558c53a59880) _finish_write discard buffer(0x558c57da4ea0 space 0x558c580f7dd8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c584cc018 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce1b0 space 0x558c584cc018 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c584cc0f8 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce240 space 0x558c584cc0f8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c584cc1d8 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce2d0 space 0x558c584cc1d8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c584cc2b8 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce360 space 0x558c584cc2b8 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c584cc398 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce3f0 space 0x558c584cc398 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c584cc478 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce480 space 0x558c584cc478 0x0~1000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 63/41 ECSubWriteReply(tid=309, last_complete=63'44, committed=1, applied=1)) v2 -- 0x558c54d77e00 con 0x558c54c64000 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585e0e88 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce000 space 0x558c585e0e88 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58bc5c88 in 0x558c53a59880) _finish_write discard buffer(0x558c580bd170 space 0x558c58bc5c88 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58bc5d68 in 0x558c53a59880) _finish_write discard buffer(0x558c580bd200 space 0x558c58bc5d68 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58bc5e48 in 0x558c53a59880) _finish_write discard buffer(0x558c580bd290 space 0x558c58bc5e48 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58bc5f28 in 0x558c53a59880) _finish_write discard buffer(0x558c580bd320 space 0x558c58bc5f28 0x0~2000 writing nocache) 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c583b7500 done 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:40.650+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c583b7500 [] 2019-12-07T19:34:40.650+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.650+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.652+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 545 ==== MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=311, reqid=client.4540.0:470, at_version=63'45, trim_to=0'0, roll_forward_to=62'43)) v2 ==== 145800+0+0 (crc 0 0 0) 0x558c574a2000 con 0x558c54c64000 2019-12-07T19:34:40.652+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c58755e40 prio 127 cost 0 latency 0.000029 epoch 63 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=311, reqid=client.4540.0:470, at_version=63'45, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.652+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=311, reqid=client.4540.0:470, at_version=63'45, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=311, reqid=client.4540.0:470, at_version=63'45, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=311, reqid=client.4540.0:470, at_version=63'45, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c58755e40 prio 127 cost 0 latency 0.000081 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=311, reqid=client.4540.0:470, at_version=63'45, trim_to=0'0, roll_forward_to=62'43)) v2 pg pg[1.7s4( v 63'44 (0'0,63'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'44 (0'0,63'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=311, reqid=client.4540.0:470, at_version=63'45, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'44 (0'0,63'44] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'44], crt=62'43) [63'45 (63'44) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:470 2019-12-07T19:34:40.645988+0000 0 ObjectCleanRegions clean_offsets: [0~737280,1310720~18446744073708240895], clean_omap: 1, new_object: 0] 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'45 (0'0,63'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'44 crt=62'43 lcod 63'44 active mbc={} ps=[20~1,23~1]] add_log_entry 63'45 (63'44) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:470 2019-12-07T19:34:40.645988+0000 0 ObjectCleanRegions clean_offsets: [0~737280,1310720~18446744073708240895], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'45 (0'0,63'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'44 crt=62'43 lcod 63'44 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 45 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'45 (0'0,63'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'44 crt=62'43 lcod 63'44 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.652+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'45, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c584d0000 seq 95 2019-12-07T19:34:40.652+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:40.652+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:40.652+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2d000~23000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ec1285ab:::smithi13140796-19:head# 0x2d000~23000 - have 0x2d000 (184320) bytes fadvise_flags 0x20 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x2d000~23000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c584d0000 18 blobs 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x23000/1000,23000,0 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc5c000~23000/1000,23000,0 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc5c000~23000] 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eb650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97260 blob([0xc5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb650 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97260) get_ref 0x0~2000 Blob(0x558c57b97260 blob([0xc5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb650 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97260) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558c57b97260 blob([0xc5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb650 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c583eb668 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57a07c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4380 blob([0xc5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07c00 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4380) get_ref 0x0~2000 Blob(0x558c585d4380 blob([0xc5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57a07c00 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4380) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558c585d4380 blob([0xc5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57a07c00 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57a07c18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b97180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b97180 blob([0xc60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3880 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97180) get_ref 0x0~2000 Blob(0x558c57b97180 blob([0xc60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3880 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b97180) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558c57b97180 blob([0xc60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3880 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c550b3898 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c579ddc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c579ddc70 blob([0xc62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc70 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c579ddc70) get_ref 0x0~2000 Blob(0x558c579ddc70 blob([0xc62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5794dc70 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c579ddc70) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558c579ddc70 blob([0xc62000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dc70 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5794dc88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792cf50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583eb5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792cf50 blob([0xc64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb5e0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c5792cf50) get_ref 0x0~2000 Blob(0x558c5792cf50 blob([0xc64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb5e0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c5792cf50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558c5792cf50 blob([0xc64000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb5e0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c583eb5f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b96fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b96fc0 blob([0xc66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96fc0) get_ref 0x0~2000 Blob(0x558c57b96fc0 blob([0xc66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b96fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558c57b96fc0 blob([0xc66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d75e30 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c54d75e48 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57b970a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d42a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57b970a0 blob([0xc68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d42a0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b970a0) get_ref 0x0~2000 Blob(0x558c57b970a0 blob([0xc68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d42a0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57b970a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558c57b970a0 blob([0xc68000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d42a0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d42b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4230 blob([0xc6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96ee0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4230) get_ref 0x0~2000 Blob(0x558c585d4230 blob([0xc6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96ee0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4230) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558c585d4230 blob([0xc6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96ee0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b96ef8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d41c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d41c0 blob([0xc6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e00 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d41c0) get_ref 0x0~2000 Blob(0x558c585d41c0 blob([0xc6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57b96e00 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d41c0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558c585d41c0 blob([0xc6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57b96e00 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57b96e18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4150 blob([0xc6e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f10 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4150) get_ref 0x0~2000 Blob(0x558c585d4150 blob([0xc6e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43f10 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4150) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558c585d4150 blob([0xc6e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43f10 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43f28 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d40e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d40e0 blob([0xc70000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43dc0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d40e0) get_ref 0x0~2000 Blob(0x558c585d40e0 blob([0xc70000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43dc0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d40e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558c585d40e0 blob([0xc70000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43dc0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43dd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57897d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4070 blob([0xc72000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897d50 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4070) get_ref 0x0~2000 Blob(0x558c585d4070 blob([0xc72000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57897d50 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4070) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558c585d4070 blob([0xc72000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57897d50 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57897d68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4000 blob([0xc74000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e70 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4000) get_ref 0x0~2000 Blob(0x558c585d4000 blob([0xc74000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42e70 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4000) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558c585d4000 blob([0xc74000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42e70 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42e88 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57567880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57567880 blob([0xc76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42f50 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57567880) get_ref 0x0~2000 Blob(0x558c57567880 blob([0xc76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42f50 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57567880) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558c57567880 blob([0xc76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c42f50 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c42f68 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bea380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c43030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bea380 blob([0xc78000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43030 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57bea380) get_ref 0x0~2000 Blob(0x558c57bea380 blob([0xc78000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c43030 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c57bea380) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558c57bea380 blob([0xc78000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c43030 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c43048 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c424d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad3b0 blob([0xc7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c424d0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c576ad3b0) get_ref 0x0~2000 Blob(0x558c576ad3b0 blob([0xc7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c424d0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c576ad3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558c576ad3b0 blob([0xc7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c424d0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c424e8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c576ad2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c425b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c576ad2d0 blob([0xc7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c425b0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c576ad2d0) get_ref 0x0~2000 Blob(0x558c576ad2d0 blob([0xc7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c425b0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c576ad2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558c576ad2d0 blob([0xc7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57c425b0 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c425c8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54df89a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57c42690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54df89a0 blob([!~1000,0xc7e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42690 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c54df89a0) get_ref 0x1000~1000 Blob(0x558c54df89a0 blob([!~1000,0xc7e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57c42690 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.blob(0x558c54df89a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4f000~1000: 0x1000~1000 Blob(0x558c54df89a0 blob([!~1000,0xc7e000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558c57c42690 sbid 0x0)) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57c426a8 in 0x558c53a59880) _discard 0x1000~1000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x50000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.extentmap(0x558c585ce388) dirty_range mark inline shard dirty 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x2d000~23000 = 0 2019-12-07T19:34:40.652+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.652+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000063.00000000000000000045' <- 0000000063.00000000000000000045 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584d0000 cost 221450 (19 ios * 4000 + 145450 bytes) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584d0000 onodes 0x558c585ce280 shared_blobs 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.extentmap(0x558c585ce388) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore.extentmap(0x558c585ce388) update inline shard 793 bytes from 41 extents 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1268 (469 bytes onode + 2 bytes spanning blobs + 797 bytes inline extents) 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584d0000 allocated 0x[c5c000~23000] released 0x[] 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 freelist allocate 0xc5c000~23000 2019-12-07T19:34:40.652+0000 7f8f0117b700 20 freelist _xor first_key 0xc00000 last_key 0xc00000 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0000 prepare 2019-12-07T19:34:40.652+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c584d0000 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'45 (0'0,63'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'44 crt=62'43 lcod 63'44 active mbc={} ps=[20~1,23~1]] op_applied version 63'45 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c58755e40 finish 2019-12-07T19:34:40.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0000 aio_wait 2019-12-07T19:34:40.653+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584d0000 2019-12-07T19:34:40.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0000 io_done 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2464 < min_flush_size 524288 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce280 had 1 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2487 < min_flush_size 524288 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2487 < min_flush_size 524288 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x5493d8~9b7 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x5493d8 0x5493d8~9b7 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x5193d8 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x3d8 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd8f and padding block with 0x271 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x549d8f 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000155089s (0.000046071s flush + 0.000109018s kv commit) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584d0000> 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0000 kv_submitted 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584d0000 2019-12-07T19:34:40.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0000 finishing 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584d0000 onodes 0x558c585ce280 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c54d75e48 in 0x558c53a59880) _finish_write discard buffer(0x558c580bcbd0 space 0x558c54d75e48 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3898 in 0x558c53a59880) _finish_write discard buffer(0x558c580bccf0 space 0x558c550b3898 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57897d68 in 0x558c53a59880) _finish_write discard buffer(0x558c584cec60 space 0x558c57897d68 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5794dc88 in 0x558c53a59880) _finish_write discard buffer(0x558c58b85c20 space 0x558c5794dc88 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57a07c18 in 0x558c53a59880) _finish_write discard buffer(0x558c57669560 space 0x558c57a07c18 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96e18 in 0x558c53a59880) _finish_write discard buffer(0x558c584ceab0 space 0x558c57b96e18 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57b96ef8 in 0x558c53a59880) _finish_write discard buffer(0x558c57ecea20 space 0x558c57b96ef8 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c424e8 in 0x558c53a59880) _finish_write discard buffer(0x558c584ceea0 space 0x558c57c424e8 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c425c8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cef30 space 0x558c57c425c8 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c426a8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cefc0 space 0x558c57c426a8 0x1000~1000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42e88 in 0x558c53a59880) _finish_write discard buffer(0x558c584cecf0 space 0x558c57c42e88 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c42f68 in 0x558c53a59880) _finish_write discard buffer(0x558c584ced80 space 0x558c57c42f68 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43048 in 0x558c53a59880) _finish_write discard buffer(0x558c584cee10 space 0x558c57c43048 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43dd8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cebd0 space 0x558c57c43dd8 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57c43f28 in 0x558c53a59880) _finish_write discard buffer(0x558c584ceb40 space 0x558c57c43f28 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eb5f8 in 0x558c53a59880) _finish_write discard buffer(0x558c5813b5f0 space 0x558c583eb5f8 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583eb668 in 0x558c53a59880) _finish_write discard buffer(0x558c58534000 space 0x558c583eb668 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d42b8 in 0x558c53a59880) _finish_write discard buffer(0x558c57ece990 space 0x558c585d42b8 0x0~2000 writing nocache) 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584d0000 done 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:40.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584d0000 [] 2019-12-07T19:34:40.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.653+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 63/41 ECSubWriteReply(tid=311, last_complete=63'45, committed=1, applied=1)) v2 -- 0x558c54d77400 con 0x558c54c64000 2019-12-07T19:34:40.653+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.653+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 546 ==== MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=314, reqid=client.4540.0:471, at_version=63'46, trim_to=0'0, roll_forward_to=62'43)) v2 ==== 51592+0+0 (crc 0 0 0) 0x558c5822e800 con 0x558c54c64000 2019-12-07T19:34:40.653+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c57f3e2c0 prio 127 cost 0 latency 0.000026 epoch 63 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=314, reqid=client.4540.0:471, at_version=63'46, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.653+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=314, reqid=client.4540.0:471, at_version=63'46, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=314, reqid=client.4540.0:471, at_version=63'46, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.653+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=314, reqid=client.4540.0:471, at_version=63'46, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3e2c0 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=314, reqid=client.4540.0:471, at_version=63'46, trim_to=0'0, roll_forward_to=62'43)) v2 pg pg[1.7s4( v 63'45 (0'0,63'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.653+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1579408 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'45 (0'0,63'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=314, reqid=client.4540.0:471, at_version=63'46, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'45 (0'0,63'45] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'45], crt=62'43) [63'46 (63'45) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:471 2019-12-07T19:34:40.647618+0000 0 ObjectCleanRegions clean_offsets: [0~1310720,1507328~18446744073708044287], clean_omap: 1, new_object: 0] 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'46 (0'0,63'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'45 crt=62'43 lcod 63'45 active mbc={} ps=[20~1,23~1]] add_log_entry 63'46 (63'45) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:471 2019-12-07T19:34:40.647618+0000 0 ObjectCleanRegions clean_offsets: [0~1310720,1507328~18446744073708044287], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'46 (0'0,63'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'45 crt=62'43 lcod 63'45 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 46 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'46 (0'0,63'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'45 crt=62'43 lcod 63'45 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.653+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'46, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.653+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:40.653+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c584d0300 seq 96 2019-12-07T19:34:40.653+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x50000~c000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ec1285ab:::smithi13140796-19:head# 0x50000~c000 - have 0x50000 (327680) bytes fadvise_flags 0x20 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x50000~c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c584d0300 6 blobs 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0xc000/1000,c000,0 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc7f000~c000/1000,c000,0 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc7f000~c000] 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585e0310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5800d490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585e0310 blob([0xc7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d490 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c585e0310) get_ref 0x0~2000 Blob(0x558c585e0310 blob([0xc7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5800d490 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c585e0310) get_ref init 0x2000, 1000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c585e0310 blob([0xc7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d490 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5800d4a8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c574e2620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c574e2620 blob([0xc81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c00 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c574e2620) get_ref 0x0~2000 Blob(0x558c574e2620 blob([0xc81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5c00 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c574e2620) get_ref init 0x2000, 1000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c574e2620 blob([0xc81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5c00 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c580a5c18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5ce0 blob([0xc83000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5ce0) get_ref 0x0~2000 Blob(0x558c580a5ce0 blob([0xc83000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c580a5ce0 blob([0xc83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e27e0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c574e27f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5490 blob([0xc85000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5490) get_ref 0x0~2000 Blob(0x558c580a5490 blob([0xc85000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5490) get_ref init 0x2000, 1000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c580a5490 blob([0xc85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e28c0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c574e28d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5650 blob([0xc87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5650) get_ref 0x0~2000 Blob(0x558c580a5650 blob([0xc87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5650) get_ref init 0x2000, 1000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c580a5650 blob([0xc87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e29a0 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c574e29b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580a5570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580a5570 blob([0xc89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5570) get_ref 0x0~2000 Blob(0x558c580a5570 blob([0xc89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.blob(0x558c580a5570) get_ref init 0x2000, 1000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c580a5570 blob([0xc89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c574e0c40 sbid 0x0)) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c574e0c58 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5c000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.extentmap(0x558c585ce388) dirty_range mark inline shard dirty 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 0x50000~c000 = 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000063.00000000000000000046' <- 0000000063.00000000000000000046 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584d0300 cost 79242 (7 ios * 4000 + 51242 bytes) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584d0300 onodes 0x558c585ce280 shared_blobs 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.extentmap(0x558c585ce388) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore.extentmap(0x558c585ce388) update inline shard 908 bytes from 47 extents 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1383 (469 bytes onode + 2 bytes spanning blobs + 912 bytes inline extents) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584d0300 allocated 0x[c7f000~c000] released 0x[] 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 freelist allocate 0xc7f000~c000 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 freelist _xor first_key 0xc00000 last_key 0xc80000 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0300 prepare 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c584d0300 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'46 (0'0,63'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'45 crt=62'43 lcod 63'45 active mbc={} ps=[20~1,23~1]] op_applied version 63'46 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3e2c0 finish 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.654+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0300 aio_wait 2019-12-07T19:34:40.654+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584d0300 2019-12-07T19:34:40.654+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0300 io_done 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2608 < min_flush_size 524288 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce280 had 1 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2631 < min_flush_size 524288 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2631 < min_flush_size 524288 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x549d8f~a47 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x549d8f 0x549d8f~a47 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x519d8f 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd8f 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x7d6 and padding block with 0x82a 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x54a7d6 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.654+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000149860s (0.000041181s flush + 0.000108679s kv commit) 2019-12-07T19:34:40.654+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584d0300> 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0300 kv_submitted 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584d0300 2019-12-07T19:34:40.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0300 finishing 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584d0300 onodes 0x558c585ce280 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e0c58 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf320 space 0x558c574e0c58 0x0~2000 writing nocache) 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e27f8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf170 space 0x558c574e27f8 0x0~2000 writing nocache) 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e28d8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf200 space 0x558c574e28d8 0x0~2000 writing nocache) 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c574e29b8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf290 space 0x558c574e29b8 0x0~2000 writing nocache) 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5800d4a8 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce510 space 0x558c5800d4a8 0x0~2000 writing nocache) 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5c18 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf0e0 space 0x558c580a5c18 0x0~2000 writing nocache) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584d0300 done 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:40.654+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584d0300 [] 2019-12-07T19:34:40.654+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.654+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 63/41 ECSubWriteReply(tid=314, last_complete=63'46, committed=1, applied=1)) v2 -- 0x558c54ceb800 con 0x558c54c64000 2019-12-07T19:34:40.654+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.654+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.654+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 547 ==== MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=315, reqid=client.4540.0:472, at_version=63'47, trim_to=0'0, roll_forward_to=62'43)) v2 ==== 2347+0+0 (crc 0 0 0) 0x558c58230800 con 0x558c54c64000 2019-12-07T19:34:40.654+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c57f3f760 prio 127 cost 0 latency 0.000027 epoch 63 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=315, reqid=client.4540.0:472, at_version=63'47, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.654+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=315, reqid=client.4540.0:472, at_version=63'47, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=315, reqid=client.4540.0:472, at_version=63'47, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=315, reqid=client.4540.0:472, at_version=63'47, trim_to=0'0, roll_forward_to=62'43)) v2) prio 127 cost 0 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3f760 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=315, reqid=client.4540.0:472, at_version=63'47, trim_to=0'0, roll_forward_to=62'43)) v2 pg pg[1.7s4( v 63'46 (0'0,63'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'46 (0'0,63'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=315, reqid=client.4540.0:472, at_version=63'47, trim_to=0'0, roll_forward_to=62'43)) v2 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'46 (0'0,63'46] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'46], crt=62'43) [63'47 (63'46) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:472 2019-12-07T19:34:40.647635+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'46 crt=62'43 lcod 63'46 active mbc={} ps=[20~1,23~1]] add_log_entry 63'47 (63'46) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:472 2019-12-07T19:34:40.647635+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'46 crt=62'43 lcod 63'46 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 47 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'46 crt=62'43 lcod 63'46 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.654+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'47, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c584d0600 seq 97 2019-12-07T19:34:40.654+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 3 keys 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# 3 keys = 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.654+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.654+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:40.654+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000063.00000000000000000047' <- 0000000063.00000000000000000047 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.655+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:40.655+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584d0600 cost 5997 (1 ios * 4000 + 1997 bytes) 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584d0600 onodes 0x558c585ce280 shared_blobs 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore.extentmap(0x558c585ce388) update 4#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ec1285ab:::smithi13140796-19:head# is 1456 (542 bytes onode + 2 bytes spanning blobs + 912 bytes inline extents) 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584d0600 allocated 0x[] released 0x[] 2019-12-07T19:34:40.655+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0600 prepare 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584d0600 2019-12-07T19:34:40.655+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0600 io_done 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.655+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'46 crt=62'43 lcod 63'46 active mbc={} ps=[20~1,23~1]] op_applied version 63'47 2019-12-07T19:34:40.655+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3f760 finish 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2552 < min_flush_size 524288 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c585ce280 had 1 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2575 < min_flush_size 524288 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2575 < min_flush_size 524288 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x54a7d6~a0f to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x54a7d6 0x54a7d6~a0f to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x51a7d6 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x7d6 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1e5 and padding block with 0xe1b 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x54b1e5 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000323696s (0.000000120s flush + 0.000323576s kv commit) 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584d0600> 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0600 kv_submitted 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584d0600 2019-12-07T19:34:40.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0600 finishing 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584d0600 onodes 0x558c585ce280 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584d0600 done 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:40.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584d0600 [] 2019-12-07T19:34:40.655+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.655+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.655+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 63/41 ECSubWriteReply(tid=315, last_complete=63'47, committed=1, applied=1)) v2 -- 0x558c54ceb600 con 0x558c54c64000 2019-12-07T19:34:40.655+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.657+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 324 ==== MOSDECSubOpRead(1.1s1 63/41 ECSubRead(tid=167, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x558c58bd6280 con 0x558c54c5bc00 2019-12-07T19:34:40.657+0000 7f8f23c56700 15 osd.7 63 enqueue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000019 epoch 63 MOSDECSubOpRead(1.1s1 63/41 ECSubRead(tid=167, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:40.657+0000 7f8f23c56700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 63/41 ECSubRead(tid=167, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e63) 2019-12-07T19:34:40.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 63/41 ECSubRead(tid=167, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 63/41 ECSubRead(tid=167, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e63) pg 0x558c5736a000 2019-12-07T19:34:40.657+0000 7f8f08189700 10 osd.7 63 dequeue_op 0x558c5740d1e0 prio 127 cost 0 latency 0.000059 MOSDECSubOpRead(1.1s1 63/41 ECSubRead(tid=167, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.657+0000 7f8f08189700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpRead(1.1s1 63/41 ECSubRead(tid=167, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:40.657+0000 7f8f08189700 25 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:40.657+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~4d000 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~4d000 size 0x4d000 (315392) 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read defaulting to buffered read 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcb570 blob([0x740000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbb90 blob([0x742000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcaa80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbb20 blob([0x744000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d3d960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54d3d9d0 blob([0x746000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576ad260 blob([0x748000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcb650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbc70 blob([0x74a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c54d74ee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcaa10 blob([0x74c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbc00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbce0 blob([0x74e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbd50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbdc0 blob([0x750000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbe30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbea0 blob([0x752000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57dcbf10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57dcbf80 blob([0x754000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d730 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800d7a0 blob([0x756000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800d880 blob([0x758000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d8f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800d960 blob([0x75a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800d9d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800da40 blob([0x75c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800db20 blob([0x75e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800db90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800dc00 blob([0x760000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dc70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800dce0 blob([0x762000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800dd50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ddc0 blob([0x764000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800de30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800dea0 blob([0x766000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800df10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800df80 blob([0x768000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66070 blob([0x76a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f660e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66150 blob([0x76c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f661c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66230 blob([0x76e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f662a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66310 blob([0x770000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66380 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f663f0 blob([0x772000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f664d0 blob([0x774000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66540 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f665b0 blob([0x776000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66690 blob([0x778000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f66700 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66770 blob([0x77a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f667e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66850 blob([0x77c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f668c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c57f66930 blob([0x77e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f669a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c576acfc0 blob([0x780000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5794dd50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5792c000 blob([0x782000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792c380 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54df8a10 blob([0x784000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cc40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ccb0 blob([0x786000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cd20 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800cd90 blob([0x788000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800ce00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800ce70 blob([0x78a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5800cee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:40.657+0000 7f8f08189700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c5800cf50 blob([0x78c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5800cfc0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:40.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~4d000 = 315392 2019-12-07T19:34:40.657+0000 7f8f08189700 20 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_sub_read read request=315392 r=315392 len=315392 2019-12-07T19:34:40.657+0000 7f8f08189700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:40.657+0000 7f8f08189700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:40.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:40.657+0000 7f8f08189700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:40.657+0000 7f8f08189700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:34:40.657+0000 7f8f08189700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:34:40.657+0000 7f8f08189700 20 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_sub_read: Checking hash of 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:40.657+0000 7f8f08189700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 63/41 ECSubReadReply(tid=167, attrs_read=0)) v2 -- 0x558c5895c280 con 0x558c54c5bc00 2019-12-07T19:34:40.657+0000 7f8f08189700 10 osd.7 63 dequeue_op 0x558c5740d1e0 finish 2019-12-07T19:34:40.657+0000 7f8f08189700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:40.658+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 548 ==== MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=316, reqid=client.4540.0:472, at_version=0'0, trim_to=0'0, roll_forward_to=63'47)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5748e800 con 0x558c54c64000 2019-12-07T19:34:40.658+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c57f3e6e0 prio 127 cost 0 latency 0.000022 epoch 63 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=316, reqid=client.4540.0:472, at_version=0'0, trim_to=0'0, roll_forward_to=63'47)) v2 2019-12-07T19:34:40.658+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=316, reqid=client.4540.0:472, at_version=0'0, trim_to=0'0, roll_forward_to=63'47)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=316, reqid=client.4540.0:472, at_version=0'0, trim_to=0'0, roll_forward_to=63'47)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=316, reqid=client.4540.0:472, at_version=0'0, trim_to=0'0, roll_forward_to=63'47)) v2) prio 127 cost 0 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3e6e0 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=316, reqid=client.4540.0:472, at_version=0'0, trim_to=0'0, roll_forward_to=63'47)) v2 pg pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=316, reqid=client.4540.0:472, at_version=0'0, trim_to=0'0, roll_forward_to=63'47)) v2 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'43 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'47], crt=62'43) [] 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'44 (62'43) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:469 2019-12-07T19:34:40.642449+0000 0 ObjectCleanRegions clean_offsets: [737280~18446744073708814335], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'45 (63'44) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:470 2019-12-07T19:34:40.645988+0000 0 ObjectCleanRegions clean_offsets: [0~737280,1310720~18446744073708240895], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'46 (63'45) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:471 2019-12-07T19:34:40.647618+0000 0 ObjectCleanRegions clean_offsets: [0~1310720,1507328~18446744073708044287], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'47 (63'46) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:472 2019-12-07T19:34:40.647635+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 47 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.658+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c584d0900 seq 98 2019-12-07T19:34:40.658+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584d0900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584d0900 onodes shared_blobs 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584d0900 allocated 0x[] released 0x[] 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0900 prepare 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584d0900 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0900 io_done 2019-12-07T19:34:40.658+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3e6e0 finish 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x54b1e5~471 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x54b1e5 0x54b1e5~471 to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~520000 x_off 0x51b1e5 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x1e5 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x656 and padding block with 0x9aa 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x54b656 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.658+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000123521s (0.000000073s flush + 0.000123448s kv commit) 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584d0900> 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0900 kv_submitted 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584d0900 2019-12-07T19:34:40.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0900 finishing 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584d0900 onodes 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584d0900 done 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:40.658+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584d0900 [] 2019-12-07T19:34:40.658+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.658+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.658+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.658+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 63/41 ECSubWriteReply(tid=316, last_complete=63'47, committed=1, applied=1)) v2 -- 0x558c58114400 con 0x558c54c64000 2019-12-07T19:34:40.658+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.664+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 549 ==== MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=318, reqid=client.4540.0:480, at_version=63'49, trim_to=0'0, roll_forward_to=63'47)) v2 ==== 372763+0+0 (crc 0 0 0) 0x558c575bf800 con 0x558c54c64000 2019-12-07T19:34:40.664+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c57f3f1e0 prio 127 cost 0 latency 0.000027 epoch 63 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=318, reqid=client.4540.0:480, at_version=63'49, trim_to=0'0, roll_forward_to=63'47)) v2 2019-12-07T19:34:40.664+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=318, reqid=client.4540.0:480, at_version=63'49, trim_to=0'0, roll_forward_to=63'47)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=318, reqid=client.4540.0:480, at_version=63'49, trim_to=0'0, roll_forward_to=63'47)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=318, reqid=client.4540.0:480, at_version=63'49, trim_to=0'0, roll_forward_to=63'47)) v2) prio 127 cost 0 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3f1e0 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=318, reqid=client.4540.0:480, at_version=63'49, trim_to=0'0, roll_forward_to=63'47)) v2 pg pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=318, reqid=client.4540.0:480, at_version=63'49, trim_to=0'0, roll_forward_to=63'47)) v2 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.add_oid 1:ecc72bdb:::smithi13140796-10:28 21,25,26,28 2019-12-07T19:34:40.664+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.onode(0x558c548f2280).flush flush done 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.get_snaps 1:ecc72bdb:::smithi13140796-10:28 got.empty() 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.set_snaps 1:ecc72bdb:::smithi13140796-10:28 21,25,26,28 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.set_snaps set OBJ_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.add_oid set SNA_1_0000000000000021_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.add_oid set SNA_1_0000000000000025_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.add_oid set SNA_1_0000000000000026_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 snap_mapper.add_oid set SNA_1_0000000000000028_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'47 (0'0,63'47] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'47], crt=63'47) [63'48 (57'39) clone 1:ecc72bdb:::smithi13140796-10:28 by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [28,26,25,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,63'49 (57'39) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:480 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0] 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'48 (0'0,63'48] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'47 crt=63'47 lcod 63'47 active mbc={} ps=[20~1,23~1]] add_log_entry 63'48 (57'39) clone 1:ecc72bdb:::smithi13140796-10:28 by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [28,26,25,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'47 crt=63'47 lcod 63'47 active mbc={} ps=[20~1,23~1]] add_log_entry 63'49 (57'39) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:480 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'47 crt=63'47 lcod 63'47 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 49 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'47 crt=63'47 lcod 63'47 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.664+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 63'48, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c584d0f00 seq 99 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ecc72bdb:::smithi13140796-10:28# key 0x848000000000000001ecc72bdb21736d'ithi13140796-10!='0x0000000000000028ffffffffffffffff'o' 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:40.664+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:28# 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1295 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.onode(0x558c575a2a00).flush flush done 2019-12-07T19:34:40.664+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:28# 0x0 2019-12-07T19:34:40.664+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_clone_range 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:28# 0x0~33000 -> 0x0~33000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x0~2000: 0x0~2000 Blob(0x558c583eb6c0 blob([0xab0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb7a0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11007 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c583eb6c0 blob([0xab0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb7a0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c583eb6c0 blob([0xab0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c57538930 blob([0xab0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57538930) get_ref 0x0~2000 Blob(0x558c57538930 blob([0xab0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57538930) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x0~2000: 0x0~2000 Blob(0x558c57538930 blob([0xab0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x2000~2000: 0x0~2000 Blob(0x558c583eb810 blob([0xab2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb880 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11008 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c583eb810 blob([0xab2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb880 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c583eb810 blob([0xab2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c57897730 blob([0xab2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57897730) get_ref 0x0~2000 Blob(0x558c57897730 blob([0xab2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57897730) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558c57897730 blob([0xab2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x4000~2000: 0x0~2000 Blob(0x558c583eb8f0 blob([0xab4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83570 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11009 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c583eb8f0 blob([0xab4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83570 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c583eb8f0 blob([0xab4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c5794df10 blob([0xab4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5794df10) get_ref 0x0~2000 Blob(0x558c5794df10 blob([0xab4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5794df10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558c5794df10 blob([0xab4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x6000~2000: 0x0~2000 Blob(0x558c58b83500 blob([0xab6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83490 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11010 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c58b83500 blob([0xab6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83490 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c58b83500 blob([0xab6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c57538a10 blob([0xab6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57538a10) get_ref 0x0~2000 Blob(0x558c57538a10 blob([0xab6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57538a10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558c57538a10 blob([0xab6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x8000~2000: 0x0~2000 Blob(0x558c58b83c00 blob([0xab8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83c70 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11011 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c58b83c00 blob([0xab8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83c70 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c58b83c00 blob([0xab8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c5742b810 blob([0xab8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5742b810) get_ref 0x0~2000 Blob(0x558c5742b810 blob([0xab8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5742b810) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558c5742b810 blob([0xab8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0xa000~2000: 0x0~2000 Blob(0x558c58b83ce0 blob([0xaba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83d50 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11012 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c58b83ce0 blob([0xaba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83d50 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c58b83ce0 blob([0xaba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c57567b90 blob([0xaba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57567b90) get_ref 0x0~2000 Blob(0x558c57567b90 blob([0xaba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57567b90) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558c57567b90 blob([0xaba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0xc000~2000: 0x0~2000 Blob(0x558c58b83dc0 blob([0xabc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83e30 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11013 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c58b83dc0 blob([0xabc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83e30 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c58b83dc0 blob([0xabc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c575bd9d0 blob([0xabc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c575bd9d0) get_ref 0x0~2000 Blob(0x558c575bd9d0 blob([0xabc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c575bd9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558c575bd9d0 blob([0xabc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0xe000~2000: 0x0~2000 Blob(0x558c58b83ea0 blob([0xabe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83f10 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11014 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c58b83ea0 blob([0xabe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83f10 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c58b83ea0 blob([0xabe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c57566d90 blob([0xabe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57566d90) get_ref 0x0~2000 Blob(0x558c57566d90 blob([0xabe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57566d90) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558c57566d90 blob([0xabe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x10000~2000: 0x0~2000 Blob(0x558c58b83f80 blob([0xac0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba000 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11015 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c58b83f80 blob([0xac0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba000 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c58b83f80 blob([0xac0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c5757e8c0 blob([0xac0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e8c0) get_ref 0x0~2000 Blob(0x558c5757e8c0 blob([0xac0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5757e8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558c5757e8c0 blob([0xac0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x12000~2000: 0x0~2000 Blob(0x558c580ba070 blob([0xac2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba0e0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11016 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba070 blob([0xac2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba0e0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba070 blob([0xac2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54e43490 blob([0xac2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43490) get_ref 0x0~2000 Blob(0x558c54e43490 blob([0xac2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43490) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558c54e43490 blob([0xac2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x14000~2000: 0x0~2000 Blob(0x558c580ba150 blob([0xac4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba1c0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11017 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba150 blob([0xac4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba1c0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba150 blob([0xac4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c57396230 blob([0xac4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57396230) get_ref 0x0~2000 Blob(0x558c57396230 blob([0xac4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57396230) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558c57396230 blob([0xac4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x16000~2000: 0x0~2000 Blob(0x558c580ba230 blob([0xac6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba2a0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11018 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba230 blob([0xac6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba2a0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba230 blob([0xac6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54e433b0 blob([0xac6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e433b0) get_ref 0x0~2000 Blob(0x558c54e433b0 blob([0xac6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e433b0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558c54e433b0 blob([0xac6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x18000~2000: 0x0~2000 Blob(0x558c580ba310 blob([0xac8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba380 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11019 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba310 blob([0xac8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba380 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba310 blob([0xac8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c573967e0 blob([0xac8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c573967e0) get_ref 0x0~2000 Blob(0x558c573967e0 blob([0xac8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c573967e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558c573967e0 blob([0xac8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558c580ba3f0 blob([0xaca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba460 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11020 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba3f0 blob([0xaca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba460 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba3f0 blob([0xaca000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54e432d0 blob([0xaca000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e432d0) get_ref 0x0~2000 Blob(0x558c54e432d0 blob([0xaca000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e432d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558c54e432d0 blob([0xaca000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558c580ba4d0 blob([0xacc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba540 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11021 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba4d0 blob([0xacc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba540 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba4d0 blob([0xacc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c57ed1f10 blob([0xacc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57ed1f10) get_ref 0x0~2000 Blob(0x558c57ed1f10 blob([0xacc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c57ed1f10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558c57ed1f10 blob([0xacc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558c580ba5b0 blob([0xace000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba620 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11022 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba5b0 blob([0xace000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba620 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba5b0 blob([0xace000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54e431f0 blob([0xace000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e431f0) get_ref 0x0~2000 Blob(0x558c54e431f0 blob([0xace000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e431f0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558c54e431f0 blob([0xace000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x20000~2000: 0x0~2000 Blob(0x558c580ba690 blob([0xad0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba700 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11023 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba690 blob([0xad0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba700 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba690 blob([0xad0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54d74cb0 blob([0xad0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74cb0) get_ref 0x0~2000 Blob(0x558c54d74cb0 blob([0xad0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54d74cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558c54d74cb0 blob([0xad0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x22000~2000: 0x0~2000 Blob(0x558c580ba770 blob([0xad2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba7e0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11024 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba770 blob([0xad2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba7e0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba770 blob([0xad2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54e43110 blob([0xad2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43110) get_ref 0x0~2000 Blob(0x558c54e43110 blob([0xad2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43110) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558c54e43110 blob([0xad2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x24000~2000: 0x0~2000 Blob(0x558c580ba850 blob([0xad4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba8c0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11025 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba850 blob([0xad4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba8c0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba850 blob([0xad4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c5829c9a0 blob([0xad4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5829c9a0) get_ref 0x0~2000 Blob(0x558c5829c9a0 blob([0xad4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5829c9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x24000~2000: 0x0~2000 Blob(0x558c5829c9a0 blob([0xad4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x26000~2000: 0x0~2000 Blob(0x558c580ba930 blob([0xad6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba9a0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11026 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580ba930 blob([0xad6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba9a0 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580ba930 blob([0xad6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54e43030 blob([0xad6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43030) get_ref 0x0~2000 Blob(0x558c54e43030 blob([0xad6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54e43030) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x26000~2000: 0x0~2000 Blob(0x558c54e43030 blob([0xad6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x28000~2000: 0x0~2000 Blob(0x558c580baa10 blob([0xad8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580baa80 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11027 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580baa10 blob([0xad8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580baa80 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580baa10 blob([0xad8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c575677a0 blob([0xad8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c575677a0) get_ref 0x0~2000 Blob(0x558c575677a0 blob([0xad8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c575677a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x28000~2000: 0x0~2000 Blob(0x558c575677a0 blob([0xad8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x2a000~2000: 0x0~2000 Blob(0x558c580baaf0 blob([0xada000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bab60 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11028 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580baaf0 blob([0xada000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bab60 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580baaf0 blob([0xada000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c55265a40 blob([0xada000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c55265a40) get_ref 0x0~2000 Blob(0x558c55265a40 blob([0xada000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c55265a40) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x2a000~2000: 0x0~2000 Blob(0x558c55265a40 blob([0xada000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x2c000~2000: 0x0~2000 Blob(0x558c580babd0 blob([0xadc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bac40 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11029 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580babd0 blob([0xadc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bac40 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580babd0 blob([0xadc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c5829ca10 blob([0xadc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5829ca10) get_ref 0x0~2000 Blob(0x558c5829ca10 blob([0xadc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c5829ca10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x2c000~2000: 0x0~2000 Blob(0x558c5829ca10 blob([0xadc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x2e000~2000: 0x0~2000 Blob(0x558c580bacb0 blob([0xade000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bad20 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11030 2019-12-07T19:34:40.664+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580bacb0 blob([0xade000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bad20 sbid 0x0)) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580bacb0 blob([0xade000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=1)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54d3ccb0 blob([0xade000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3ccb0) get_ref 0x0~2000 Blob(0x558c54d3ccb0 blob([0xade000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3ccb0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x2e000~2000: 0x0~2000 Blob(0x558c54d3ccb0 blob([0xade000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=2)))) 2019-12-07T19:34:40.664+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x30000~2000: 0x0~2000 Blob(0x558c580bad90 blob([0xae0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bae00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11031 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580bad90 blob([0xae0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bae00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580bad90 blob([0xae0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=1)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c550b2af0 blob([0xae0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=2)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c550b2af0) get_ref 0x0~2000 Blob(0x558c550b2af0 blob([0xae0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=2)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c550b2af0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x30000~2000: 0x0~2000 Blob(0x558c550b2af0 blob([0xae0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=2)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup src 0x32000~1000: 0x0~1000 Blob(0x558c580bae70 blob([0xae2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580baee0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_blobid 11032 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared Blob(0x558c580bae70 blob([0xae2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580baee0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) make_blob_shared now Blob(0x558c580bae70 blob([0xae2000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=1)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup new Blob(0x558c54d3d030 blob([0xae2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=2)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d030) get_ref 0x0~1000 Blob(0x558c54d3d030 blob([0xae2000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=2)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d030) get_ref init 0x1000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dup dst 0x32000~1000: 0x0~1000 Blob(0x558c54d3d030 blob([0xae2000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=2)))) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795ab08) dirty_range mark inline shard dirty 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _clone 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:28# = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:28# snapset 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rmattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:28# snapset = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:28# 1 keys 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:28# 1 keys = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:28# hinfo_key (42 bytes) 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:28# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) get_onode oid 4#1:ecc72bdb:::smithi13140796-10:head#31 key 0x848000000000000001ecc72bdb21736d'ithi13140796-10!='0xfffffffffffffffe0000000000000031'o' 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.7s4_head 0x558c54d39680) r -2 v.len 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:head#31 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _rename 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# -> 4#1:ecc72bdb:::smithi13140796-10:head#31 = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1296 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 23 keys 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 23 keys = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~4d000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~4d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~4d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c584d0f00 39 blobs 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate 0x4d000/1000,4d000,0 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xc8b000~4d000/1000,4d000,0 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xc8b000~4d000] 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3d180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5829c930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3d180 blob([0xc8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c930 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d180) get_ref 0x0~2000 Blob(0x558c54d3d180 blob([0xc8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5829c930 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c54d3d180) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c54d3d180 blob([0xc8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5829c930 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c5829c948 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d43f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d43f0 blob([0xc8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4460 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d43f0) get_ref 0x0~2000 Blob(0x558c585d43f0 blob([0xc8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4460 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d43f0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c585d43f0 blob([0xc8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4460 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4478 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d44d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d44d0 blob([0xc8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4540 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d44d0) get_ref 0x0~2000 Blob(0x558c585d44d0 blob([0xc8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4540 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d44d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c585d44d0 blob([0xc8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4540 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4558 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d45b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d45b0 blob([0xc91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4620 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d45b0) get_ref 0x0~2000 Blob(0x558c585d45b0 blob([0xc91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4620 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d45b0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c585d45b0 blob([0xc91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4620 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4638 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4690 blob([0xc93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4700 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4690) get_ref 0x0~2000 Blob(0x558c585d4690 blob([0xc93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4700 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4690) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c585d4690 blob([0xc93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4700 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4718 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d47e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4770 blob([0xc95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d47e0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4770) get_ref 0x0~2000 Blob(0x558c585d4770 blob([0xc95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d47e0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4770) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c585d4770 blob([0xc95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d47e0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d47f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d48c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4850 blob([0xc97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d48c0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4850) get_ref 0x0~2000 Blob(0x558c585d4850 blob([0xc97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d48c0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4850) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c585d4850 blob([0xc97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d48c0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d48d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d49a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4930 blob([0xc99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d49a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4930) get_ref 0x0~2000 Blob(0x558c585d4930 blob([0xc99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d49a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4930) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c585d4930 blob([0xc99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d49a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d49b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4a10 blob([0xc9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4a80 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4a10) get_ref 0x0~2000 Blob(0x558c585d4a10 blob([0xc9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4a80 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4a10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c585d4a10 blob([0xc9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4a80 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4a98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4af0 blob([0xc9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4b60 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4af0) get_ref 0x0~2000 Blob(0x558c585d4af0 blob([0xc9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4b60 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4af0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c585d4af0 blob([0xc9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4b60 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4b78 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4bd0 blob([0xc9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4c40 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4bd0) get_ref 0x0~2000 Blob(0x558c585d4bd0 blob([0xc9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4c40 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c585d4bd0 blob([0xc9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4c40 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4c58 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585d4d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4cb0 blob([0xca1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4d20 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4cb0) get_ref 0x0~2000 Blob(0x558c585d4cb0 blob([0xca1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585d4d20 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c585d4cb0 blob([0xca1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c585d4d20 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c585d4d38 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c585d4d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1ae00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c585d4d90 blob([0xca3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1ae00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4d90) get_ref 0x0~2000 Blob(0x558c585d4d90 blob([0xca3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1ae00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c585d4d90) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c585d4d90 blob([0xca3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1ae00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1ae18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1ae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1aee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1ae70 blob([0xca5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1aee0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1ae70) get_ref 0x0~2000 Blob(0x558c57e1ae70 blob([0xca5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1aee0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1ae70) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c57e1ae70 blob([0xca5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1aee0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1aef8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1af50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1afc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1af50 blob([0xca7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1afc0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1af50) get_ref 0x0~2000 Blob(0x558c57e1af50 blob([0xca7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1afc0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1af50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c57e1af50 blob([0xca7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1afc0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1afd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b030 blob([0xca9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b0a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b030) get_ref 0x0~2000 Blob(0x558c57e1b030 blob([0xca9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b0a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b030) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c57e1b030 blob([0xca9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b0a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b0b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b110 blob([0xcab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b180 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b110) get_ref 0x0~2000 Blob(0x558c57e1b110 blob([0xcab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b180 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b110) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c57e1b110 blob([0xcab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b180 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b198 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b1f0 blob([0xcad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b260 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b1f0) get_ref 0x0~2000 Blob(0x558c57e1b1f0 blob([0xcad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b260 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c57e1b1f0 blob([0xcad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b260 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b278 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b2d0 blob([0xcaf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b340 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b2d0) get_ref 0x0~2000 Blob(0x558c57e1b2d0 blob([0xcaf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b340 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c57e1b2d0 blob([0xcaf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b340 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b358 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b3b0 blob([0xcb1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b420 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b3b0) get_ref 0x0~2000 Blob(0x558c57e1b3b0 blob([0xcb1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b420 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c57e1b3b0 blob([0xcb1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b420 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b438 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b490 blob([0xcb3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b500 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b490) get_ref 0x0~2000 Blob(0x558c57e1b490 blob([0xcb3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b500 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b490) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c57e1b490 blob([0xcb3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b500 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b518 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b570 blob([0xcb5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b5e0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b570) get_ref 0x0~2000 Blob(0x558c57e1b570 blob([0xcb5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b5e0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b570) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c57e1b570 blob([0xcb5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b5e0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b5f8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b650 blob([0xcb7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b6c0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b650) get_ref 0x0~2000 Blob(0x558c57e1b650 blob([0xcb7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b6c0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b650) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c57e1b650 blob([0xcb7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b6c0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b6d8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b730 blob([0xcb9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b7a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b730) get_ref 0x0~2000 Blob(0x558c57e1b730 blob([0xcb9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b7a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b730) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57e1b730 blob([0xcb9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b7a0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b7b8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b810 blob([0xcbb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b880 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b810) get_ref 0x0~2000 Blob(0x558c57e1b810 blob([0xcbb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b880 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b810) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57e1b810 blob([0xcbb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b880 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b898 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b8f0 blob([0xcbd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b960 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b8f0) get_ref 0x0~2000 Blob(0x558c57e1b8f0 blob([0xcbd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1b960 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c57e1b8f0 blob([0xcbd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1b960 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1b978 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1ba40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1b9d0 blob([0xcbf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1ba40 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b9d0) get_ref 0x0~2000 Blob(0x558c57e1b9d0 blob([0xcbf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1ba40 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1b9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c57e1b9d0 blob([0xcbf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1ba40 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1ba58 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1bab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1bab0 blob([0xcc1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bb20 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bab0) get_ref 0x0~2000 Blob(0x558c57e1bab0 blob([0xcc1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bb20 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bab0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c57e1bab0 blob([0xcc1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1bb20 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1bb38 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1bb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1bb90 blob([0xcc3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bc00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bb90) get_ref 0x0~2000 Blob(0x558c57e1bb90 blob([0xcc3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bc00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bb90) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c57e1bb90 blob([0xcc3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1bc00 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1bc18 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1bc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1bc70 blob([0xcc5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bce0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bc70) get_ref 0x0~2000 Blob(0x558c57e1bc70 blob([0xcc5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bce0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bc70) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c57e1bc70 blob([0xcc5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1bce0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1bcf8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1bd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1bd50 blob([0xcc7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bdc0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bd50) get_ref 0x0~2000 Blob(0x558c57e1bd50 blob([0xcc7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bdc0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bd50) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c57e1bd50 blob([0xcc7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1bdc0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1bdd8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1be30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1be30 blob([0xcc9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bea0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1be30) get_ref 0x0~2000 Blob(0x558c57e1be30 blob([0xcc9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bea0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1be30) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c57e1be30 blob([0xcc9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1bea0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1beb8 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57e1bf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bf80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57e1bf10 blob([0xccb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bf80 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bf10) get_ref 0x0~2000 Blob(0x558c57e1bf10 blob([0xccb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57e1bf80 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57e1bf10) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c57e1bf10 blob([0xccb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57e1bf80 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57e1bf98 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f18000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f18070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f18000 blob([0xccd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18070 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f18000) get_ref 0x0~2000 Blob(0x558c57f18000 blob([0xccd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18070 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f18000) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c57f18000 blob([0xccd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f18070 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57f18088 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f180e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f18150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f180e0 blob([0xccf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18150 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f180e0) get_ref 0x0~2000 Blob(0x558c57f180e0 blob([0xccf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18150 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f180e0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c57f180e0 blob([0xccf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f18150 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57f18168 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f181c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f18230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f181c0 blob([0xcd1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18230 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f181c0) get_ref 0x0~2000 Blob(0x558c57f181c0 blob([0xcd1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18230 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f181c0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c57f181c0 blob([0xcd1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f18230 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57f18248 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f182a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f18310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f182a0 blob([0xcd3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18310 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f182a0) get_ref 0x0~2000 Blob(0x558c57f182a0 blob([0xcd3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f18310 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f182a0) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c57f182a0 blob([0xcd3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f18310 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57f18328 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f18380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f183f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f18380 blob([0xcd5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f183f0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f18380) get_ref 0x0~2000 Blob(0x558c57f18380 blob([0xcd5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f183f0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f18380) get_ref init 0x2000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c57f18380 blob([0xcd5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f183f0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57f18408 in 0x558c53a59880) _discard 0x0~2000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f18460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f184d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f18460 blob([0xcd7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f184d0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f18460) get_ref 0x0~1000 Blob(0x558c57f18460 blob([0xcd7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f184d0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.blob(0x558c57f18460) get_ref init 0x1000, 1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~1000: 0x0~1000 Blob(0x558c57f18460 blob([0xcd7000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c57f184d0 sbid 0x0)) 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.BufferSpace(0x558c57f184e8 in 0x558c53a59880) _discard 0x0~1000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x4d000 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:40.665+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795ad88) dirty_range mark inline shard dirty 2019-12-07T19:34:40.665+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# 0x0~4d000 = 0 2019-12-07T19:34:40.665+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:40.666+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.4_0000000000000001.733E4DBD.28.smithi13140796-10..' <- OBJ_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:40.666+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000021_.4_0000000000000001.733E4DBD.28.smithi13140796-10..' <- SNA_1_0000000000000021_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000025_.4_0000000000000001.733E4DBD.28.smithi13140796-10..' <- SNA_1_0000000000000025_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000026_.4_0000000000000001.733E4DBD.28.smithi13140796-10..' <- SNA_1_0000000000000026_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000028_.4_0000000000000001.733E4DBD.28.smithi13140796-10..' <- SNA_1_0000000000000028_.4_0000000000000001.733E4DBD.28.smithi13140796-10.. 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:40.666+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000063.00000000000000000048' <- 0000000063.00000000000000000048 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.0000000063.00000000000000000049' <- 0000000063.00000000000000000049 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584d0f00 cost 534272 (40 ios * 4000 + 374272 bytes) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584d0f00 onodes 0x558c575a2a00,0x558c5795aa00,0x558c5795ac80 shared_blobs 0x558c580ba000,0x558c580ba0e0,0x558c580ba1c0,0x558c580ba2a0,0x558c580ba380,0x558c580ba460,0x558c580ba540,0x558c580ba620,0x558c580ba700,0x558c580ba7e0,0x558c580ba8c0,0x558c580ba9a0,0x558c580baa80,0x558c580bab60,0x558c580bac40,0x558c580bad20,0x558c580bae00,0x558c580baee0,0x558c583eb7a0,0x558c583eb880,0x558c58b83490,0x558c58b83570,0x558c58b83c70,0x558c58b83d50,0x558c58b83e30,0x558c58b83f10 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) update 4#1:ecc72bdb:::smithi13140796-10:head#31 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) update inline shard 702 bytes from 26 extents 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head#31 is 1249 (541 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795ab08) update 4#1:ecc72bdb:::smithi13140796-10:28# 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795ab08) update inline shard 702 bytes from 26 extents 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:28# is 1120 (412 bytes onode + 2 bytes spanning blobs + 706 bytes inline extents) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795ad88) update 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore.extentmap(0x558c5795ad88) update inline shard 741 bytes from 39 extents 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 4#1:ecc72bdb:::smithi13140796-10:head# is 55273 (54526 bytes onode + 2 bytes spanning blobs + 745 bytes inline extents) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b07 is 11 SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b08 is 11 SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b09 is 11 SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0a is 11 SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0b is 11 SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0c is 11 SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0d is 11 SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0e is 11 SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0f is 11 SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b10 is 11 SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b11 is 11 SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b12 is 11 SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b13 is 11 SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b14 is 11 SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b15 is 11 SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b16 is 11 SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b17 is 11 SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b18 is 11 SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aff is 11 SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b00 is 11 SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b02 is 11 SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b01 is 11 SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b03 is 11 SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b04 is 11 SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b05 is 11 SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b06 is 11 SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=2))) 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584d0f00 allocated 0x[c8b000~4d000] released 0x[] 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 freelist allocate 0xc8b000~4d000 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 freelist _xor first_key 0xc80000 last_key 0xc80000 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0f00 prepare 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c584d0f00 2019-12-07T19:34:40.666+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 lua=63'47 crt=63'47 lcod 63'47 active mbc={} ps=[20~1,23~1]] op_applied version 63'49 2019-12-07T19:34:40.666+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0f00 aio_wait 2019-12-07T19:34:40.666+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584d0f00 2019-12-07T19:34:40.666+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3f1e0 finish 2019-12-07T19:34:40.666+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.666+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0f00 io_done 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 18858 < min_flush_size 524288 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 51626 < min_flush_size 524288 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61304 < min_flush_size 524288 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795aa00 had 1 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5795ac80 had 1 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2280 had 1 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61327 < min_flush_size 524288 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 61327 < min_flush_size 524288 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x54b656~ef8f to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x54b656 0x54b656~ef8f to file(ino 16 size 0x550000 mtime 2019-12-07T19:34:39.638906+0000 allocated 550000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~520000]) 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _allocate len 0xa5e5 from 0 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _allocate len 0xa5e5 from 1 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd680000 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd680000~10000/10000,10000,acd680000 2019-12-07T19:34:40.666+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x560000 to include allocated 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 80 (was clean) 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~530000 x_off 0x51b656 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x656 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x5e5 and padding block with 0xa1b 2019-12-07T19:34:40.666+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x560000 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x55a5e5 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (80) on file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]), flushing log 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 80 len 0x4a crc 0x5e41180b) 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x4f000~1000 to file(ino 1 size 0x4f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x4f000 0x4f000~1000 to file(ino 1 size 0x4f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x50000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4f000 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x50000 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:40.667+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000021 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 80 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000431922s (0.000045803s flush + 0.000386119s kv commit) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.667+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584d0f00> 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.667+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0f00 kv_submitted 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584d0f00 2019-12-07T19:34:40.667+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0f00 finishing 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584d0f00 onodes 0x558c575a2a00,0x558c5795aa00,0x558c5795ac80 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1ae18 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf9e0 space 0x558c57e1ae18 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1aef8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfa70 space 0x558c57e1aef8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1afd8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfb00 space 0x558c57e1afd8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b0b8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfb90 space 0x558c57e1b0b8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b198 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfc20 space 0x558c57e1b198 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b278 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfcb0 space 0x558c57e1b278 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 63/41 ECSubWriteReply(tid=318, last_complete=63'49, committed=1, applied=1)) v2 -- 0x558c576b5c00 con 0x558c54c64000 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b358 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfd40 space 0x558c57e1b358 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b438 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfdd0 space 0x558c57e1b438 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b518 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfe60 space 0x558c57e1b518 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b5f8 in 0x558c53a59880) _finish_write discard buffer(0x558c551a8bd0 space 0x558c57e1b5f8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b6d8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cfef0 space 0x558c57e1b6d8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b7b8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c000 space 0x558c57e1b7b8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b898 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c090 space 0x558c57e1b898 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1b978 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c120 space 0x558c57e1b978 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1ba58 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c1b0 space 0x558c57e1ba58 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1bb38 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c240 space 0x558c57e1bb38 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1bc18 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c2d0 space 0x558c57e1bc18 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1bcf8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c360 space 0x558c57e1bcf8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1bdd8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c3f0 space 0x558c57e1bdd8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1beb8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c480 space 0x558c57e1beb8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57e1bf98 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c510 space 0x558c57e1bf98 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f18088 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c5a0 space 0x558c57f18088 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f18168 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c630 space 0x558c57f18168 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f18248 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c6c0 space 0x558c57f18248 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f18328 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c750 space 0x558c57f18328 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f18408 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c7e0 space 0x558c57f18408 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f184e8 in 0x558c53a59880) _finish_write discard buffer(0x558c57f1c870 space 0x558c57f184e8 0x0~1000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5829c948 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf560 space 0x558c5829c948 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4478 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce750 space 0x558c585d4478 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4558 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce6c0 space 0x558c585d4558 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4638 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce630 space 0x558c585d4638 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4718 in 0x558c53a59880) _finish_write discard buffer(0x558c584ce5a0 space 0x558c585d4718 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d47f8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf5f0 space 0x558c585d47f8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d48d8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf680 space 0x558c585d48d8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d49b8 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf710 space 0x558c585d49b8 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4a98 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf7a0 space 0x558c585d4a98 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4b78 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf830 space 0x558c585d4b78 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4c58 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf8c0 space 0x558c585d4c58 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c585d4d38 in 0x558c53a59880) _finish_write discard buffer(0x558c584cf950 space 0x558c585d4d38 0x0~2000 writing nocache) 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584d0f00 done 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:40.667+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584d0f00 [] 2019-12-07T19:34:40.667+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.667+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.670+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 550 ==== MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=319, reqid=client.4540.0:480, at_version=0'0, trim_to=0'0, roll_forward_to=63'49)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c5831b000 con 0x558c54c64000 2019-12-07T19:34:40.670+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c57f3e420 prio 127 cost 0 latency 0.000020 epoch 63 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=319, reqid=client.4540.0:480, at_version=0'0, trim_to=0'0, roll_forward_to=63'49)) v2 2019-12-07T19:34:40.670+0000 7f8f24457700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=319, reqid=client.4540.0:480, at_version=0'0, trim_to=0'0, roll_forward_to=63'49)) v2) prio 127 cost 0 e63) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=319, reqid=client.4540.0:480, at_version=0'0, trim_to=0'0, roll_forward_to=63'49)) v2) prio 127 cost 0 e63) queued 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGOpItem(op=MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=319, reqid=client.4540.0:480, at_version=0'0, trim_to=0'0, roll_forward_to=63'49)) v2) prio 127 cost 0 e63) pg 0x558c54d5f000 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3e420 prio 127 cost 0 latency 0.000346 MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=319, reqid=client.4540.0:480, at_version=0'0, trim_to=0'0, roll_forward_to=63'49)) v2 pg pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.7s4 63/41 ECSubWrite(tid=319, reqid=client.4540.0:480, at_version=0'0, trim_to=0'0, roll_forward_to=63'49)) v2 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'47 active mbc={} ps=[20~1,23~1]] append_log log((0'0,63'49], crt=63'47) [] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'48 (57'39) clone 1:ecc72bdb:::smithi13140796-10:28 by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [28,26,25,21] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] rollforward: entry=63'49 (57'39) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:480 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 49 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:40.670+0000 7f8f0117b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54d39680 1.7s4_head 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54cf7b80 = 0x558c584d0c00 seq 100 2019-12-07T19:34:40.670+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head#31 onode 0x558c575a2a00 txc 0x558c584d0c00 2019-12-07T19:34:40.670+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _do_truncate 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head#31 0x0 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c583eb6c0) put_ref 0x0~2000 Blob(0x558c583eb6c0 blob([0xab0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c583eb810) put_ref 0x0~2000 Blob(0x558c583eb810 blob([0xab2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c583eb8f0) put_ref 0x0~2000 Blob(0x558c583eb8f0 blob([0xab4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c58b83500) put_ref 0x0~2000 Blob(0x558c58b83500 blob([0xab6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c58b83c00) put_ref 0x0~2000 Blob(0x558c58b83c00 blob([0xab8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c58b83ce0) put_ref 0x0~2000 Blob(0x558c58b83ce0 blob([0xaba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c58b83dc0) put_ref 0x0~2000 Blob(0x558c58b83dc0 blob([0xabc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c58b83ea0) put_ref 0x0~2000 Blob(0x558c58b83ea0 blob([0xabe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c58b83f80) put_ref 0x0~2000 Blob(0x558c58b83f80 blob([0xac0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba070) put_ref 0x0~2000 Blob(0x558c580ba070 blob([0xac2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba150) put_ref 0x0~2000 Blob(0x558c580ba150 blob([0xac4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba230) put_ref 0x0~2000 Blob(0x558c580ba230 blob([0xac6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba310) put_ref 0x0~2000 Blob(0x558c580ba310 blob([0xac8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba3f0) put_ref 0x0~2000 Blob(0x558c580ba3f0 blob([0xaca000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba4d0) put_ref 0x0~2000 Blob(0x558c580ba4d0 blob([0xacc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba5b0) put_ref 0x0~2000 Blob(0x558c580ba5b0 blob([0xace000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba690) put_ref 0x0~2000 Blob(0x558c580ba690 blob([0xad0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba770) put_ref 0x0~2000 Blob(0x558c580ba770 blob([0xad2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba850) put_ref 0x0~2000 Blob(0x558c580ba850 blob([0xad4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580ba930) put_ref 0x0~2000 Blob(0x558c580ba930 blob([0xad6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580baa10) put_ref 0x0~2000 Blob(0x558c580baa10 blob([0xad8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580baaf0) put_ref 0x0~2000 Blob(0x558c580baaf0 blob([0xada000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580babd0) put_ref 0x0~2000 Blob(0x558c580babd0 blob([0xadc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580bacb0) put_ref 0x0~2000 Blob(0x558c580bacb0 blob([0xade000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580bad90) put_ref 0x0~2000 Blob(0x558c580bad90 blob([0xae0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.blob(0x558c580bae70) put_ref 0x0~1000 Blob(0x558c580bae70 blob([0xae2000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore.extentmap(0x558c575a2b08) dirty_range mark inline shard dirty 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558c583eb6c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xab0000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558c583eb810 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xab2000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558c583eb8f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xab4000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558c58b83500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xab6000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558c58b83c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xab8000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558c58b83ce0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaba000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558c58b83dc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xabc000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558c58b83ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xabe000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558c58b83f80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xac0000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558c580ba070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xac2000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558c580ba150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xac4000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558c580ba230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xac6000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558c580ba310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xac8000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558c580ba3f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xaca000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558c580ba4d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xacc000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558c580ba5b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xace000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558c580ba690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xad0000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558c580ba770 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xad2000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558c580ba850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xad4000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558c580ba930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xad6000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558c580baa10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xad8000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558c580baaf0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xada000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558c580babd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xadc000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558c580bacb0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xade000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558c580bad90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xae0000~2000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x32000~1000: 0x0~1000 Blob(0x558c580bae70 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=2)))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish blob release [0xae2000~1000] 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish shared_blob release [] from SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove gen and maybe_unshared_blobs 0x558c580ba000,0x558c580ba0e0,0x558c580ba1c0,0x558c580ba2a0,0x558c580ba380,0x558c580ba460,0x558c580ba540,0x558c580ba620,0x558c580ba700,0x558c580ba7e0,0x558c580ba8c0,0x558c580ba9a0,0x558c580baa80,0x558c580bab60,0x558c580bac40,0x558c580bad20,0x558c580bae00,0x558c580baee0,0x558c583eb7a0,0x558c583eb880,0x558c58b83490,0x558c58b83570,0x558c58b83c70,0x558c58b83d50,0x558c58b83e30,0x558c58b83f10 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_remove checking for unshareable blobs on 0x558c5795ac80 4#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _remove 1.7s4_head 4#1:ecc72bdb:::smithi13140796-10:head#31 = 0 2019-12-07T19:34:40.670+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.7s4_head 4#1:e0000000::::head# = 0 2019-12-07T19:34:40.670+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c584d0c00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c584d0c00 onodes shared_blobs 0x558c580ba000,0x558c580ba0e0,0x558c580ba1c0,0x558c580ba2a0,0x558c580ba380,0x558c580ba460,0x558c580ba540,0x558c580ba620,0x558c580ba700,0x558c580ba7e0,0x558c580ba8c0,0x558c580ba9a0,0x558c580baa80,0x558c580bab60,0x558c580bac40,0x558c580bad20,0x558c580bae00,0x558c580baee0,0x558c583eb7a0,0x558c583eb880,0x558c58b83490,0x558c58b83570,0x558c58b83c70,0x558c58b83d50,0x558c58b83e30,0x558c58b83f10 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b07 is 11 SharedBlob(0x558c580ba000 loaded (sbid 0x2b07 ref_map(0xac0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b08 is 11 SharedBlob(0x558c580ba0e0 loaded (sbid 0x2b08 ref_map(0xac2000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b09 is 11 SharedBlob(0x558c580ba1c0 loaded (sbid 0x2b09 ref_map(0xac4000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0a is 11 SharedBlob(0x558c580ba2a0 loaded (sbid 0x2b0a ref_map(0xac6000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0b is 11 SharedBlob(0x558c580ba380 loaded (sbid 0x2b0b ref_map(0xac8000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0c is 11 SharedBlob(0x558c580ba460 loaded (sbid 0x2b0c ref_map(0xaca000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0d is 11 SharedBlob(0x558c580ba540 loaded (sbid 0x2b0d ref_map(0xacc000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0e is 11 SharedBlob(0x558c580ba620 loaded (sbid 0x2b0e ref_map(0xace000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b0f is 11 SharedBlob(0x558c580ba700 loaded (sbid 0x2b0f ref_map(0xad0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b10 is 11 SharedBlob(0x558c580ba7e0 loaded (sbid 0x2b10 ref_map(0xad2000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b11 is 11 SharedBlob(0x558c580ba8c0 loaded (sbid 0x2b11 ref_map(0xad4000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b12 is 11 SharedBlob(0x558c580ba9a0 loaded (sbid 0x2b12 ref_map(0xad6000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b13 is 11 SharedBlob(0x558c580baa80 loaded (sbid 0x2b13 ref_map(0xad8000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b14 is 11 SharedBlob(0x558c580bab60 loaded (sbid 0x2b14 ref_map(0xada000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b15 is 11 SharedBlob(0x558c580bac40 loaded (sbid 0x2b15 ref_map(0xadc000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b16 is 11 SharedBlob(0x558c580bad20 loaded (sbid 0x2b16 ref_map(0xade000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b17 is 11 SharedBlob(0x558c580bae00 loaded (sbid 0x2b17 ref_map(0xae0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b18 is 11 SharedBlob(0x558c580baee0 loaded (sbid 0x2b18 ref_map(0xae2000~1000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2aff is 11 SharedBlob(0x558c583eb7a0 loaded (sbid 0x2aff ref_map(0xab0000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b00 is 11 SharedBlob(0x558c583eb880 loaded (sbid 0x2b00 ref_map(0xab2000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b02 is 11 SharedBlob(0x558c58b83490 loaded (sbid 0x2b02 ref_map(0xab6000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b01 is 11 SharedBlob(0x558c58b83570 loaded (sbid 0x2b01 ref_map(0xab4000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b03 is 11 SharedBlob(0x558c58b83c70 loaded (sbid 0x2b03 ref_map(0xab8000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b04 is 11 SharedBlob(0x558c58b83d50 loaded (sbid 0x2b04 ref_map(0xaba000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b05 is 11 SharedBlob(0x558c58b83e30 loaded (sbid 0x2b05 ref_map(0xabc000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes shared_blob 0x2b06 is 11 SharedBlob(0x558c58b83f10 loaded (sbid 0x2b06 ref_map(0xabe000~2000=1))) 2019-12-07T19:34:40.670+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c584d0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:40.671+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0c00 prepare 2019-12-07T19:34:40.671+0000 7f8f0117b700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c584d0c00 2019-12-07T19:34:40.671+0000 7f8f0117b700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0c00 io_done 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:40.671+0000 7f8f0117b700 10 osd.7 63 dequeue_op 0x558c57f3e420 finish 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:40.671+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1845 < min_flush_size 524288 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55148c80 had 1 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575a2a00 had 1 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1868 < min_flush_size 524288 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1868 < min_flush_size 524288 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x55a5e5~74c to file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x55a5e5 0x55a5e5~74c to file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~530000 x_off 0x52a5e5 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x5e5 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd31 and padding block with 0x2cf 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x55ad31 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:40.671+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000089158s (0.000000081s flush + 0.000089077s kv commit) 2019-12-07T19:34:40.671+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c584d0c00> 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:40.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0c00 kv_submitted 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c584d0c00 2019-12-07T19:34:40.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c584d0c00 finishing 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c584d0c00 onodes 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c584d0c00 done 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54cf7b80 q now empty 2019-12-07T19:34:40.671+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c584d0c00 [] 2019-12-07T19:34:40.671+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:40.671+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.671+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:40.671+0000 7f8f05183700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 63/41 ECSubWriteReply(tid=319, last_complete=63'49, committed=1, applied=1)) v2 -- 0x558c58115800 con 0x558c54c64000 2019-12-07T19:34:40.671+0000 7f8f05183700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:40.704+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.755+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.805+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 10 osd.7 63 tick_without_osd_lock 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f328000/0x40000000/0x1680000000, data 0xfb574f/0xcd6000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 25 osd.7 63 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476453+0000 last_rx_front 2019-12-07T19:34:38.476418+0000 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 25 osd.7 63 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476352+0000 last_rx_front 2019-12-07T19:34:38.476394+0000 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 25 osd.7 63 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476378+0000 last_rx_front 2019-12-07T19:34:38.476470+0000 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 25 osd.7 63 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476367+0000 last_rx_front 2019-12-07T19:34:38.476505+0000 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 25 osd.7 63 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476409+0000 last_rx_front 2019-12-07T19:34:38.476432+0000 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 20 osd.7 63 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 10 osd.7 63 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 20 osd.7 63 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:40.827+0000 7f8f1fbbe700 10 osd.7 63 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:40.855+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.905+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:40.955+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.004+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.046+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 16 ==== osd_ping(ping e63 up_from 13 ping_stamp 2019-12-07T19:34:41.046885+0000/64.513208568s send_stamp 64.513208568s delta_ub 0.031079930s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57950a00 con 0x558c54c5ac00 2019-12-07T19:34:41.046+0000 7f8f23c56700 20 osd.7 63 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030717650s,0.031079930s]) 2019-12-07T19:34:41.046+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e63 up_from 13 ping_stamp 2019-12-07T19:34:41.046885+0000/64.513208568s send_stamp 64.482490918s delta_ub 18446744073.678833966s) v5 -- 0x558c57950c00 con 0x558c54c5ac00 2019-12-07T19:34:41.046+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 16 ==== osd_ping(ping e63 up_from 13 ping_stamp 2019-12-07T19:34:41.046885+0000/64.513208568s send_stamp 64.513208568s delta_ub 0.031079930s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c58115400 con 0x558c54c5a400 2019-12-07T19:34:41.046+0000 7f8f23c56700 10 osd.7 63 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 62 -> 63 (as per caller) 2019-12-07T19:34:41.046+0000 7f8f24c58700 20 osd.7 63 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030701232s,0.031079930s]) 2019-12-07T19:34:41.046+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e63 up_from 13 ping_stamp 2019-12-07T19:34:41.046885+0000/64.513208568s send_stamp 64.482507336s delta_ub 18446744073.678850384s) v5 -- 0x558c5795d800 con 0x558c54c5a400 2019-12-07T19:34:41.054+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.080+0000 7f8f24c58700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c58158000 0x558c574b9700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:41.080+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> conn(0x558c58158000 0x558c574b9700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:41.080+0000 7f8f23c56700 10 osd.7 63 ms_handle_authentication new session 0x558c54c0b380 con 0x558c58158000 entity client.admin addr 2019-12-07T19:34:41.080+0000 7f8f23c56700 10 osd.7 63 ms_handle_authentication session 0x558c54c0b380 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:41.080+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1547547467 conn(0x558c58158000 0x558c574b9700 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=client.4568 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:41.080+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4568 172.21.15.131:0/1547547467 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558c5852a960 con 0x558c58158000 2019-12-07T19:34:41.080+0000 7f8f164bb700 20 osd.7 63 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:41.080+0000 7f8f164bb700 10 osd.7 63 do_waiters -- start 2019-12-07T19:34:41.080+0000 7f8f164bb700 10 osd.7 63 do_waiters -- finish 2019-12-07T19:34:41.080+0000 7f8f164bb700 20 osd.7 63 _dispatch 0x558c5852a960 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:41.080+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1547547467 -- command_reply(tid 13: 0 ) v1 -- 0x558c54ce1500 con 0x558c58158000 2019-12-07T19:34:41.103+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4568 172.21.15.131:0/1547547467 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558c576dcb40 con 0x558c58158000 2019-12-07T19:34:41.103+0000 7f8f164bb700 20 osd.7 63 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:41.103+0000 7f8f164bb700 10 osd.7 63 do_waiters -- start 2019-12-07T19:34:41.103+0000 7f8f164bb700 10 osd.7 63 do_waiters -- finish 2019-12-07T19:34:41.103+0000 7f8f164bb700 20 osd.7 63 _dispatch 0x558c576dcb40 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:41.104+0000 7f8f22c54700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/1547547467 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558c54ce1340 con 0x558c58158000 2019-12-07T19:34:41.104+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.107+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1547547467 conn(0x558c58158000 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 80 2019-12-07T19:34:41.107+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1547547467 conn(0x558c58158000 msgr2=0x558c574b9700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:41.107+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1547547467 conn(0x558c58158000 0x558c574b9700 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:41.107+0000 7f8f23c56700 1 --2- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] >> 172.21.15.131:0/1547547467 conn(0x558c58158000 0x558c574b9700 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:41.107+0000 7f8f164bb700 2 osd.7 63 ms_handle_reset con 0x558c58158000 session 0x558c54c0b380 2019-12-07T19:34:41.155+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.206+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.226+0000 7f8f21451700 10 osd.7 63 tick 2019-12-07T19:34:41.226+0000 7f8f21451700 10 osd.7 63 do_waiters -- start 2019-12-07T19:34:41.226+0000 7f8f21451700 10 osd.7 63 do_waiters -- finish 2019-12-07T19:34:41.226+0000 7f8f21451700 20 osd.7 63 tick last_purged_snaps_scrub 2019-12-07T19:34:38.205126+0000 next 2019-12-07T19:35:38.205126+0000 2019-12-07T19:34:41.255+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.287+0000 7f8f21c52700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 36933) UID: 0 2019-12-07T19:34:41.305+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.355+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.406+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.455+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.505+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.555+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.605+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1654716 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.644+0000 7f8f24457700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== client.4540 172.21.15.131:0/224485236 29 ==== osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8 ==== 1034+0+70370 (crc 0 0 0) 0x558c552318c0 con 0x558c54c9f000 2019-12-07T19:34:41.644+0000 7f8f24457700 15 osd.7 63 enqueue_op 0x558c57f3ef20 prio 63 cost 70370 latency 0.000027 epoch 64 osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8 2019-12-07T19:34:41.644+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8) prio 63 cost 70370 e64) 2019-12-07T19:34:41.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8) prio 63 cost 70370 e64) queued 2019-12-07T19:34:41.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8) prio 63 cost 70370 e64) pg 0x558c57e88000 2019-12-07T19:34:41.644+0000 7f8f07988700 10 osd.7 63 dequeue_op 0x558c57f3ef20 prio 63 cost 70370 latency 0.000063 osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.644+0000 7f8f07988700 10 osd.7 63 maybe_share_map con 0x558c54c9f000 172.21.15.131:0/224485236 map epoch 60 -> 64 (as per caller) 2019-12-07T19:34:41.644+0000 7f8f07988700 20 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_request min 64, queue on waiting_for_map client.4540 2019-12-07T19:34:41.644+0000 7f8f07988700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=64}) v3 -- 0x558c5841e600 con 0x558c5476d000 2019-12-07T19:34:41.644+0000 7f8f07988700 10 osd.7 63 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:41.644+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.644+0000 7f8f164bb700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] <== mon.0 v2:172.21.15.109:3300/0 35 ==== osd_map(64..64 src has 1..64) v4 ==== 633+0+0 (crc 0 0 0) 0x558c582346c0 con 0x558c5476d000 2019-12-07T19:34:41.644+0000 7f8f164bb700 20 osd.7 63 OSD::ms_dispatch: osd_map(64..64 src has 1..64) v4 2019-12-07T19:34:41.644+0000 7f8f164bb700 10 osd.7 63 do_waiters -- start 2019-12-07T19:34:41.644+0000 7f8f164bb700 10 osd.7 63 do_waiters -- finish 2019-12-07T19:34:41.644+0000 7f8f164bb700 20 osd.7 63 _dispatch 0x558c582346c0 osd_map(64..64 src has 1..64) v4 2019-12-07T19:34:41.644+0000 7f8f164bb700 3 osd.7 63 handle_osd_map epochs [64,64], i have 63, src has [1,64] 2019-12-07T19:34:41.644+0000 7f8f164bb700 10 osd.7 63 handle_osd_map got inc map for epoch 64 2019-12-07T19:34:41.644+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3f84e826:::osdmap.63:0# 0x0~0 2019-12-07T19:34:41.644+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~1438 size 0x1438 (5176) 2019-12-07T19:34:41.644+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:41.644+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c580f7e30 blob([0xc2d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c580f7ea0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:41.644+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c579dc310 blob([!~1000,0xc2e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c57ff5c70 sbid 0x0)) need 0x1000~438 cache has 0x[1000~438] 2019-12-07T19:34:41.644+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:3f84e826:::osdmap.63:0# 0x0~1438 = 5176 2019-12-07T19:34:41.644+0000 7f8f164bb700 10 osd.7 63 add_map_bl 63 5176 bytes 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 osd.7 63 got_full_map 64, nothing requested 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 osd.7 63 handle_osd_map pg_num_history pg_num_history(e64 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps purged_snaps {64={}} 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 osd.7 63 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.7 0f8b5d06-9920-4212-962e-b5b150d77eaa e64 [1,64] lci=[13,64]) 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c53a70960 meta 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54ea4b40 = 0x558c583b7800 seq 56 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:cd9f972f:::inc_osdmap.64:0# key 0x7f7fffffffffffffffcd9f97'/!inc_osdmap.64!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cd9f972f:::inc_osdmap.64:0# 0x0~24d 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1297 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:cd9f972f:::inc_osdmap.64:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24d 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c583b7800 1 blobs 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xcd8000~1000/1000,1000,0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xcd8000~1000] 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54e56fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c585e0e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54e56fc0 blob([0xcd8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585e0e00 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c54e56fc0) get_ref 0x0~24d Blob(0x558c54e56fc0 blob([0xcd8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c585e0e00 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c54e56fc0) get_ref init 0x1000, 1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558c54e56fc0 blob([0xcd8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c585e0e00 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c585e0e18 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x24d 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57977c88) dirty_range mark inline shard dirty 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:cd9f972f:::inc_osdmap.64:0# 0x0~24d = 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) get_onode oid #-1:3164e826:::osdmap.64:0# key 0x7f7fffffffffffffff3164e8'&!osdmap.64!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(meta 0x558c53a70960) r -2 v.len 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3164e826:::osdmap.64:0# 0x0~1438 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1298 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:3164e826:::osdmap.64:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x1000~438 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c583b7800 2 blobs 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xcd9000~2000/1000,2000,0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xcd9000~2000] 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5792cd20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58767030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5792cd20 blob([0xcd9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58767030 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c5792cd20) get_ref 0x0~1000 Blob(0x558c5792cd20 blob([0xcd9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58767030 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c5792cd20) get_ref init 0x1000, 1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558c5792cd20 blob([0xcd9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c58767030 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c58767048 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58226150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792c7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58226150 blob([!~1000,0xcda000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792c7e0 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c58226150) get_ref 0x1000~438 Blob(0x558c58226150 blob([!~1000,0xcda000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558c5792c7e0 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c58226150) get_ref init 0x2000, 1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558c58226150 blob([!~1000,0xcda000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558c5792c7e0 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c5792c7f8 in 0x558c53a59340) _discard 0x1000~1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x1438 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57977a08) dirty_range mark inline shard dirty 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:3164e826:::osdmap.64:0# 0x0~1438 = 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~26 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) bstart 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c55278558 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) get_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c552784d0) put_ref 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c583b7800 0 blobs 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558c552784d0 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558c55278540 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) dirty_range mark inline shard dirty 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small 0x0~24e 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small considering Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.BufferSpace(0x558c54ea25c8 in 0x558c53a59340) _discard 0x0~1000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) get_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.blob(0x558c54ea2700) put_ref 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c583b7800 0 blobs 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558c54ea2700 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558c54ea25b0 sbid 0x0)) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) dirty_range mark inline shard dirty 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c583b7800 cost 23246 (4 ios * 4000 + 7246 bytes) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c583b7800 onodes 0x558c53a91900,0x558c55276c80,0x558c57977900,0x558c57977b80 shared_blobs 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c53a91a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c55276d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57977a08) update #-1:3164e826:::osdmap.64:0# 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57977a08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:3164e826:::osdmap.64:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57977c88) update #-1:cd9f972f:::inc_osdmap.64:0# 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore.extentmap(0x558c57977c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode #-1:cd9f972f:::inc_osdmap.64:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c583b7800 allocated 0x[cd8000~3000] released 0x[] 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 freelist allocate 0xcd8000~3000 2019-12-07T19:34:41.645+0000 7f8f164bb700 20 freelist _xor first_key 0xc80000 last_key 0xc80000 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7800 prepare 2019-12-07T19:34:41.645+0000 7f8f164bb700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c583b7800 2019-12-07T19:34:41.645+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7800 aio_wait 2019-12-07T19:34:41.645+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c583b7800 2019-12-07T19:34:41.645+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7800 io_done 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c53a91900 had 1 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c55276c80 had 1 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57977900 had 1 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57977b80 had 1 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c548f2500 had 1 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs get_usage bdev 1 free 3855220736 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluestore(/var/lib/ceph/osd/ceph-7) _get_bluefs_size_delta bluefs 3.6 GiB free (0.997338) bluestore 86 GiB free (0.95986), bluefs_ratio 0.0399038 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x55ad31~226c to file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x55ad31 0x55ad31~226c to file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~530000 x_off 0x52ad31 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd31 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf9d and padding block with 0x63 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x55cf9d 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.645+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000182521s (0.000044039s flush + 0.000138482s kv commit) 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.645+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c583b7800> 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.645+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c583b7800 kv_submitted 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c583b7800 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _deferred_queue txc 0x558c583b7800 osr 0x558c54ea4b40 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 53 0x1b000~1000 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7 63 _committed_osd_maps 64..64 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 54 0x1b000~1000 crc 73c00bd8 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7 63 advance to epoch 64 (<= last 64 <= newest_map 64) 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) _discard drop 53 0x2000~1000 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore.DeferredBatch(0x558c575c2280) prepare_write seq 54 0x2000~1000 crc df74cf14 2019-12-07T19:34:41.645+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.645+0000 7f8f141a1700 7 osd.7 64 consume_map version 64 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.8s5 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.0s3 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.1s1 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.9s4 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.as5 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.2s0 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.bs1 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.3s4 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.cs0 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.4s0 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.5s4 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.ds2 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.es4 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.6s2 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.fs5 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 identify_splits_and_merges 1.7s4 e63 to e64 pg_nums {18=16} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 64 prune_pg_created 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:0.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.8s5 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:0.consume_map consume_map 1.0s3 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:1.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.1s1 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:1.consume_map consume_map 1.9s4 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:2.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.as5 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:2.consume_map consume_map 1.2s0 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:3.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.bs1 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:3.consume_map consume_map 1.3s4 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:4.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.cs0 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:4.consume_map consume_map 1.4s0 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:5.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.5s4 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:5.consume_map consume_map 1.ds2 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:6.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.es4 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:6.consume_map consume_map 1.6s2 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7:7.consume_map 64 (was 63) 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.fs5 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7:7.consume_map consume_map 1.7s4 2019-12-07T19:34:41.645+0000 7f8f141a1700 10 osd.7 64 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.8s5 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.0s3 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.1s1 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.9s4 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(1) _enqueue OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c57380000 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.as5 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.2s0 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.as5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c5736a000 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.as5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c57e8b000 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.bs1 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.8s5 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.3s4 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(3) _enqueue OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.cs0 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 63 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,5,2,3,7]/[1,4,5,2,3,7] -- 1/1 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c57e88000 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7:2.update_pg_epoch 1.as5 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 63 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,2,5,3,7]/[1,4,2,5,3,7] -- 1/1 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.1s1 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c57676000 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.645+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.8s5_head options 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_advance_map [3,7,2,4,5,1]/[3,7,2,4,5,1] -- 3/3 2019-12-07T19:34:41.645+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.as5_head options 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7:2.update_pg_epoch 1.2s0 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7:4.update_pg_epoch 1.cs0 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 63 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] handle_advance_map [7,4,1,5,2,3]/[7,4,1,5,2,3] -- 7/7 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 63 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] handle_advance_map [7,5,4,1,2,3]/[7,5,4,1,2,3] -- 7/7 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.645+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.1s1_head options 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.645+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: Active advmap 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: Active advmap 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.as5( v 57'9 (0'0,57'9] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,2,5,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'9 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f03980700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.645+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.2s0_head options 2019-12-07T19:34:41.645+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f06986700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.cs0_head options 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.0s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.0s3 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c5500d000 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.645+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7:0.update_pg_epoch 1.0s3 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,7,2,4,5,1]p3(0) r=1 lpr=41 luod=0'0 crt=29'9 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 63 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,5,2,7,3,4]/[1,5,2,7,3,4] -- 1/1 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process 1.9s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process OpSchedulerItem(1.9s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c5500a000 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:41.645+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.0s3_head options 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.645+0000 7f8f06986700 7 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:41.645+0000 7f8f07988700 7 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7:1.update_pg_epoch 1.9s4 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 63 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,5,2,7,3]/[1,4,5,2,7,3] -- 1/1 2019-12-07T19:34:41.645+0000 7f8f06986700 15 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:218 2019-12-07T19:34:41.645+0000 7f8f07988700 15 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:181 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.645+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.645+0000 7f8f04181700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.9s4_head options 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.645+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f06986700 10 osd.7 pg_epoch: 64 pg[1.cs0( v 62'56 (0'0,62'56] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [7,5,4,1,2,3]p7(0) r=0 lpr=28 crt=62'56 lcod 62'54 mlcod 62'54 active+clean ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] requeue_map_waiters client.4540 0x558c57f3ef20 2019-12-07T19:34:41.645+0000 7f8f06986700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.645+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.0s3( v 53'21 (0'0,53'21] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,5,2,7,3,4]p1(0) r=3 lpr=18 luod=0'0 crt=53'21 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f06986700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:41.645+0000 7f8f04982700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f07988700 20 osd.7 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8) prio 63 cost 70370 e64) 2019-12-07T19:34:41.645+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.4s0 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(4) _enqueue OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.645+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.5s4 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f07988700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8) prio 63 cost 70370 e64) queued 2019-12-07T19:34:41.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.ds2 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(5) _enqueue OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process 1.4s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f0297e700 20 osd.7 op_wq(4) _process OpSchedulerItem(1.4s0 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c548ab000 2019-12-07T19:34:41.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.es4 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.645+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8) prio 63 cost 70370 e64) pg 0x558c57e88000 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.645+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c575ef000 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.6s2 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(6) _enqueue OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.645+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c57f3ef20 prio 63 cost 70370 latency 0.001334 osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.fs5 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f04181700 10 osd.7 pg_epoch: 64 pg[1.9s4( v 62'50 (0'0,62'50] local-lis/les=41/42 n=5 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,7,3]p1(0) r=4 lpr=41 luod=0'0 crt=62'50 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f0297e700 20 osd.7:4.update_pg_epoch 1.4s0 63 -> 64 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f04181700 10 log is not dirty 2019-12-07T19:34:41.645+0000 7f8f04181700 20 osd.7 op_wq(1) _process empty q, waiting 2019-12-07T19:34:41.645+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] _handle_message: osd_op(client.4540.0:483 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e64) v8 2019-12-07T19:34:41.645+0000 7f8f141a1700 15 osd.7 64 enqueue_peering_evt 1.7s4 epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.645+0000 7f8f141a1700 20 osd.7 op_wq(7) _enqueue OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) 2019-12-07T19:34:41.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 63 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] handle_advance_map [7,3,1,5,2,4]/[7,3,1,5,2,4] -- 7/7 2019-12-07T19:34:41.645+0000 7f8f0317f700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.645+0000 7f8f141a1700 7 osd.7 64 activate_map version 64 2019-12-07T19:34:41.645+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.bs1 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f0918b700 20 osd.7 64 agent_entry empty queue 2019-12-07T19:34:41.645+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: Active advmap 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 63 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_advance_map [4,7,2,3,1,5]/[4,7,2,3,1,5] -- 4/4 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c54d98000 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f0297e700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.4s0_head options 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_op: op osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) v8 2019-12-07T19:34:41.646+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.bs1_head options 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.es4 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] op_has_sufficient_caps session=0x558c54c5cb60 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f0297e700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,3,4,5,7,2]/[1,3,4,5,7,2] -- 1/1 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: Active: handling ActMap 2019-12-07T19:34:41.646+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.es4_head options 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_op osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] Active: kicking snap trim 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.bs1( v 62'55 (0'0,62'55] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,7,2,3,1,5]p4(0) r=1 lpr=41 luod=0'0 crt=62'55 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0297e700 7 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] state: handle ActMap primary 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] get_object_context: obc NOT found in cache: 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.646+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# _ 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process 1.3s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process OpSchedulerItem(1.3s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c5513e000 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f0297e700 15 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] publish_stats_to_osd 62:169 2019-12-07T19:34:41.646+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f0297e700 20 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 osd.7 pg_epoch: 64 pg[1.4s0( v 53'10 (0'0,53'10] local-lis/les=41/42 n=1 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,3,1,5,2,4]p7(0) r=0 lpr=41 crt=53'10 lcod 45'8 mlcod 45'8 active+clean ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.646+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7:3.update_pg_epoch 1.3s4 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f0297e700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.es4( v 62'12 (0'0,62'12] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,3,4,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=62'12 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# _ = -2 2019-12-07T19:34:41.646+0000 7f8f0297e700 20 osd.7 op_wq(4) _process empty q, waiting 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 63 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] handle_advance_map [4,1,2,5,7,3]/[4,1,2,5,7,3] -- 4/4 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process 1.6s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] get_object_context: no obc for soid 1:4c9e444c:::smithi13140796-48:head but can_create 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process OpSchedulerItem(1.6s2 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c54d5c000 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c57522000 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f0317f700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.3s4_head options 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] create_object_context 0x558c57570b40 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] populate_obc_watchers 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.5s4 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7:6.update_pg_epoch 1.6s2 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 63 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] handle_advance_map [2,1,3,4,7,5]/[2,1,3,4,7,5] -- 2/2 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 63 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] handle_advance_map [3,2,7,1,4,5]/[3,2,7,1,4,5] -- 3/3 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] get_object_context: 0x558c57570b40 1:4c9e444c:::smithi13140796-48:head rwstate(none n=0 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x558c576b6900 snapset: 0=[]:{} 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.5s4_head options 2019-12-07T19:34:41.646+0000 7f8f0197c700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.6s2_head options 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 osd.7 pg_epoch: 64 pg[1.3s4( v 55'20 (0'0,55'20] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [4,1,2,5,7,3]p4(0) r=4 lpr=41 luod=0'0 crt=55'20 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f0317f700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f03980700 25 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_op oi 1:4c9e444c:::smithi13140796-48:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:41.646+0000 7f8f0317f700 20 osd.7 op_wq(3) _process empty q, waiting 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_op obc obc(1:4c9e444c:::smithi13140796-48:head(dne) rwstate(write n=1 w=0)) 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c57383000 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] execute_ctx 0x558c5829e700 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 63 while current is 64 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 osd.7 pg_epoch: 64 pg[1.6s2( v 63'81 (0'0,63'81] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,2,7,1,4,5]p3(0) r=2 lpr=28 luod=0'0 crt=63'81 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,3,4,7,5]p2(0) r=4 lpr=41 luod=0'0 crt=27'8 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f0197c700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0197c700 20 osd.7 op_wq(6) _process empty q, waiting 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process 1.ds2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] execute_ctx 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] ov 0'0 av 64'16 snapc 29=[29,28,26,25,21,1b] snapset 0=[]:{} 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process OpSchedulerItem(1.ds2 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c55141000 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.fs5 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] handle_advance_map [3,2,5,4,1,7]/[3,2,5,4,1,7] -- 3/3 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7:5.update_pg_epoch 1.ds2 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 63 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] handle_advance_map [2,1,7,3,4,5]/[2,1,7,3,4,5] -- 2/2 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f06185700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.ds2_head options 2019-12-07T19:34:41.646+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.fs5_head options 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op create 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f06185700 10 osd.7 pg_epoch: 64 pg[1.ds2( v 62'32 (0'0,62'32] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [2,1,7,3,4,5]p2(0) r=2 lpr=41 luod=0'0 crt=62'32 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f06185700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f06185700 20 osd.7 op_wq(5) _process empty q, waiting 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 62 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.fs5( v 57'17 (0'0,57'17] local-lis/les=41/42 n=4 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [3,2,5,4,1,7]p3(0) r=5 lpr=41 luod=0'0 crt=57'17 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) queued 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process 1.7s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process OpSchedulerItem(1.7s4 PGPeeringEvent(epoch_sent: 64 epoch_requested: 64 NullEvt) prio 255 cost 10 e64) pg 0x558c54d5f000 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] handle_advance_map: 64 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7:7.update_pg_epoch 1.7s4 63 -> 64 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 63 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] handle_advance_map [1,4,3,5,7,2]/[1,4,3,5,7,2] -- 1/1 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] state: Started advmap 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:41.646+0000 7f8f0117b700 15 bluestore(/var/lib/ceph/osd/ceph-7) set_collection_opts 1.7s4_head options 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] plpg_on_pool_change 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] hit_set_clear 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] agent_stop 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 64 advance_pg new pool opts old pool opts 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] handle_activate_map: 64 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] activate_map 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] activate_map: Not dirtying info: last_persisted is 63 while current is 64 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 osd.7 pg_epoch: 64 pg[1.7s4( v 63'49 (0'0,63'49] local-lis/les=41/42 n=6 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,3,5,7,2]p1(0) r=4 lpr=41 luod=0'0 crt=63'49 active mbc={} ps=[20~1,23~1]] do_peering_event: epoch_sent: 64 epoch_requested: 64 NullEvt 2019-12-07T19:34:41.646+0000 7f8f0117b700 10 log is not dirty 2019-12-07T19:34:41.646+0000 7f8f0117b700 20 osd.7 op_wq(7) _process empty q, waiting 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr dgXPi (630) in=635b 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] do_osd_op setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] make_writeable 1:4c9e444c:::smithi13140796-48:head snapset=0=[]:{} snapc=29=[29,28,26,25,21,1b] 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] setting DIRTY flag 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] make_writeable 1:4c9e444c:::smithi13140796-48:head done, snapset=29=[]:{} 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] finish_ctx 1:4c9e444c:::smithi13140796-48:head 0x558c5829e700 op modify 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] set mtime to 2019-12-07T19:34:41.645666+0000 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] final snapset 29=[]:{} in 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] finish_ctx object 1:4c9e444c:::smithi13140796-48:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] execute_ctx alloc reply 0x558c57d9b180 result 0 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] calc_trim_to_aggressive limit = 29'15 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] op order client.4540 tid 483 (first) 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] new_repop rep_tid 62 on osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) v8 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] new_repop: repgather(0x558c55159200 0'0 rep_tid=62 committed?=0 r=0) 2019-12-07T19:34:41.646+0000 7f8f03980700 7 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] issue_repop rep_tid 62 o 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] submit_transaction: op Op(1:4c9e444c:::smithi13140796-48:head v=64'16 tt=0'0 tid=62 reqid=client.4540.0:483 client_op=osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] get_hash_info: Getting attr on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] get_hash_info: not in cache 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:41.646+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:41.646+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:41.646+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) r -2 v.len 0 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] operator(): 1:4c9e444c:::smithi13140796-48:head projected size 0 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] start_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=64'16 tt=0'0 tid=62 reqid=client.4540.0:483 client_op=osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_state_to_reads: Op(1:4c9e444c:::smithi13140796-48:head v=64'16 tt=0'0 tid=62 reqid=client.4540.0:483 client_op=osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=64'16 tt=0'0 tid=62 reqid=client.4540.0:483 client_op=osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=2 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] generate_transactions: 1:4c9e444c:::smithi13140796-48:head, current size is 0 buffers are {} 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] operator(): new_size start 0 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] operator(): to_overwrite: {} 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] operator(): to_append: {} 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] operator(): 1:4c9e444c:::smithi13140796-48:head resetting hinfo to logical size 0 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_reads_to_commit: written: {1:4c9e444c:::smithi13140796-48:head={}} 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=64'16 tt=0'0 tid=62 reqid=client.4540.0:483 client_op=osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_reads_to_commit: written_set: {1:4c9e444c:::smithi13140796-48:head=[]} 2019-12-07T19:34:41.646+0000 7f8f03980700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] try_reads_to_commit: 1:4c9e444c:::smithi13140796-48:head,{} 2019-12-07T19:34:41.646+0000 7f8f03980700 10 osd.7 64 send_incremental_map 63 -> 64 to 0x558c54c64000 v2:172.21.15.131:6823/37078 2019-12-07T19:34:41.646+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cd9f972f:::inc_osdmap.64:0# 0x0~0 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c54e56fc0 blob([0xcd8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c585e0e00 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:cd9f972f:::inc_osdmap.64:0# 0x0~24d = 589 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 add_map_inc_bl 64 589 bytes 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(64..64 src has 1..64) v4 -- 0x558c5730dd40 con 0x558c54c64000 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 maybe_share_map con 0x558c54c64000 v2:172.21.15.131:6823/37078 map epoch 63 -> 64 (shared) 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 64/41 ECSubWrite(tid=62, reqid=client.4540.0:483, at_version=64'16, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c58154000 con 0x558c54c64000 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 send_incremental_map 62 -> 64 to 0x558c54c19800 v2:172.21.15.131:6804/37077 2019-12-07T19:34:41.647+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c71f972f:::inc_osdmap.63:0# 0x0~0 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_read will do buffered read 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _read_cache blob Blob(0x558c58b79110 blob([0xc2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558c5792dab0 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) read meta #-1:c71f972f:::inc_osdmap.63:0# 0x0~24d = 589 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 add_map_inc_bl 63 589 bytes 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(63..64 src has 1..64) v4 -- 0x558c58762b40 con 0x558c54c19800 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 maybe_share_map con 0x558c54c19800 v2:172.21.15.131:6804/37077 map epoch 62 -> 64 (shared) 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.2s4 64/41 ECSubWrite(tid=62, reqid=client.4540.0:483, at_version=64'16, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c58155800 con 0x558c54c19800 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 send_incremental_map 63 -> 64 to 0x558c54c5bc00 v2:172.21.15.131:6812/37076 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(64..64 src has 1..64) v4 -- 0x558c58762900 con 0x558c54c5bc00 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 maybe_share_map con 0x558c54c5bc00 v2:172.21.15.131:6812/37076 map epoch 63 -> 64 (shared) 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 64/41 ECSubWrite(tid=62, reqid=client.4540.0:483, at_version=64'16, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c54e3f000 con 0x558c54c5bc00 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 send_incremental_map 62 -> 64 to 0x558c54c5b400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(63..64 src has 1..64) v4 -- 0x558c587626c0 con 0x558c54c5b400 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 maybe_share_map con 0x558c54c5b400 v2:172.21.15.109:6810/36938 map epoch 62 -> 64 (shared) 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 64/41 ECSubWrite(tid=62, reqid=client.4540.0:483, at_version=64'16, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c58bce000 con 0x558c54c5b400 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 send_incremental_map 63 -> 64 to 0x558c54c19c00 v2:172.21.15.109:6802/36936 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(64..64 src has 1..64) v4 -- 0x558c5884e000 con 0x558c54c19c00 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 maybe_share_map con 0x558c54c19c00 v2:172.21.15.109:6802/36936 map epoch 63 -> 64 (shared) 2019-12-07T19:34:41.647+0000 7f8f03980700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s3 64/41 ECSubWrite(tid=62, reqid=client.4540.0:483, at_version=64'16, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558c58bce800 con 0x558c54c19c00 2019-12-07T19:34:41.647+0000 7f8f03980700 20 earliest_dup_version = 0 2019-12-07T19:34:41.647+0000 7f8f03980700 20 trim 64'16 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:483 2019-12-07T19:34:41.645666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] append_log log((0'0,29'15], crt=29'15) [64'16 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:483 2019-12-07T19:34:41.645666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 lua=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] add_log_entry 64'16 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:483 2019-12-07T19:34:41.645666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 lua=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] append_log approx pg log length = 16 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 lua=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.647+0000 7f8f03980700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.647+0000 7f8f03980700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 64'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c57ca0300 seq 34 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7).collection(1.2s0_head 0x558c575cc780) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:41.647+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _assign_nid 1299 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _touch 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# = 0 2019-12-07T19:34:41.647+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 22 keys 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 22 keys = 0 2019-12-07T19:34:41.647+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:41.647+0000 7f8f03980700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.0000000064.00000000000000000016' <- 0000000064.00000000000000000016 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._epoch' <- _epoch 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c57ca0300 cost 76832 (1 ios * 4000 + 72832 bytes) 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c57ca0300 onodes 0x558c575f8c80 shared_blobs 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore.extentmap(0x558c575f8d88) update 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore.extentmap(0x558c575f8d88) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 0#1:4c9e444c:::smithi13140796-48:head# is 70969 (70961 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c57ca0300 allocated 0x[] released 0x[] 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ca0300 prepare 2019-12-07T19:34:41.647+0000 7f8f03980700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c57ca0300 2019-12-07T19:34:41.647+0000 7f8f03980700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ca0300 io_done 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 lua=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] op_applied version 64'16 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 12387 < min_flush_size 524288 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 45155 < min_flush_size 524288 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 72436 < min_flush_size 524288 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] eval_repop repgather(0x558c55159200 64'16 rep_tid=62 committed?=0 r=0) 2019-12-07T19:34:41.647+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c57f3ef20 finish 2019-12-07T19:34:41.647+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c575f8c80 had 1 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 72459 < min_flush_size 524288 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 72459 < min_flush_size 524288 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x55cf9d~11b0b to file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x55cf9d 0x55cf9d~11b0b to file(ino 16 size 0x560000 mtime 2019-12-07T19:34:40.668165+0000 allocated 560000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~530000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _allocate len 0xeaa8 from 0 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _allocate len 0xeaa8 from 1 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd690000 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd690000~10000/10000,10000,acd690000 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x570000 to include allocated 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 81 (was clean) 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~540000 x_off 0x52cf9d 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf9d 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xaa8 and padding block with 0x558 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x570000 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x56eaa8 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (81) on file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]), flushing log 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 81 len 0x4a crc 0xc1ccff6) 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x50000~1000 to file(ino 1 size 0x50000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x50000 0x50000~1000 to file(ino 1 size 0x50000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x51000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x50000 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x51000 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:41.647+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000050 2019-12-07T19:34:41.647+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.648+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 81 2019-12-07T19:34:41.648+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.648+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000505642s (0.000000100s flush + 0.000505542s kv commit) 2019-12-07T19:34:41.648+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c57ca0300> 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.648+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ca0300 kv_submitted 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c57ca0300 2019-12-07T19:34:41.648+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c57ca0300 finishing 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c57ca0300 onodes 0x558c575f8c80 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c57ca0300 done 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:41.648+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c57ca0300 [] 2019-12-07T19:34:41.648+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.648+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 125 ==== MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c584fa000 con 0x558c54c19c00 2019-12-07T19:34:41.648+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000014 epoch 63 MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e63) 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e63) queued 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e63) pg 0x558c57e88000 2019-12-07T19:34:41.648+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 551 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c54ceb800 con 0x558c54c64000 2019-12-07T19:34:41.648+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c57f3e160 prio 196 cost 0 latency 0.000049 MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.648+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57f3fe40 prio 196 cost 0 latency 0.000011 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.648+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c57f3e160 finish 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.648+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c57f3fe40 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.648+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c57f3fe40 finish 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.648+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 325 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5841f600 con 0x558c54c5bc00 2019-12-07T19:34:41.648+0000 7f8f23c56700 15 osd.7 64 enqueue_op 0x558c5740d760 prio 196 cost 0 latency 0.000018 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.648+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c5740d760 prio 196 cost 0 latency 0.000271 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.648+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c5740d760 finish 2019-12-07T19:34:41.648+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.648+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 223 ==== MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c58115600 con 0x558c54c19800 2019-12-07T19:34:41.648+0000 7f8f24c58700 15 osd.7 64 enqueue_op 0x558c58bd4420 prio 196 cost 0 latency 0.000013 epoch 63 MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e63) 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e63) queued 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e63) pg 0x558c57e88000 2019-12-07T19:34:41.648+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c58bd4420 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.648+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 63/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.648+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c58bd4420 finish 2019-12-07T19:34:41.648+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.649+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 340 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c58115400 con 0x558c54c5b400 2019-12-07T19:34:41.649+0000 7f8f24c58700 15 osd.7 64 enqueue_op 0x558c58bd4580 prio 196 cost 0 latency 0.000009 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c58bd4580 prio 196 cost 0 latency 0.000033 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=62, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] handle_sub_write_reply Calling on_all_commit on Op(1:4c9e444c:::smithi13140796-48:head v=64'16 tt=0'0 tid=62 reqid=client.4540.0:483 client_op=osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 luod=29'15 crt=29'15 lcod 29'15 mlcod 0'0 active+clean ps=[20~1,23~1]] repop_all_committed: repop tid 62 all committed 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] eval_repop repgather(0x558c55159200 64'16 rep_tid=62 committed?=1 r=0) 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] commit: repgather(0x558c55159200 64'16 rep_tid=62 committed?=1 r=0) 2019-12-07T19:34:41.649+0000 7f8f07988700 15 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] log_op_stats osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) v8 inb 0 outb 0 lat 0.004497 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] sending reply on osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) v8 0x558c57d9b180 2019-12-07T19:34:41.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6824/36939,v1:172.21.15.109:6825/36939] --> 172.21.15.131:0/224485236 -- osd_op_reply(483 smithi13140796-48 [create,setxattr (19508),setxattr (630),setxattr (630),setxattr (1190),setxattr (138),setxattr (1719),setxattr (8420),setxattr (525),setxattr (245),setxattr (1435),setxattr (630),setxattr (13472),setxattr (14664),setxattr (523),setxattr (33),setxattr (1191),setxattr (630),setxattr (525),setxattr (2876),setxattr (631)] v64'16 uv16 ondisk = 0) v8 -- 0x558c57d9b180 con 0x558c54c9f000 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] prepare_stats_for_publish reporting purged_snaps [20~1,23~1] 2019-12-07T19:34:41.649+0000 7f8f07988700 15 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] publish_stats_to_osd 63:182 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] removing repgather(0x558c55159200 64'16 rep_tid=62 committed?=1 r=0) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] q front is repgather(0x558c55159200 64'16 rep_tid=62 committed?=1 r=0) 2019-12-07T19:34:41.649+0000 7f8f07988700 15 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] do_osd_op_effects client.4540 con 0x558c54c9f000 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] remove_repop repgather(0x558c55159200 64'16 rep_tid=62 committed?=1 r=0) 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=64'16 tt=0'0 tid=62 reqid=client.4540.0:483 client_op=osd_op(client.4540.0:483 1.2s0 1:4c9e444c:::smithi13140796-48:head [create,setxattr Rj\mZOfQXo^b_ZWWfqN]m[iPt^aRVgsYNetVd`VO (19508) in=19548b,setxattr [adnUNmYmWdN__]tR]msPnVSRiY^psh`dY\Z (630) in=666b,setxattr cPsWXtVeXTPagdn_WVZbVcV\UjSZUqsVlPZbY` (630) in=668b,setxattr pqbrReOSsgZlmftc]`Z`]USaXjsacf`fao_VP`eU (1190) in=1230b,setxattr aYfgr^q[\aPg_aZsakfUfsdhYbra^QXjlkrpdY\\ (138) in=178b,setxattr e\XXWiVknihNQQ[OsThQUbRmtprrfYPeRX[[mRcQ (1719) in=1759b,setxattr TNQYR^jgR`pPaYitmrcliV[dttobncTntcfTlQkt (8420) in=8460b,setxattr W_aq[RVsaUpQtX_RpV^Vpqrg`QgQViPeVaRqZSmf (525) in=565b,setxattr RWdkeaaUP\PZteTNscS^UVRtjo`oqqdNqe]cdZUa (245) in=285b,setxattr R^`al]TqifatYOiOU]p`ZegfXmNsXVaZcdd]b]YX (1435) in=1475b,setxattr p]OS_Znl[dNa^qTn`WW[j\dtgbNac_bijYmg (630) in=666b,setxattr n_jr[tV[VhYPqU]UenciobhtYfY^a^c\hd^YrcQ[ (13472) in=13512b,setxattr gReQSUg`cdl_ilU_lo[ggg\fZsVQY[PgY]jPsZ_U (14664) in=14704b,setxattr cNWaQrsdt^dic_qN_^mTXmc`_X]YOTXNk[WldRe` (523) in=563b,setxattr tZ]grhlaq]kUO[k^d^dr``XNio^^Oaebpj\[Ytdq (33) in=73b,setxattr jqS_j^cefaXab`pQoPXfRkSmdako`fZ_heb^_fma (1191) in=1231b,setxattr dgXPi (630) in=635b,setxattr Or_kkpbnNkTrQ\apUaUsZR\XW^nabhaiQ\]gOjR\ (525) in=565b,setxattr Rf`aR[fdtjQ\NaqZke]l_rUssrVpOPXYWpXkl_kd (2876) in=2916b,setxattr _sUSYWYUmOrlUVhZajZeYbbnsdra[]`Q`^Ogea[m (631) in=671b] snapc 29=[29,28,26,25,21,1b] ondisk+write+known_if_redirected e64) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=63 reqid=client.4540.0:483 roll_forward_to=64'16 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=63 reqid=client.4540.0:483 roll_forward_to=64'16 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:41.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 64/41 ECSubWrite(tid=63, reqid=client.4540.0:483, at_version=0'0, trim_to=0'0, roll_forward_to=64'16)) v2 -- 0x558c575bf000 con 0x558c54c64000 2019-12-07T19:34:41.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.2s4 64/41 ECSubWrite(tid=63, reqid=client.4540.0:483, at_version=0'0, trim_to=0'0, roll_forward_to=64'16)) v2 -- 0x558c58bcf000 con 0x558c54c19800 2019-12-07T19:34:41.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 64/41 ECSubWrite(tid=63, reqid=client.4540.0:483, at_version=0'0, trim_to=0'0, roll_forward_to=64'16)) v2 -- 0x558c58bcf800 con 0x558c54c5bc00 2019-12-07T19:34:41.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 64/41 ECSubWrite(tid=63, reqid=client.4540.0:483, at_version=0'0, trim_to=0'0, roll_forward_to=64'16)) v2 -- 0x558c54d84000 con 0x558c54c5b400 2019-12-07T19:34:41.649+0000 7f8f07988700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s3 64/41 ECSubWrite(tid=63, reqid=client.4540.0:483, at_version=0'0, trim_to=0'0, roll_forward_to=64'16)) v2 -- 0x558c57e7b800 con 0x558c54c19c00 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=29'15 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] append_log log((0'0,64'16], crt=29'15) [] 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] rollforward: entry=64'16 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:483 2019-12-07T19:34:41.645666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] append_log approx pg log length = 16 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.649+0000 7f8f07988700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.649+0000 7f8f07988700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.649+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c575cc780 1.2s0_head 2019-12-07T19:34:41.649+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c54e10dc0 = 0x558c587ce300 seq 35 2019-12-07T19:34:41.649+0000 7f8f07988700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:41.649+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'._info' <- _info 2019-12-07T19:34:41.649+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.649+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x0000000000000493'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.649+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:41.649+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c587ce300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c587ce300 onodes shared_blobs 2019-12-07T19:34:41.649+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c587ce300 allocated 0x[] released 0x[] 2019-12-07T19:34:41.649+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce300 prepare 2019-12-07T19:34:41.649+0000 7f8f07988700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c587ce300 2019-12-07T19:34:41.649+0000 7f8f07988700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce300 io_done 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c58bd4580 finish 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c57e8e280 had 1 2019-12-07T19:34:41.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:41.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:41.649+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.649+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x56eaa8~471 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.649+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x56eaa8 0x56eaa8~471 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~540000 x_off 0x53eaa8 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xaa8 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.649+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf19 and padding block with 0xe7 2019-12-07T19:34:41.649+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.4 v2:172.21.15.109:6810/36938 341 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558c54d76000 con 0x558c54c5b400 2019-12-07T19:34:41.649+0000 7f8f24c58700 15 osd.7 64 enqueue_op 0x558c58bd46e0 prio 196 cost 0 latency 0.000014 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.649+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c58bd46e0 prio 196 cost 0 latency 0.000039 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.5 v2:172.21.15.109:6802/36936 126 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c55045400 con 0x558c54c19c00 2019-12-07T19:34:41.649+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57f3eb00 prio 196 cost 0 latency 0.000008 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.649+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c58bd46e0 finish 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.649+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c57f3eb00 prio 196 cost 0 latency 0.000022 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.649+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c57f3eb00 finish 2019-12-07T19:34:41.649+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.649+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 552 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c575b8c00 con 0x558c54c64000 2019-12-07T19:34:41.649+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57f3f080 prio 196 cost 0 latency 0.000007 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f24457700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c57f3f080 prio 196 cost 0 latency 0.000024 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.649+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c57f3f080 finish 2019-12-07T19:34:41.649+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.650+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.2 v2:172.21.15.131:6804/37077 224 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c5795d200 con 0x558c54c19800 2019-12-07T19:34:41.650+0000 7f8f24c58700 15 osd.7 64 enqueue_op 0x558c58bd4840 prio 196 cost 0 latency 0.000008 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.650+0000 7f8f24c58700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.650+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.650+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.650+0000 7f8f03980700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.650+0000 7f8f03980700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.650+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c58bd4840 prio 196 cost 0 latency 0.000026 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.650+0000 7f8f03980700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.650+0000 7f8f03980700 10 osd.7 64 dequeue_op 0x558c58bd4840 finish 2019-12-07T19:34:41.650+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.650+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.3 v2:172.21.15.131:6812/37076 326 ==== MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x558c57519000 con 0x558c54c5bc00 2019-12-07T19:34:41.650+0000 7f8f23c56700 15 osd.7 64 enqueue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000010 epoch 64 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.650+0000 7f8f23c56700 20 osd.7 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) queued 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2) prio 196 cost 0 e64) pg 0x558c57e88000 2019-12-07T19:34:41.650+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c54e791e0 prio 196 cost 0 latency 0.000029 MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 pg pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] 2019-12-07T19:34:41.650+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 64/41 ECSubWriteReply(tid=63, last_complete=64'16, committed=1, applied=1)) v2 2019-12-07T19:34:41.650+0000 7f8f07988700 10 osd.7 64 dequeue_op 0x558c54e791e0 finish 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.650+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x56ef19 2019-12-07T19:34:41.650+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.650+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.650+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.650+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001063507s (0.000000109s flush + 0.001063398s kv commit) 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.650+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c587ce300> 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.650+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce300 kv_submitted 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c587ce300 2019-12-07T19:34:41.650+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c587ce300 finishing 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c587ce300 onodes 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c587ce300 done 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c54e10dc0 q now empty 2019-12-07T19:34:41.650+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c587ce300 [] 2019-12-07T19:34:41.650+0000 7f8f03980700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.650+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.650+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.650+0000 7f8f07988700 10 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=63 reqid=client.4540.0:483 roll_forward_to=64'16 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 pg_epoch: 64 pg[1.2s0( v 64'16 (0'0,64'16] local-lis/les=41/42 n=3 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [7,4,1,5,2,3]p7(0) r=0 lpr=41 crt=64'16 lcod 29'15 mlcod 29'15 active+clean ps=[20~1,23~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:41.650+0000 7f8f07988700 20 osd.7 op_wq(2) _process empty q, waiting 2019-12-07T19:34:41.651+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 553 ==== MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=320, reqid=client.4540.0:484, at_version=64'56, trim_to=0'0, roll_forward_to=57'55)) v2 ==== 190840+0+0 (crc 0 0 0) 0x558c575bf000 con 0x558c54c64000 2019-12-07T19:34:41.651+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57f3fb80 prio 127 cost 0 latency 0.000029 epoch 64 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=320, reqid=client.4540.0:484, at_version=64'56, trim_to=0'0, roll_forward_to=57'55)) v2 2019-12-07T19:34:41.651+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=320, reqid=client.4540.0:484, at_version=64'56, trim_to=0'0, roll_forward_to=57'55)) v2) prio 127 cost 0 e64) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=320, reqid=client.4540.0:484, at_version=64'56, trim_to=0'0, roll_forward_to=57'55)) v2) prio 127 cost 0 e64) queued 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=320, reqid=client.4540.0:484, at_version=64'56, trim_to=0'0, roll_forward_to=57'55)) v2) prio 127 cost 0 e64) pg 0x558c57380000 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 osd.7 64 dequeue_op 0x558c57f3fb80 prio 127 cost 0 latency 0.000116 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=320, reqid=client.4540.0:484, at_version=64'56, trim_to=0'0, roll_forward_to=57'55)) v2 pg pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=320, reqid=client.4540.0:484, at_version=64'56, trim_to=0'0, roll_forward_to=57'55)) v2 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 57'55 (0'0,57'55] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] append_log log((0'0,57'55], crt=57'55) [64'56 (57'53) modify 1:10779340:::smithi13140796-32:head by client.4540.0:484 2019-12-07T19:34:41.650070+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0] 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'55 crt=57'55 lcod 57'55 active mbc={} ps=[20~1,23~1]] add_log_entry 64'56 (57'53) modify 1:10779340:::smithi13140796-32:head by client.4540.0:484 2019-12-07T19:34:41.650070+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'55 crt=57'55 lcod 57'55 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 56 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'55 crt=57'55 lcod 57'55 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.652+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 64'56, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58500600 seq 57 2019-12-07T19:34:41.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:41.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:41.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x0~2e000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:10779340:::smithi13140796-32:head# 0x0~2e000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x0~2e000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58500600 23 blobs 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2e000/1000,2e000,0 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xcdb000~2e000/1000,2e000,0 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xcdb000~2e000] 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ebce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c66e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ebce0 blob([0xcdb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66e70 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebce0) get_ref 0x0~2000 Blob(0x558c583ebce0 blob([0xcdb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66e70 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebce0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558c583ebce0 blob([0xcdb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c66e70 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c58c66e88 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b3420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5792ce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b3420 blob([0xcdd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792ce70 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c550b3420) get_ref 0x0~2000 Blob(0x558c550b3420 blob([0xcdd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5792ce70 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c550b3420) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558c550b3420 blob([0xcdd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5792ce70 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5792ce88 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5898bb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580a5a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5898bb90 blob([0xcdf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5a40 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5898bb90) get_ref 0x0~2000 Blob(0x558c5898bb90 blob([0xcdf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580a5a40 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5898bb90) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558c5898bb90 blob([0xcdf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580a5a40 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c580a5a58 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840f7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840f7a0 blob([0xce1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb20 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f7a0) get_ref 0x0~2000 Blob(0x558c5840f7a0 blob([0xce1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840fb20 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558c5840f7a0 blob([0xce1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840fb20 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840fb38 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840f8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840f8f0 blob([0xce3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a4d0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f8f0) get_ref 0x0~2000 Blob(0x558c5840f8f0 blob([0xce3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57f3a4d0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558c5840f8f0 blob([0xce3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57f3a4d0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57f3a4e8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840f880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840f810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840f880 blob([0xce5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f810 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f880) get_ref 0x0~2000 Blob(0x558c5840f880 blob([0xce5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f810 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f880) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558c5840f880 blob([0xce5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f810 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840f828 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5840f5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5840f570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5840f5e0 blob([0xce7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f570 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f5e0) get_ref 0x0~2000 Blob(0x558c5840f5e0 blob([0xce7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5840f570 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5840f5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558c5840f5e0 blob([0xce7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5840f570 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5840f588 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c583ebb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c583ebb20 blob([0xce9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec1c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebb20) get_ref 0x0~2000 Blob(0x558c583ebb20 blob([0xce9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec1c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c583ebb20) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558c583ebb20 blob([0xce9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec1c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c580ec1d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57a06fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57a06fc0 blob([0xceb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc00 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c57a06fc0) get_ref 0x0~2000 Blob(0x558c57a06fc0 blob([0xceb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ebc00 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c57a06fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558c57a06fc0 blob([0xceb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ebc00 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c583ebc18 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c578f0cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c583ea8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c578f0cb0 blob([0xced000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea8c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c578f0cb0) get_ref 0x0~2000 Blob(0x558c578f0cb0 blob([0xced000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c583ea8c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c578f0cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558c578f0cb0 blob([0xced000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c583ea8c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c583ea8d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c580ec2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c580ec2a0 blob([0xcef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f0a0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c580ec2a0) get_ref 0x0~2000 Blob(0x558c580ec2a0 blob([0xcef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f0a0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c580ec2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558c580ec2a0 blob([0xcef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f0a0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f0b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f110 blob([0xcf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f180 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f110) get_ref 0x0~2000 Blob(0x558c5811f110 blob([0xcf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f180 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f110) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558c5811f110 blob([0xcf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f180 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f198 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f1f0 blob([0xcf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f260 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f1f0) get_ref 0x0~2000 Blob(0x558c5811f1f0 blob([0xcf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f260 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558c5811f1f0 blob([0xcf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f260 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f278 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f2d0 blob([0xcf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f340 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f2d0) get_ref 0x0~2000 Blob(0x558c5811f2d0 blob([0xcf5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f340 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558c5811f2d0 blob([0xcf5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f340 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f358 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f3b0 blob([0xcf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f420 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f3b0) get_ref 0x0~2000 Blob(0x558c5811f3b0 blob([0xcf7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f420 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558c5811f3b0 blob([0xcf7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f420 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f438 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f490 blob([0xcf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f500 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f490) get_ref 0x0~2000 Blob(0x558c5811f490 blob([0xcf9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f500 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f490) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558c5811f490 blob([0xcf9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f500 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f518 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f570 blob([0xcfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f5e0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f570) get_ref 0x0~2000 Blob(0x558c5811f570 blob([0xcfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f5e0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f570) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558c5811f570 blob([0xcfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f5e0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f5f8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f650 blob([0xcfd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f6c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f650) get_ref 0x0~2000 Blob(0x558c5811f650 blob([0xcfd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f6c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f650) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558c5811f650 blob([0xcfd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f6c0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f6d8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f730 blob([0xcff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f7a0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f730) get_ref 0x0~2000 Blob(0x558c5811f730 blob([0xcff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f7a0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f730) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558c5811f730 blob([0xcff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f7a0 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f7b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f810 blob([0xd01000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f880 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f810) get_ref 0x0~2000 Blob(0x558c5811f810 blob([0xd01000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f880 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f810) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558c5811f810 blob([0xd01000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f880 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f898 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811f960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f8f0 blob([0xd03000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f960 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f8f0) get_ref 0x0~2000 Blob(0x558c5811f8f0 blob([0xd03000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811f960 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558c5811f8f0 blob([0xd03000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811f960 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811f978 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811f9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811fa40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811f9d0 blob([0xd05000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811fa40 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f9d0) get_ref 0x0~2000 Blob(0x558c5811f9d0 blob([0xd05000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811fa40 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811f9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558c5811f9d0 blob([0xd05000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811fa40 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811fa58 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5811fab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5811fb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5811fab0 blob([0xd07000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811fb20 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811fab0) get_ref 0x0~2000 Blob(0x558c5811fab0 blob([0xd07000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5811fb20 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.blob(0x558c5811fab0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558c5811fab0 blob([0xd07000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c5811fb20 sbid 0x0)) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5811fb38 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x2e000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) dirty_range mark inline shard dirty 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x0~2e000 = 0 2019-12-07T19:34:41.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:41.652+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000064.00000000000000000056' <- 0000000064.00000000000000000056 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._epoch' <- _epoch 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58500600 cost 287298 (24 ios * 4000 + 191298 bytes) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58500600 onodes 0x558c5840c780 shared_blobs 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update inline shard 440 bytes from 23 extents 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 915 (469 bytes onode + 2 bytes spanning blobs + 444 bytes inline extents) 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58500600 allocated 0x[cdb000~2e000] released 0x[] 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 freelist allocate 0xcdb000~2e000 2019-12-07T19:34:41.652+0000 7f8f0898a700 20 freelist _xor first_key 0xc80000 last_key 0xd00000 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500600 prepare 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58500600 2019-12-07T19:34:41.652+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=57'55 crt=57'55 lcod 57'55 active mbc={} ps=[20~1,23~1]] op_applied version 64'56 2019-12-07T19:34:41.653+0000 7f8f0898a700 10 osd.7 64 dequeue_op 0x558c57f3fb80 finish 2019-12-07T19:34:41.653+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500600 aio_wait 2019-12-07T19:34:41.653+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58500600 2019-12-07T19:34:41.653+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500600 io_done 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2938 < min_flush_size 524288 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2961 < min_flush_size 524288 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2961 < min_flush_size 524288 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x56ef19~b91 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x56ef19 0x56ef19~b91 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~540000 x_off 0x53ef19 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf19 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xaaa and padding block with 0x556 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x56faaa 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000148620s (0.000043529s flush + 0.000105091s kv commit) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58500600> 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500600 kv_submitted 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58500600 2019-12-07T19:34:41.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500600 finishing 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58500600 onodes 0x558c5840c780 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5792ce88 in 0x558c53a59960) _finish_write discard buffer(0x558c5813a6c0 space 0x558c5792ce88 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57f3a4e8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5c20 space 0x558c57f3a4e8 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580a5a58 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5b00 space 0x558c580a5a58 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec1d8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5dd0 space 0x558c580ec1d8 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f0b8 in 0x558c53a59960) _finish_write discard buffer(0x558c58c17680 space 0x558c5811f0b8 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f198 in 0x558c53a59960) _finish_write discard buffer(0x558c58c17710 space 0x558c5811f198 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f278 in 0x558c53a59960) _finish_write discard buffer(0x558c58c17830 space 0x558c5811f278 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f358 in 0x558c53a59960) _finish_write discard buffer(0x558c58c178c0 space 0x558c5811f358 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f438 in 0x558c53a59960) _finish_write discard buffer(0x558c58c17950 space 0x558c5811f438 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f518 in 0x558c53a59960) _finish_write discard buffer(0x558c58c179e0 space 0x558c5811f518 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f5f8 in 0x558c53a59960) _finish_write discard buffer(0x558c58c17440 space 0x558c5811f5f8 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f6d8 in 0x558c53a59960) _finish_write discard buffer(0x558c58c16510 space 0x558c5811f6d8 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 64/41 ECSubWriteReply(tid=320, last_complete=64'56, committed=1, applied=1)) v2 -- 0x558c58114200 con 0x558c54c64000 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f7b8 in 0x558c53a59960) _finish_write discard buffer(0x558c58c165a0 space 0x558c5811f7b8 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f898 in 0x558c53a59960) _finish_write discard buffer(0x558c58c16630 space 0x558c5811f898 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811f978 in 0x558c53a59960) _finish_write discard buffer(0x558c58c166c0 space 0x558c5811f978 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811fa58 in 0x558c53a59960) _finish_write discard buffer(0x558c58c16750 space 0x558c5811fa58 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5811fb38 in 0x558c53a59960) _finish_write discard buffer(0x558c58c167e0 space 0x558c5811fb38 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ea8d8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5ef0 space 0x558c583ea8d8 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c583ebc18 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5e60 space 0x558c583ebc18 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840f588 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5d40 space 0x558c5840f588 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840f828 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5cb0 space 0x558c5840f828 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5840fb38 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5b90 space 0x558c5840fb38 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c66e88 in 0x558c53a59960) _finish_write discard buffer(0x558c580f3950 space 0x558c58c66e88 0x0~2000 writing nocache) 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58500600 done 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:41.653+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58500600 [] 2019-12-07T19:34:41.653+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.653+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.653+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 554 ==== MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=321, reqid=client.4540.0:484, at_version=0'0, trim_to=0'0, roll_forward_to=64'56)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c57e7b800 con 0x558c54c64000 2019-12-07T19:34:41.653+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57f3edc0 prio 127 cost 0 latency 0.000017 epoch 64 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=321, reqid=client.4540.0:484, at_version=0'0, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.653+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=321, reqid=client.4540.0:484, at_version=0'0, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) 2019-12-07T19:34:41.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=321, reqid=client.4540.0:484, at_version=0'0, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) queued 2019-12-07T19:34:41.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=321, reqid=client.4540.0:484, at_version=0'0, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) pg 0x558c57380000 2019-12-07T19:34:41.653+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57f3edc0 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=321, reqid=client.4540.0:484, at_version=0'0, trim_to=0'0, roll_forward_to=64'56)) v2 pg pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:41.653+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=321, reqid=client.4540.0:484, at_version=0'0, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.653+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=57'55 active mbc={} ps=[20~1,23~1]] append_log log((0'0,64'56], crt=57'55) [] 2019-12-07T19:34:41.653+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] rollforward: entry=64'56 (57'53) modify 1:10779340:::smithi13140796-32:head by client.4540.0:484 2019-12-07T19:34:41.650070+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.653+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 56 2019-12-07T19:34:41.653+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.653+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.653+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.653+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:41.653+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5875af00 seq 58 2019-12-07T19:34:41.653+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:41.653+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:41.653+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.653+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.653+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:41.653+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875af00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:41.653+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875af00 onodes shared_blobs 2019-12-07T19:34:41.653+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875af00 allocated 0x[] released 0x[] 2019-12-07T19:34:41.653+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875af00 prepare 2019-12-07T19:34:41.653+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875af00 2019-12-07T19:34:41.653+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875af00 io_done 2019-12-07T19:34:41.653+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57f3edc0 finish 2019-12-07T19:34:41.653+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:41.653+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.653+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x56faaa~471 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.654+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x56faaa 0x56faaa~471 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.654+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.654+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~540000 x_off 0x53faaa 2019-12-07T19:34:41.654+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xaaa 2019-12-07T19:34:41.654+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.654+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xf1b and padding block with 0xe5 2019-12-07T19:34:41.655+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x56ff1b 2019-12-07T19:34:41.655+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.655+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.655+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001233667s (0.000000090s flush + 0.001233577s kv commit) 2019-12-07T19:34:41.655+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875af00> 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875af00 kv_submitted 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875af00 2019-12-07T19:34:41.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875af00 finishing 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875af00 onodes 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875af00 done 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:41.655+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875af00 [] 2019-12-07T19:34:41.655+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.655+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.655+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.655+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 64/41 ECSubWriteReply(tid=321, last_complete=64'56, committed=1, applied=1)) v2 -- 0x558c548cda00 con 0x558c54c64000 2019-12-07T19:34:41.655+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.656+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1738646 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.656+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 555 ==== MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=322, reqid=client.4540.0:485, at_version=64'57, trim_to=0'0, roll_forward_to=64'56)) v2 ==== 182664+0+0 (crc 0 0 0) 0x558c58bce800 con 0x558c54c64000 2019-12-07T19:34:41.656+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000022 epoch 64 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=322, reqid=client.4540.0:485, at_version=64'57, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.656+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=322, reqid=client.4540.0:485, at_version=64'57, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=322, reqid=client.4540.0:485, at_version=64'57, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) queued 2019-12-07T19:34:41.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=322, reqid=client.4540.0:485, at_version=64'57, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) pg 0x558c57380000 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57334dc0 prio 127 cost 0 latency 0.000043 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=322, reqid=client.4540.0:485, at_version=64'57, trim_to=0'0, roll_forward_to=64'56)) v2 pg pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=322, reqid=client.4540.0:485, at_version=64'57, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'56 (0'0,64'56] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] append_log log((0'0,64'56], crt=64'56) [64'57 (64'56) modify 1:10779340:::smithi13140796-32:head by client.4540.0:485 2019-12-07T19:34:41.654499+0000 0 ObjectCleanRegions clean_offsets: [0~753664,1474560~18446744073708077055], clean_omap: 1, new_object: 0] 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'57 (0'0,64'57] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'56 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] add_log_entry 64'57 (64'56) modify 1:10779340:::smithi13140796-32:head by client.4540.0:485 2019-12-07T19:34:41.654499+0000 0 ObjectCleanRegions clean_offsets: [0~753664,1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'57 (0'0,64'57] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'56 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 57 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'57 (0'0,64'57] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'56 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.656+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.656+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 64'57, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5875b200 seq 59 2019-12-07T19:34:41.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:41.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:41.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x2e000~2c000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:10779340:::smithi13140796-32:head# 0x2e000~2c000 - have 0x2e000 (188416) bytes fadvise_flags 0x20 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x2e000~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c5875b200 22 blobs 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.656+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate 0x2c000/1000,2c000,0 2019-12-07T19:34:41.656+0000 7f8f04982700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xd09000~2c000/1000,2c000,0 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xd09000~2c000] 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db4700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6850 blob([0xd09000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4700 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6850) get_ref 0x0~2000 Blob(0x558c57bb6850 blob([0xd09000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4700 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6850) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558c57bb6850 blob([0xd09000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db4700 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4718 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b2460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a1c0 blob([0xd0b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2460 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a1c0) get_ref 0x0~2000 Blob(0x558c57f3a1c0 blob([0xd0b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b2460 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558c57f3a1c0 blob([0xd0b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b2460 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c550b2478 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c66bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db4620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c66bd0 blob([0xd0d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4620 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66bd0) get_ref 0x0~2000 Blob(0x558c58c66bd0 blob([0xd0d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4620 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558c58c66bd0 blob([0xd0d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db4620 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4638 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db45b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a230 blob([0xd0f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db45b0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a230) get_ref 0x0~2000 Blob(0x558c57f3a230 blob([0xd0f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db45b0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a230) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558c57f3a230 blob([0xd0f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db45b0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db45c8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db4540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6a10 blob([0xd11000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4540 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6a10) get_ref 0x0~2000 Blob(0x558c57bb6a10 blob([0xd11000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4540 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6a10) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558c57bb6a10 blob([0xd11000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db4540 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4558 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db43f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a2a0 blob([0xd13000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db43f0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a2a0) get_ref 0x0~2000 Blob(0x558c57f3a2a0 blob([0xd13000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db43f0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558c57f3a2a0 blob([0xd13000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db43f0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4408 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c66b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c66b60 blob([0xd15000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec5b0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66b60) get_ref 0x0~2000 Blob(0x558c58c66b60 blob([0xd15000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec5b0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66b60) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558c58c66b60 blob([0xd15000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec5b0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ec5c8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57ed1b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a310 blob([0xd17000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed1b90 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a310) get_ref 0x0~2000 Blob(0x558c57f3a310 blob([0xd17000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57ed1b90 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a310) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558c57f3a310 blob([0xd17000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57ed1b90 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57ed1ba8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57bb6930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c580ec540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57bb6930 blob([0xd19000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec540 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6930) get_ref 0x0~2000 Blob(0x558c57bb6930 blob([0xd19000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c580ec540 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57bb6930) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558c57bb6930 blob([0xd19000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c580ec540 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c580ec558 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c66af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db44d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c66af0 blob([0xd1b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db44d0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66af0) get_ref 0x0~2000 Blob(0x558c58c66af0 blob([0xd1b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db44d0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66af0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558c58c66af0 blob([0xd1b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db44d0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db44e8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db50a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a3f0 blob([0xd1d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db50a0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a3f0) get_ref 0x0~2000 Blob(0x558c57f3a3f0 blob([0xd1d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db50a0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558c57f3a3f0 blob([0xd1d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db50a0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db50b8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c58c66a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db5030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c58c66a80 blob([0xd1f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5030 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66a80) get_ref 0x0~2000 Blob(0x558c58c66a80 blob([0xd1f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db5030 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c58c66a80) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558c58c66a80 blob([0xd1f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db5030 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db5048 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5898b730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db4e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5898b730 blob([0xd21000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4e00 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c5898b730) get_ref 0x0~2000 Blob(0x558c5898b730 blob([0xd21000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4e00 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c5898b730) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558c5898b730 blob([0xd21000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db4e00 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4e18 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c550b3340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a460 blob([0xd23000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3340 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a460) get_ref 0x0~2000 Blob(0x558c57f3a460 blob([0xd23000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c550b3340 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a460) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558c57f3a460 blob([0xd23000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c550b3340 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c550b3358 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c550b24d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db4ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c550b24d0 blob([0xd25000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4ee0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c550b24d0) get_ref 0x0~2000 Blob(0x558c550b24d0 blob([0xd25000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4ee0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c550b24d0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558c550b24d0 blob([0xd25000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db4ee0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4ef8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57db4fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a380 blob([0xd27000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4fc0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a380) get_ref 0x0~2000 Blob(0x558c57f3a380 blob([0xd27000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57db4fc0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a380) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558c57f3a380 blob([0xd27000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57db4fc0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57db4fd8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c54d3c2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c54d3c2a0 blob([0xd29000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6af0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c54d3c2a0) get_ref 0x0~2000 Blob(0x558c54d3c2a0 blob([0xd29000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6af0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c54d3c2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558c54d3c2a0 blob([0xd29000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6af0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6b08 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c66c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a150 blob([0xd2b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66c40 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a150) get_ref 0x0~2000 Blob(0x558c57f3a150 blob([0xd2b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66c40 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a150) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558c57f3a150 blob([0xd2b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c66c40 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c58c66c58 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a0e0 blob([0xd2d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6bd0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a0e0) get_ref 0x0~2000 Blob(0x558c57f3a0e0 blob([0xd2d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6bd0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558c57f3a0e0 blob([0xd2d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6bd0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6be8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c66cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a070 blob([0xd2f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66cb0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a070) get_ref 0x0~2000 Blob(0x558c57f3a070 blob([0xd2f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66cb0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a070) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558c57f3a070 blob([0xd2f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c66cb0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c58c66cc8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57f3a000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57f3a000 blob([0xd31000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6cb0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a000) get_ref 0x0~2000 Blob(0x558c57f3a000 blob([0xd31000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57bb6cb0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57f3a000) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558c57f3a000 blob([0xd31000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57bb6cb0 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c57bb6cc8 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c57efdf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c58c66d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c57efdf80 blob([0xd33000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66d20 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57efdf80) get_ref 0x0~2000 Blob(0x558c57efdf80 blob([0xd33000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c58c66d20 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.blob(0x558c57efdf80) get_ref init 0x2000, 1000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558c57efdf80 blob([0xd33000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c58c66d20 sbid 0x0)) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.BufferSpace(0x558c58c66d38 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5a000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c888) dirty_range mark inline shard dirty 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x2e000~2c000 = 0 2019-12-07T19:34:41.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:41.656+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000064.00000000000000000057' <- 0000000064.00000000000000000057 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._fastinfo' <- _fastinfo 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875b200 cost 274314 (23 ios * 4000 + 182314 bytes) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875b200 onodes 0x558c5840c780 shared_blobs 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c888) update inline shard 858 bytes from 45 extents 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 1333 (469 bytes onode + 2 bytes spanning blobs + 862 bytes inline extents) 2019-12-07T19:34:41.656+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875b200 allocated 0x[d09000~2c000] released 0x[] 2019-12-07T19:34:41.656+0000 7f8f04982700 10 freelist allocate 0xd09000~2c000 2019-12-07T19:34:41.656+0000 7f8f04982700 20 freelist _xor first_key 0xd00000 last_key 0xd00000 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b200 prepare 2019-12-07T19:34:41.656+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c5875b200 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'57 (0'0,64'57] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'56 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] op_applied version 64'57 2019-12-07T19:34:41.656+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57334dc0 finish 2019-12-07T19:34:41.656+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.656+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b200 aio_wait 2019-12-07T19:34:41.656+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875b200 2019-12-07T19:34:41.656+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b200 io_done 2019-12-07T19:34:41.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.656+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:41.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 229 < min_flush_size 524288 2019-12-07T19:34:41.656+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2540 < min_flush_size 524288 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2563 < min_flush_size 524288 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2563 < min_flush_size 524288 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x56ff1b~a03 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x56ff1b 0x56ff1b~a03 to file(ino 16 size 0x570000 mtime 2019-12-07T19:34:41.648831+0000 allocated 570000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~540000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _allocate len 0x91e from 0 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _allocate len 0x91e from 1 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate 0x10000/10000,10000,acd6a0000 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 fbmap_alloc 0x558c53a73100 allocate extent: 0xacd6a0000~10000/10000,10000,acd6a0000 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush_range extending WAL size to 0x580000 to include allocated 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range dirty_seq = 82 (was clean) 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~550000 x_off 0x53ff1b 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xf1b 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x91e and padding block with 0x6e2 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range zeroing WAL log up to 0x580000 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x57091e 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _fsync file metadata was dirty (82) on file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]), flushing log 2019-12-07T19:34:41.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 556 ==== MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=323, reqid=client.4540.0:486, at_version=64'58, trim_to=0'0, roll_forward_to=64'56)) v2 ==== 14728+0+0 (crc 0 0 0) 0x558c58154000 con 0x558c54c64000 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush_and_sync_log txn(seq 82 len 0x4a crc 0xadaa83dc) 2019-12-07T19:34:41.657+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c573358c0 prio 127 cost 0 latency 0.000026 epoch 64 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=323, reqid=client.4540.0:486, at_version=64'58, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush 0x558c53be5a40 0x51000~1000 to file(ino 1 size 0x51000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:41.657+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=323, reqid=client.4540.0:486, at_version=64'58, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _flush_range 0x558c53be5a40 pos 0x51000 0x51000~1000 to file(ino 1 size 0x51000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 1 size 0x52000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x51000 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c53be5a40 pos now 0x52000 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 2019-12-07T19:34:41.657+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c53be5a40 done in 0.000025 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_and_sync_log log_seq_stable 82 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000538360s (0.000045310s flush + 0.000493050s kv commit) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.657+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875b200> 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b200 kv_submitted 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875b200 2019-12-07T19:34:41.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b200 finishing 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875b200 onodes 0x558c5840c780 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b2478 in 0x558c53a59960) _finish_write discard buffer(0x558c57a087e0 space 0x558c550b2478 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c550b3358 in 0x558c53a59960) _finish_write discard buffer(0x558c5813bc20 space 0x558c550b3358 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6b08 in 0x558c53a59960) _finish_write discard buffer(0x558c5813bdd0 space 0x558c57bb6b08 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6be8 in 0x558c53a59960) _finish_write discard buffer(0x558c5813bef0 space 0x558c57bb6be8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57bb6cc8 in 0x558c53a59960) _finish_write discard buffer(0x558c57f1dc20 space 0x558c57bb6cc8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4408 in 0x558c53a59960) _finish_write discard buffer(0x558c57f76b40 space 0x558c57db4408 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=323, reqid=client.4540.0:486, at_version=64'58, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) queued 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db44e8 in 0x558c53a59960) _finish_write discard buffer(0x558c5813b9e0 space 0x558c57db44e8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4558 in 0x558c53a59960) _finish_write discard buffer(0x558c550e59e0 space 0x558c57db4558 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db45c8 in 0x558c53a59960) _finish_write discard buffer(0x558c550e5950 space 0x558c57db45c8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4638 in 0x558c53a59960) _finish_write discard buffer(0x558c5813b0e0 space 0x558c57db4638 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4718 in 0x558c53a59960) _finish_write discard buffer(0x558c580f3680 space 0x558c57db4718 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4e18 in 0x558c53a59960) _finish_write discard buffer(0x558c5813bb90 space 0x558c57db4e18 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4ef8 in 0x558c53a59960) _finish_write discard buffer(0x558c5813bcb0 space 0x558c57db4ef8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db4fd8 in 0x558c53a59960) _finish_write discard buffer(0x558c5813bd40 space 0x558c57db4fd8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=323, reqid=client.4540.0:486, at_version=64'58, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) pg 0x558c57380000 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db5048 in 0x558c53a59960) _finish_write discard buffer(0x558c5813bb00 space 0x558c57db5048 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57db50b8 in 0x558c53a59960) _finish_write discard buffer(0x558c5813ba70 space 0x558c57db50b8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57ed1ba8 in 0x558c53a59960) _finish_write discard buffer(0x558c5813b8c0 space 0x558c57ed1ba8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec558 in 0x558c53a59960) _finish_write discard buffer(0x558c5813b950 space 0x558c580ec558 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c580ec5c8 in 0x558c53a59960) _finish_write discard buffer(0x558c5813b830 space 0x558c580ec5c8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c66c58 in 0x558c53a59960) _finish_write discard buffer(0x558c5813be60 space 0x558c58c66c58 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c66cc8 in 0x558c53a59960) _finish_write discard buffer(0x558c57f1dcb0 space 0x558c58c66cc8 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c58c66d38 in 0x558c53a59960) _finish_write discard buffer(0x558c57f1db90 space 0x558c58c66d38 0x0~2000 writing nocache) 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875b200 done 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:41.657+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875b200 [] 2019-12-07T19:34:41.657+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 64 dequeue_op 0x558c573358c0 prio 127 cost 0 latency 0.000301 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=323, reqid=client.4540.0:486, at_version=64'58, trim_to=0'0, roll_forward_to=64'56)) v2 pg pg[1.8s5( v 64'57 (0'0,64'57] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:41.657+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'57 (0'0,64'57] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=323, reqid=client.4540.0:486, at_version=64'58, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'57 (0'0,64'57] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] append_log log((0'0,64'57], crt=64'56) [64'58 (64'57) modify 1:10779340:::smithi13140796-32:head by client.4540.0:486 2019-12-07T19:34:41.654821+0000 0 ObjectCleanRegions clean_offsets: [0~1474560,1523712~18446744073708027903], clean_omap: 1, new_object: 0] 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'58 (0'0,64'58] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'57 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] add_log_entry 64'58 (64'57) modify 1:10779340:::smithi13140796-32:head by client.4540.0:486 2019-12-07T19:34:41.654821+0000 0 ObjectCleanRegions clean_offsets: [0~1474560,1523712~18446744073708027903], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'58 (0'0,64'58] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'57 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 58 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'58 (0'0,64'58] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'57 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.657+0000 7f8f0898a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 64'58, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c58500900 seq 60 2019-12-07T19:34:41.657+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:41.657+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:41.657+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x5a000~3000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write 5#1:10779340:::smithi13140796-32:head# 0x5a000~3000 - have 0x5a000 (368640) bytes fadvise_flags 0x20 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _do_write_big 0x5a000~3000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write txc 0x558c58500900 2 blobs 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate 0x3000/1000,3000,0 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 fbmap_alloc 0x558c53a72e00 allocate extent: 0xd35000~3000/1000,3000,0 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write prealloc [0xd35000~3000] 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5898bf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c57881e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5898bf10 blob([0xd35000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881e30 sbid 0x0)) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.blob(0x558c5898bf10) get_ref 0x0~2000 Blob(0x558c5898bf10 blob([0xd35000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c57881e30 sbid 0x0)) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.blob(0x558c5898bf10) get_ref init 0x2000, 1000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558c5898bf10 blob([0xd35000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558c57881e30 sbid 0x0)) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c57881e48 in 0x558c53a59960) _discard 0x0~2000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write initialize csum setting for new blob Blob(0x558c5788c690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558c5788c700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write blob Blob(0x558c5788c690 blob([0xd37000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5788c700 sbid 0x0)) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.blob(0x558c5788c690) get_ref 0x0~1000 Blob(0x558c5788c690 blob([0xd37000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558c5788c700 sbid 0x0)) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.blob(0x558c5788c690) get_ref init 0x1000, 1000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_alloc_write lex 0x5c000~1000: 0x0~1000 Blob(0x558c5788c690 blob([0xd37000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558c5788c700 sbid 0x0)) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.BufferSpace(0x558c5788c718 in 0x558c53a59960) _discard 0x0~1000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write extending size to 0x5d000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) dirty_range mark inline shard dirty 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x5a000~3000 = 0 2019-12-07T19:34:41.657+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:41.657+0000 7f8f0898a700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000064.00000000000000000058' <- 0000000064.00000000000000000058 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._fastinfo' <- _fastinfo 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c58500900 cost 26378 (3 ios * 4000 + 14378 bytes) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c58500900 onodes 0x558c5840c780 shared_blobs 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore.extentmap(0x558c5840c888) update inline shard 893 bytes from 47 extents 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 1368 (469 bytes onode + 2 bytes spanning blobs + 897 bytes inline extents) 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c58500900 allocated 0x[d35000~3000] released 0x[] 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 freelist allocate 0xd35000~3000 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 freelist _xor first_key 0xd00000 last_key 0xd00000 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500900 prepare 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_aio_submit txc 0x558c58500900 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'58 (0'0,64'58] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'57 crt=64'56 lcod 64'56 active mbc={} ps=[20~1,23~1]] op_applied version 64'58 2019-12-07T19:34:41.657+0000 7f8f0898a700 10 osd.7 64 dequeue_op 0x558c573358c0 finish 2019-12-07T19:34:41.657+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 64/41 ECSubWriteReply(tid=322, last_complete=64'57, committed=1, applied=1)) v2 -- 0x558c55271e00 con 0x558c54c64000 2019-12-07T19:34:41.657+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 557 ==== MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=324, reqid=client.4540.0:487, at_version=64'59, trim_to=0'0, roll_forward_to=64'56)) v2 ==== 2347+0+0 (crc 0 0 0) 0x558c57dff000 con 0x558c54c64000 2019-12-07T19:34:41.657+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57334f20 prio 127 cost 0 latency 0.000022 epoch 64 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=324, reqid=client.4540.0:487, at_version=64'59, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.657+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=324, reqid=client.4540.0:487, at_version=64'59, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) 2019-12-07T19:34:41.657+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.657+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=324, reqid=client.4540.0:487, at_version=64'59, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) queued 2019-12-07T19:34:41.657+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.657+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=324, reqid=client.4540.0:487, at_version=64'59, trim_to=0'0, roll_forward_to=64'56)) v2) prio 127 cost 0 e64) pg 0x558c57380000 2019-12-07T19:34:41.657+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57334f20 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=324, reqid=client.4540.0:487, at_version=64'59, trim_to=0'0, roll_forward_to=64'56)) v2 pg pg[1.8s5( v 64'58 (0'0,64'58] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 lcod 64'57 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:41.657+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.657+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'58 (0'0,64'58] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 lcod 64'57 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=324, reqid=client.4540.0:487, at_version=64'59, trim_to=0'0, roll_forward_to=64'56)) v2 2019-12-07T19:34:41.657+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'58 (0'0,64'58] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 lcod 64'57 active mbc={} ps=[20~1,23~1]] append_log log((0'0,64'58], crt=64'56) [64'59 (64'58) modify 1:10779340:::smithi13140796-32:head by client.4540.0:487 2019-12-07T19:34:41.654835+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:41.657+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'58 crt=64'56 lcod 64'57 active mbc={} ps=[20~1,23~1]] add_log_entry 64'59 (64'58) modify 1:10779340:::smithi13140796-32:head by client.4540.0:487 2019-12-07T19:34:41.654835+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.657+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'58 crt=64'56 lcod 64'57 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 59 2019-12-07T19:34:41.657+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'58 crt=64'56 lcod 64'57 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.657+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.657+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 64'59, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.657+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:41.657+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5875b500 seq 61 2019-12-07T19:34:41.657+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 3 keys 2019-12-07T19:34:41.657+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 3 keys = 0 2019-12-07T19:34:41.657+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:41.657+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:41.657+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:41.657+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.0000000064.00000000000000000059' <- 0000000064.00000000000000000059 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._fastinfo' <- _fastinfo 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.658+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:41.658+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875b500 cost 5997 (1 ios * 4000 + 1997 bytes) 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875b500 onodes 0x558c5840c780 shared_blobs 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore.extentmap(0x558c5840c888) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 1441 (542 bytes onode + 2 bytes spanning blobs + 897 bytes inline extents) 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875b500 allocated 0x[] released 0x[] 2019-12-07T19:34:41.658+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b500 prepare 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875b500 2019-12-07T19:34:41.658+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875b500 blocked by 0x558c58500900 aio_wait 2019-12-07T19:34:41.658+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 lua=64'58 crt=64'56 lcod 64'57 active mbc={} ps=[20~1,23~1]] op_applied version 64'59 2019-12-07T19:34:41.658+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57334f20 finish 2019-12-07T19:34:41.658+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.658+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500900 aio_wait 2019-12-07T19:34:41.658+0000 7f8f1d3b9700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c58500900 2019-12-07T19:34:41.658+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500900 io_done 2019-12-07T19:34:41.658+0000 7f8f1d3b9700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b500 io_done 2019-12-07T19:34:41.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.658+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2564 < min_flush_size 524288 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 2 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 2 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2587 < min_flush_size 524288 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2587 < min_flush_size 524288 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x57091e~a1b to file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x57091e 0x57091e~a1b to file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~550000 x_off 0x54091e 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x91e 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x339 and padding block with 0xcc7 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x571339 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.001972468s (0.001873197s flush + 0.000099271s kv commit) 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c58500900> 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500900 kv_submitted 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2537 < min_flush_size 524288 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c58500900 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c58500900 finishing 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c58500900 onodes 0x558c5840c780 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c57881e48 in 0x558c53a59960) _finish_write discard buffer(0x558c57da4510 space 0x558c57881e48 0x0~2000 writing nocache) 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5840c780 had 1 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore.BufferSpace(0x558c5788c718 in 0x558c53a59960) _finish_write discard buffer(0x558c57669950 space 0x558c5788c718 0x0~1000 writing nocache) 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c58500900 done 2019-12-07T19:34:41.660+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875b500 kv_submitted 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2560 < min_flush_size 524288 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 2560 < min_flush_size 524288 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c58500900 [] 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 64/41 ECSubWriteReply(tid=323, last_complete=64'58, committed=1, applied=1)) v2 -- 0x558c548cda00 con 0x558c54c64000 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x571339~a00 to file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x571339 0x571339~a00 to file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~550000 x_off 0x541339 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0x339 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.660+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0xd39 and padding block with 0x2c7 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x571d39 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.660+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000101775s (0.000000108s flush + 0.000101667s kv commit) 2019-12-07T19:34:41.660+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875b500> 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b500 kv_submitted 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875b500 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b500 finishing 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875b500 onodes 0x558c5840c780 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875b500 done 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:41.660+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875b500 [] 2019-12-07T19:34:41.660+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.660+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.660+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 64/41 ECSubWriteReply(tid=324, last_complete=64'59, committed=1, applied=1)) v2 -- 0x558c580b8200 con 0x558c54c64000 2019-12-07T19:34:41.660+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.660+0000 7f8f24457700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] <== osd.1 v2:172.21.15.131:6823/37078 558 ==== MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=325, reqid=client.4540.0:487, at_version=0'0, trim_to=0'0, roll_forward_to=64'59)) v2 ==== 910+0+0 (crc 0 0 0) 0x558c576c9000 con 0x558c54c64000 2019-12-07T19:34:41.661+0000 7f8f24457700 15 osd.7 64 enqueue_op 0x558c57335080 prio 127 cost 0 latency 0.000021 epoch 64 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=325, reqid=client.4540.0:487, at_version=0'0, trim_to=0'0, roll_forward_to=64'59)) v2 2019-12-07T19:34:41.661+0000 7f8f24457700 20 osd.7 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=325, reqid=client.4540.0:487, at_version=0'0, trim_to=0'0, roll_forward_to=64'59)) v2) prio 127 cost 0 e64) 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=325, reqid=client.4540.0:487, at_version=0'0, trim_to=0'0, roll_forward_to=64'59)) v2) prio 127 cost 0 e64) queued 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=325, reqid=client.4540.0:487, at_version=0'0, trim_to=0'0, roll_forward_to=64'59)) v2) prio 127 cost 0 e64) pg 0x558c57380000 2019-12-07T19:34:41.661+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57335080 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=325, reqid=client.4540.0:487, at_version=0'0, trim_to=0'0, roll_forward_to=64'59)) v2 pg pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] 2019-12-07T19:34:41.661+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] _handle_message: MOSDECSubOpWrite(1.8s5 64/41 ECSubWrite(tid=325, reqid=client.4540.0:487, at_version=0'0, trim_to=0'0, roll_forward_to=64'59)) v2 2019-12-07T19:34:41.661+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'56 active mbc={} ps=[20~1,23~1]] append_log log((0'0,64'59], crt=64'56) [] 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'59 active mbc={} ps=[20~1,23~1]] rollforward: entry=64'57 (64'56) modify 1:10779340:::smithi13140796-32:head by client.4540.0:485 2019-12-07T19:34:41.654499+0000 0 ObjectCleanRegions clean_offsets: [0~753664,1474560~18446744073708077055], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'59 active mbc={} ps=[20~1,23~1]] rollforward: entry=64'58 (64'57) modify 1:10779340:::smithi13140796-32:head by client.4540.0:486 2019-12-07T19:34:41.654821+0000 0 ObjectCleanRegions clean_offsets: [0~1474560,1523712~18446744073708027903], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'59 active mbc={} ps=[20~1,23~1]] rollforward: entry=64'59 (64'58) modify 1:10779340:::smithi13140796-32:head by client.4540.0:487 2019-12-07T19:34:41.654835+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:41.661+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'59 active mbc={} ps=[20~1,23~1]] append_log approx pg log length = 59 2019-12-07T19:34:41.661+0000 7f8f04982700 10 osd.7 pg_epoch: 64 pg[1.8s5( v 64'59 (0'0,64'59] local-lis/les=41/42 n=7 ec=18/18 lis/c=41/41 les/c/f=42/42/0 sis=41) [1,4,5,2,3,7]p1(0) r=5 lpr=41 luod=0'0 crt=64'59 active mbc={} ps=[20~1,23~1]] append_log transaction_applied = 1 2019-12-07T19:34:41.661+0000 7f8f04982700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:41.661+0000 7f8f04982700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:41.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) queue_transactions ch 0x558c54df6960 1.8s5_head 2019-12-07T19:34:41.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_create osr 0x558c57f06c80 = 0x558c5875b800 seq 62 2019-12-07T19:34:41.661+0000 7f8f04982700 15 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:41.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'._info' <- _info 2019-12-07T19:34:41.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:41.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 0x000000000000049f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:41.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:41.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_calc_cost 0x558c5875b800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:41.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_write_nodes txc 0x558c5875b800 onodes shared_blobs 2019-12-07T19:34:41.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finalize_kv txc 0x558c5875b800 allocated 0x[] released 0x[] 2019-12-07T19:34:41.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b800 prepare 2019-12-07T19:34:41.661+0000 7f8f04982700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish_io 0x558c5875b800 2019-12-07T19:34:41.661+0000 7f8f04982700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b800 io_done 2019-12-07T19:34:41.661+0000 7f8f04982700 10 osd.7 64 dequeue_op 0x558c57335080 finish 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread wake 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_apply_kv onode 0x558c5738db80 had 1 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs _fsync 0x558c54ea43c0 file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs _flush 0x558c54ea43c0 0x571d39~471 to file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs _flush_range 0x558c54ea43c0 pos 0x571d39 0x571d39~471 to file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluefs _flush_range file now file(ino 16 size 0x580000 mtime 2019-12-07T19:34:41.658231+0000 allocated 580000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~550000]) 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluefs _flush_range in 1:0xacd160000~550000 x_off 0x541d39 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluefs _flush_range using partial tail 0xd39 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluefs _flush_range caching tail of 0x1aa and padding block with 0xe56 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluefs _flush_range h 0x558c54ea43c0 pos now 0x5721aa 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 2019-12-07T19:34:41.661+0000 7f8f149a2700 10 bluefs wait_for_aio 0x558c54ea43c0 done in 0.000000 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluefs flush_bdev 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread committed 1 cleaned 0 in 0.000116769s (0.000000081s flush + 0.000116688s kv commit) 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread wake 2019-12-07T19:34:41.661+0000 7f8f149a2700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_sync_thread sleep 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread kv_committed <0x558c5875b800> 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:41.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b800 kv_submitted 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_committed_kv txc 0x558c5875b800 2019-12-07T19:34:41.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_state_proc txc 0x558c5875b800 finishing 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish 0x558c5875b800 onodes 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish txc 0x558c5875b800 done 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _txc_finish osr 0x558c57f06c80 q now empty 2019-12-07T19:34:41.661+0000 7f8f199ac700 10 bluestore(/var/lib/ceph/osd/ceph-7) _txc_release_alloc(sync) 0x558c5875b800 [] 2019-12-07T19:34:41.661+0000 7f8f199ac700 10 fbmap_alloc 0x558c53a72e00 release done 2019-12-07T19:34:41.661+0000 7f8f199ac700 20 bluestore(/var/lib/ceph/osd/ceph-7) _kv_finalize_thread sleep 2019-12-07T19:34:41.661+0000 7f8f04982700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.661+0000 7f8f0898a700 1 -- [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 64/41 ECSubWriteReply(tid=325, last_complete=64'59, committed=1, applied=1)) v2 -- 0x558c58114200 con 0x558c54c64000 2019-12-07T19:34:41.661+0000 7f8f0898a700 20 osd.7 op_wq(0) _process empty q, waiting 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:41.706+0000 7f8f191ab700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0259408 2019-12-07T19:34:41.706+0000 7f8f191ab700 5 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.756+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.806+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 10 osd.7 64 tick_without_osd_lock 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 20 bluestore(/var/lib/ceph/osd/ceph-7) statfs store_statfs(0x163f2c8000/0x40000000/0x1680000000, data 0x1013dd4/0xd36000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 25 osd.7 64 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476453+0000 last_rx_front 2019-12-07T19:34:38.476418+0000 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 25 osd.7 64 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476352+0000 last_rx_front 2019-12-07T19:34:38.476394+0000 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 25 osd.7 64 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476378+0000 last_rx_front 2019-12-07T19:34:38.476470+0000 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 25 osd.7 64 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476367+0000 last_rx_front 2019-12-07T19:34:38.476505+0000 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 25 osd.7 64 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.271181+0000 last_tx 2019-12-07T19:34:38.476103+0000 last_rx_back 2019-12-07T19:34:38.476409+0000 last_rx_front 2019-12-07T19:34:38.476432+0000 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 20 osd.7 64 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 10 osd.7 64 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 20 osd.7 64 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:41.815+0000 7f8f1fbbe700 10 osd.7 64 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:41.856+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.906+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:41.956+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:42.006+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:42.056+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:42.106+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:42.146+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] <== osd.5 172.21.15.109:0/36936 17 ==== osd_ping(ping e64 up_from 13 ping_stamp 2019-12-07T19:34:42.147149+0000/65.613472604s send_stamp 65.613472604s delta_ub 0.030988787s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c5795d800 con 0x558c54c5a400 2019-12-07T19:34:42.146+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] <== osd.5 172.21.15.109:0/36936 17 ==== osd_ping(ping e64 up_from 13 ping_stamp 2019-12-07T19:34:42.147149+0000/65.613472604s send_stamp 65.613472604s delta_ub 0.030988787s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558c57950c00 con 0x558c54c5ac00 2019-12-07T19:34:42.146+0000 7f8f24c58700 20 osd.7 64 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030693517s,0.030988787s]) 2019-12-07T19:34:42.146+0000 7f8f24c58700 1 -- [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e64 up_from 13 ping_stamp 2019-12-07T19:34:42.147149+0000/65.613472604s send_stamp 65.582779087s delta_ub 18446744073.678858099s) v5 -- 0x558c53a5e600 con 0x558c54c5a400 2019-12-07T19:34:42.146+0000 7f8f23c56700 20 osd.7 64 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.030632764s,0.030988787s]) 2019-12-07T19:34:42.146+0000 7f8f23c56700 1 -- [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e64 up_from 13 ping_stamp 2019-12-07T19:34:42.147149+0000/65.613472604s send_stamp 65.582839840s delta_ub 18446744073.678918852s) v5 -- 0x558c57950a00 con 0x558c54c5ac00 2019-12-07T19:34:42.156+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:42.207+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:42.224+0000 7f8f21451700 10 osd.7 64 tick 2019-12-07T19:34:42.224+0000 7f8f21451700 10 osd.7 64 do_waiters -- start 2019-12-07T19:34:42.224+0000 7f8f21451700 10 osd.7 64 do_waiters -- finish 2019-12-07T19:34:42.224+0000 7f8f21451700 20 osd.7 64 tick last_purged_snaps_scrub 2019-12-07T19:34:38.205126+0000 next 2019-12-07T19:35:38.205126+0000 2019-12-07T19:34:42.258+0000 7f8f191ab700 20 bluestore.MempoolThread(0x558c53b08a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 1747704 data_alloc: 671088640 data_used: 0